@tangle-network/agent-eval 0.86.0 → 0.90.0

package/dist/{agent-profile-D0PBIWlV.d.ts → pre-registration-DELOEJ8v.d.ts}

@@ -1,6 +1,7 @@
-import { A as AgentEvalError } from './errors-Dwqw-T_m.js';
-import { R as RunRecord } from './run-record-De9VarXR.js';
+import { A as AgentEvalError } from './errors-CzMUYo7b.js';
+import { R as RunRecord } from './run-record-e7vj1uZQ.js';
 import { TCloud } from '@tangle-network/tcloud';
+import { D as DefaultVerdict } from './verdict-C9MlYujm.js';
 
 /**
  * Backend-integrity guard: distinguish "agent failed" from "eval ran against
@@ -216,7 +217,10 @@ interface RequirementCheck {
     /** Human-readable evidence for the verdict. */
     evidence: string[];
 }
-interface CompletionVerdict {
+/** Extends the substrate verdict spine: `valid` = `fullyComplete` and
+ *  `score` = `completionRate` — derived in `completionVerdict()`, the one
+ *  place those equalities hold by construction. */
+interface CompletionVerdict extends DefaultVerdict {
     taskId: string;
     requirements: RequirementCheck[];
     /** satisfied / total requirements. */
@@ -224,6 +228,17 @@ interface CompletionVerdict {
     /** Every requirement satisfied. */
     fullyComplete: boolean;
 }
+/**
+ * Construct a `CompletionVerdict` from the per-requirement checks, deriving
+ * `completionRate` / `fullyComplete` and the spine fields (`valid` =
+ * `fullyComplete`, `score` = `completionRate`) in one place. Throws on zero
+ * requirements — a verdict over nothing is a misconfiguration, mirroring
+ * `verifyCompletion`'s gold-spec guard.
+ */
+declare function completionVerdict(input: {
+    taskId: string;
+    requirements: RequirementCheck[];
+}): CompletionVerdict;
 /**
  * Decides whether a produced item's content actually fulfils a requirement.
  * Injected so the structural verifier stays pure and unit-testable; the
@@ -373,4 +388,129 @@ interface AgentProfile {
  */
 declare function agentProfileHash(profile: AgentProfile): string;
 
-export { type AgentProfile as A, type BackendIntegrityReport as B, type CompletionRequirement as C, type LlmCorrectnessCheckerOpts as L, type ProducedState as P, type RuntimeEventLike as R, type SatisfiedBy as S, type TaskGold as T, type ValidationContext as V, type CompletionVerdict as a, type CorrectnessChecker as b, type Artifact as c, type ArtifactEventLike as d, type ArtifactValidator as e, BackendIntegrityError as f, type ProducedProposal as g, type ProposalEventLike as h, type RequirementCheck as i, type ToolCallEventLike as j, type ValidationIssue as k, type ValidationResult as l, agentProfileHash as m, assertRealBackend as n, byteLengthRange as o, composeValidators as p, containsAll as q, createLlmCorrectnessChecker as r, createTokenRecallChecker as s, extractProducedState as t, jsonHasKeys as u, parseCorrectnessResponse as v, regexMatch as w, summarizeBackendIntegrity as x, verifyCompletion as y };
+/**
+ * Pre-registered hypotheses — declare what you're testing BEFORE the
+ * run, check it AFTER. Prevents p-hacking, optional stopping, and the
+ * "we ran until it looked good" failure mode.
+ *
+ * Manifest is a plain JSON-friendly object. Sign it with a content hash
+ * + timestamp; the registered record becomes immutable. Post-run,
+ * evaluate the manifest against observed results — the library refuses
+ * to let you re-interpret a different metric as the declared one.
+ */
+interface HypothesisManifest {
+    id: string;
+    /** Human prose — goes into the audit trail. */
+    hypothesis: string;
+    /** Metric the hypothesis claims to move. */
+    metric: string;
+    /** 'increase' = candidate should score higher than baseline; 'decrease' = lower. */
+    direction: 'increase' | 'decrease';
+    /** Minimum effect size to count (same units as the metric). */
+    minEffect: number;
+    /** Alpha threshold. */
+    alpha: number;
+    /** Target statistical power at which sample size was pre-computed. */
+    power: number;
+    /** Declared N per arm before running. */
+    preRegisteredN: number;
+    /** ISO8601 timestamp the manifest was registered. */
+    registeredAt: string;
+    /** Optional identifiers to tie into the trace corpus. */
+    baselineLabel?: string;
+    candidateLabel?: string;
+}
+/**
+ * Identifier for the hashing scheme used to produce `contentHash`.
+ *
+ * `'sha256-content'` — sha256 hex over the canonicalized manifest with
+ * the `contentHash` and `algo` fields stripped. Held as a string union
+ * so future schemes can be added without breaking parsers; SignedManifest
+ * values without `algo` deserialize cleanly because the field is optional.
+ */
+type SignedManifestAlgo = 'sha256-content';
+interface SignedManifest extends HypothesisManifest {
+    /** sha256 hex of canonicalized manifest (everything except contentHash and algo). */
+    contentHash: string;
+    /**
+     * Algorithm string describing how `contentHash` was produced.
+     *
+     * Optional on the type so serialized manifests without it still parse,
+     * but ALWAYS populated by {@link signManifest}. Consumers that want to
+     * enforce a known algorithm should reject manifests where this field
+     * is missing or unrecognized.
+     */
+    algo?: SignedManifestAlgo;
+}
+interface HypothesisResult {
+    manifest: SignedManifest;
+    observedN: number;
+    observedEffect: number;
+    observedPValue: number;
+    /** True iff the observed effect hits the pre-declared direction with
+     *  magnitude ≥ minEffect AND p < alpha. */
+    confirmed: boolean;
+    /** Enumerated reasons the hypothesis was rejected (each a machine-tag). */
+    rejectionReasons: Array<'wrong_direction' | 'effect_too_small' | 'not_significant' | 'undersampled'>;
+    notes?: string;
+}
+/**
+ * Deterministic JSON canonicalization — sort object keys recursively.
+ *
+ * Two semantically-equal objects produce byte-identical canonicalized output;
+ * this is what makes a content-hash stable across encoders, key insertion
+ * orders, and runtime versions. Exported for any consumer that needs the same
+ * canonicalization guarantee outside the manifest-signing path (e.g., signing
+ * an artifact bundle, hashing a dataset version, etc.).
+ */
+declare function canonicalize(v: unknown): unknown;
+/**
+ * SHA-256 hex (full 64 chars) over the canonicalized JSON encoding of `obj`.
+ *
+ * The same primitive `signManifest` and `verifyManifest` are built on, exposed
+ * directly so consumers signing arbitrary structured content (artifact bundles,
+ * production packets, dataset manifests, etc.) don't have to re-derive
+ * canonicalize+sha256 from scratch.
+ *
+ * Stable across:
+ *   - object key insertion order (canonicalization sorts keys recursively)
+ *   - encoder choice (UTF-8 via TextEncoder, fixed)
+ *   - runtime (uses the Web Crypto subtle digest, present in Node ≥18 and browsers)
+ *
+ * Named `hashJson` to disambiguate from `prompt-registry.ts`'s `hashContent`,
+ * which takes a string input and returns a truncated 12-char prompt id.
+ * Use `hashJson` when you mean "canonicalize then hash."
+ *
+ * @example
+ *   const hash = await hashJson({ id: '1', kind: 'spec' })
+ *   // 'a3f1...' (64 hex chars)
+ */
+declare function hashJson<T>(obj: T): Promise<string>;
+/**
+ * Sign a manifest with a SHA-256 content hash.
+ *
+ * The hash covers the canonicalized manifest with the `contentHash`
+ * and `algo` fields stripped; this lets verifiers re-sign the rest and
+ * compare. Returned manifest always carries `algo: 'sha256-content'`
+ * so downstream consumers can identify the scheme; manifests without
+ * `algo` still verify because it is stripped before hashing on both sides.
+ */
+declare function signManifest(m: HypothesisManifest): Promise<SignedManifest>;
+/**
+ * Verify that a signed manifest has not been tampered with.
+ *
+ * Strips `contentHash` and `algo` before re-signing so manifests without
+ * `algo` verify identically to ones that carry it.
+ */
+declare function verifyManifest(m: SignedManifest): Promise<boolean>;
+/**
+ * Evaluate a pre-registered hypothesis against observed results.
+ * Mechanical — no re-interpretation permitted.
+ */
+declare function evaluateHypothesis(manifest: SignedManifest, observed: {
+    n: number;
+    effect: number;
+    pValue: number;
+}): Promise<HypothesisResult>;
+
+export { type AgentProfile as A, type BackendIntegrityReport as B, type CompletionRequirement as C, hashJson as D, jsonHasKeys as E, parseCorrectnessResponse as F, regexMatch as G, type HypothesisManifest as H, signManifest as I, summarizeBackendIntegrity as J, verifyCompletion as K, type LlmCorrectnessCheckerOpts as L, verifyManifest as M, type ProducedState as P, type RuntimeEventLike as R, type SignedManifest as S, type TaskGold as T, type ValidationContext as V, type CompletionVerdict as a, type CorrectnessChecker as b, type Artifact as c, type ArtifactEventLike as d, type ArtifactValidator as e, BackendIntegrityError as f, type HypothesisResult as g, type ProducedProposal as h, type ProposalEventLike as i, type RequirementCheck as j, type SatisfiedBy as k, type SignedManifestAlgo as l, type ToolCallEventLike as m, type ValidationIssue as n, type ValidationResult as o, agentProfileHash as p, assertRealBackend as q, byteLengthRange as r, canonicalize as s, completionVerdict as t, composeValidators as u, containsAll as v, createLlmCorrectnessChecker as w, createTokenRecallChecker as x, evaluateHypothesis as y, extractProducedState as z };

package/dist/{provenance-DPpNIOJD.d.ts → provenance-LnqRT0sS.d.ts}

@@ -1,9 +1,9 @@
-import { o as Mutator, I as ImprovementDriver, S as Scenario, G as Gate, k as GateResult, j as GateContext, g as CampaignResult, M as MutableSurface, c as GateDecision } from './types-D7lLRYe9.js';
-import { R as RedTeamCase } from './red-team-DW9Ca_tj.js';
-import { R as RunRecord } from './run-record-De9VarXR.js';
+import { o as Mutator, I as ImprovementDriver, S as Scenario, G as Gate, k as GateResult, j as GateContext, g as CampaignResult, M as MutableSurface, c as GateDecision } from './types-BU-7W85F.js';
+import { R as RedTeamCase } from './red-team-BXHil6c8.js';
+import { R as RunRecord } from './run-record-e7vj1uZQ.js';
 import { D as Direction } from './pareto-E-pembql.js';
-import { a as PairedBootstrapResult } from './statistics-CnC1FMbx.js';
-import { b as RunCampaignOptions, C as CampaignStorage } from './run-improvement-loop-CNqQckTj.js';
+import { a as PairedBootstrapResult } from './statistics-C7PozGrZ.js';
+import { b as RunCampaignOptions, C as CampaignStorage } from './run-improvement-loop-5z_l5zDz.js';
 import { HostedClient, TraceSpanEvent } from './hosted/index.js';
 
 /**

package/dist/{red-team-DW9Ca_tj.d.ts → red-team-BXHil6c8.d.ts}

@@ -1,4 +1,4 @@
-import { a as DatasetScenario, b as Dataset } from './dataset-B2kL-fSM.js';
+import { a as DatasetScenario, b as Dataset } from './dataset-BbGkaN2I.js';
 import { T as TraceStore } from './store-CKUAgsJz.js';
 
 /**

package/dist/{release-report-hlNtD12q.d.ts → release-report-euXIV_Sk.d.ts}

@@ -1,6 +1,6 @@
-import { D as DatasetSplit, c as DatasetManifest, a as DatasetScenario } from './dataset-B2kL-fSM.js';
-import { m as GateDecision } from './summary-report-Db0dDSWP.js';
-import { R as RunRecord, a as RunSplitTag } from './run-record-De9VarXR.js';
+import { D as DatasetSplit, c as DatasetManifest, a as DatasetScenario } from './dataset-BbGkaN2I.js';
+import { m as GateDecision } from './summary-report-DGmUucwQ.js';
+import { R as RunRecord, a as RunSplitTag } from './run-record-e7vj1uZQ.js';
 
 /**
  * Release confidence gate.

package/dist/reporting.d.ts

@@ -1,14 +1,14 @@
-export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-CnEl9Jc8.js';
-export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, f as ReleaseConfidenceScorecard, g as ReleaseConfidenceStatus, h as ReleaseConfidenceThresholds, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-hlNtD12q.js';
+export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from './rubric-predictive-validity-Cy_W-hWZ.js';
+export { B as BootstrapOptions, a as BootstrapResult, J as JudgeReplayGateArgs, R as ReleaseConfidenceAxis, b as ReleaseConfidenceAxisName, c as ReleaseConfidenceInput, d as ReleaseConfidenceIssue, e as ReleaseConfidenceMetrics, f as ReleaseConfidenceScorecard, g as ReleaseConfidenceStatus, h as ReleaseConfidenceThresholds, i as ReleaseTraceEvidence, j as RenderReleaseReportOptions, V as Verdict, k as assertReleaseConfidence, l as bootstrapCi, m as evaluateReleaseConfidence, n as judgeReplayGate, r as renderReleaseReport } from './release-report-euXIV_Sk.js';
 export { I as InterimReleaseConfidence, a as InterimReleaseConfidenceInput, P as PairedEvalueOptions, b as PairedEvalueSequence, c as PairedEvalueStep, S as SequentialDecision, e as evaluateInterimReleaseConfidence, p as pairedEvalueSequence } from './sequential-5iSVfzl2.js';
-export { P as PairedBootstrapOptions, a as PairedBootstrapResult, b as benjaminiHochberg, p as pairedBootstrap, w as wilcoxonSignedRank } from './statistics-CnC1FMbx.js';
-export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-Db0dDSWP.js';
-import './run-record-De9VarXR.js';
-import './errors-Dwqw-T_m.js';
+export { P as PairedBootstrapOptions, a as PairedBootstrapResult, b as benjaminiHochberg, p as pairedBootstrap, w as wilcoxonSignedRank } from './statistics-C7PozGrZ.js';
+export { G as GainDistributionBin, a as GainDistributionFigureSpec, b as GainDistributionOptions, P as ParetoFigureSpec, c as ParetoPoint, R as RESEARCH_REPORT_HARD_PAIR_FLOOR, d as ResearchReport, e as ResearchReportCandidate, f as ResearchReportDecision, g as ResearchReportMethodology, h as ResearchReportOptions, i as ResearchReportRecommendation, S as SummaryTable, j as SummaryTableOptions, k as SummaryTableRow, l as gainHistogram, p as paretoChart, r as researchReport, s as summaryTable } from './summary-report-DGmUucwQ.js';
+import './run-record-e7vj1uZQ.js';
+import './errors-CzMUYo7b.js';
 import './schema-m0gsnbt3.js';
 import './outcome-store-rnXLEqSn.js';
-import './dataset-B2kL-fSM.js';
-import './judge-calibration-DilmB3Ml.js';
+import './dataset-BbGkaN2I.js';
+import './judge-calibration-0p2QcWNE.js';
 import './types-Croy5h7V.js';
 import '@tangle-network/tcloud';
 import './failure-cluster-CL7IVgkJ.js';

package/dist/reporting.js

@@ -4,7 +4,7 @@ import {
   evaluateReleaseConfidence,
   judgeReplayGate,
   renderReleaseReport
-} from "./chunk-FZWAFVAA.js";
+} from "./chunk-4FBZZIYD.js";
 import {
   rubricPredictiveValidity
 } from "./chunk-YRZ4M5GS.js";
@@ -18,12 +18,12 @@ import {
   paretoChart,
   researchReport,
   summaryTable
-} from "./chunk-Z7VFTS2J.js";
+} from "./chunk-CY6U5S3X.js";
 import {
   benjaminiHochberg,
   pairedBootstrap,
   wilcoxonSignedRank
-} from "./chunk-IDVBLYCY.js";
+} from "./chunk-LMZQ2Z4U.js";
 import "./chunk-VSMTAMNK.js";
 import "./chunk-3BFEG2F6.js";
 import "./chunk-PZ5AY32C.js";

package/dist/{researcher-BLPHBbNV.d.ts → researcher-DE6Gpnb4.d.ts}

@@ -1,8 +1,8 @@
-import { a as RunSplitTag, b as RunTokenUsage, c as RunJudgeMetadata, J as JudgeScoresRecord, A as AgentProfileCell, d as AgentProfileCellInput, R as RunRecord } from './run-record-De9VarXR.js';
-import { L as LlmClientOptions, a as LlmRouteRequirements } from './llm-client-CuUg2Mn3.js';
-import { h as ResearchReportOptions, d as ResearchReport, m as GateDecision } from './summary-report-Db0dDSWP.js';
+import { a as RunSplitTag, b as RunTokenUsage, c as RunJudgeMetadata, J as JudgeScoresRecord, A as AgentProfileCell, d as AgentProfileCellInput, R as RunRecord } from './run-record-e7vj1uZQ.js';
+import { L as LlmClientOptions, a as LlmRouteRequirements } from './llm-client-BeEcAokY.js';
+import { h as ResearchReportOptions, d as ResearchReport, m as GateDecision } from './summary-report-DGmUucwQ.js';
 import { T as TraceEmitter, R as RunCompleteHook } from './emitter-DEZwY14K.js';
-import { R as RunIntegrityExpectations, a as RunIntegrityReport } from './integrity-CJzrpUua.js';
+import { R as RunIntegrityExpectations, a as RunIntegrityReport } from './integrity-VJ9A7aST.js';
 import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
 import { F as FailureClass } from './schema-m0gsnbt3.js';
 import { T as TraceStore } from './store-CKUAgsJz.js';