@tangle-network/agent-eval 0.171.0 → 0.172.1

package/dist/{matrix-DiHmUobV.d.ts → matrix-Ch8JO1pG.d.ts}

@@ -1,5 +1,5 @@
 import { p as CostProvenance } from "./cost-ledger-DbQdN3nO.js";
-import { w as JudgeScore } from "./types-JHMOqZI4.js";
+import { w as JudgeScore } from "./types-nokrtr7M.js";
 import { o as MatrixResult } from "./index-DNgf5gyG.js";
 import { AgentProfile } from "@tangle-network/agent-interface";
 //#region src/multishot/types.d.ts
@@ -398,4 +398,4 @@ interface RunMultishotMatrixResult {
 declare function runMultishotMatrix<TPersona extends MultishotPersona>(opts: RunMultishotMatrixOptions<TPersona>): Promise<RunMultishotMatrixResult>;
 //#endregion
 export { MultishotToolExecutor as A, MultishotFatalToolError as C, MultishotShape as D, MultishotResult as E, assertMultishotShotResult as F, MultishotTransportRequest as M, MultishotTransportResponse as N, MultishotShotResultError as O, MultishotTransportToolCall as P, MultishotDriverEmptyError as S, MultishotPersona as T, JudgeRunResult as _, MultishotCellOutput as a, runJudge as b, RunMultishotMatrixResult as c, MultishotShot as d, RunMultishotOptions as f, JudgeDimension as g, JudgeConfig as h, ConversationJudgeInput as i, MultishotTransport as j, MultishotToolDefinition as k, computeCellComposite as l, DEFAULT_JUDGE_MODEL as m, CellCompositeInput as n, MultishotJudges as o, runMultishot as p, CellCompositeScore as r, RunMultishotMatrixOptions as s, ArtifactJudgeInput as t, runMultishotMatrix as u, renderDimensions as v, MultishotMessage as w, MultishotArtifact as x, renderJsonFooter as y };
-//# sourceMappingURL=matrix-DiHmUobV.d.ts.map
+//# sourceMappingURL=matrix-Ch8JO1pG.d.ts.map

package/dist/{matrix-DiHmUobV.d.ts.map → matrix-Ch8JO1pG.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"matrix-DiHmUobV.d.ts","names":[],"sources":["../src/multishot/types.ts","../src/multishot/judges.ts","../src/multishot/multishot.ts","../src/multishot/matrix.ts"],"mappings":";;;;;UAIiB;EACf;EACA;EACA;EACA,YAAY;IAAQ;IAAY;IAAc,MAAM;;;UAGrC;EACf;EACA;EACA;IAAc;IAAc,MAAM;;EAClC;;UAGe;EACf,YAAY;EACZ,WAAW;EACX;EACA;;;EAGA;;;;;;;;EAQA,iBAAiB;;UAGF;EACf;EACA;IACE;IACA;IACA,YAAY;;;;;;UAOC;EACf;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;;EAGlC;;;;EAIA;;;;;;;;KASU,sBACV,KAAK,8BACF,QAAQ;KAED,yBACV,MAAM,yBACN;;EAEE,WAAW;EACX,SAAS;MAER;EAAU;EAAiB;;UAEf;;EAEf;;GAEC;;;;;;;;UASc,eAAe,iBAAiB;;EAE/C,eAAe,SAAS;;;EAGxB,2BAA2B,SAAS;;cAGzB,kCAAkC;WACjB;EAA5B,YAA4B;;cAMjB,gCAAgC;EAC3C,YAAY;;cAMD,iCAAiC;EAC5C,YAAY;;;;;;;;;;;;;;;;;;;iBAyBE,0BAA0B,yBAAyB,SAAS;;;cCvI/D;UAEI;;EAEf;;EAEA;;UAGe,YAAY;;EAE3B;;;EAGA,WAAW;;EAEX;;EAEA,YAAY;;EAEZ;;;EAGA,cAAc,OAAO;;EAErB;;UAGe;;EAEf,OAAO;;EAEP,MAAM;;iBAGc,SAAS,QAC7B,OAAO,YAAY,SACnB,OAAO,SACN,QAAQ;;;iBAsIK,iBAAiB,eAAe;;iBAKhC,iBAAiB,eAAe;;;UCxK/B,oBAAoB,iBAAiB;EACpD,SAAS;EACT,SAAS;;;EAGT,QAAQ,eAAe;;EAEvB,QAAQ;;EAER,gBAAgB,eAAe;;;;EAI/B,mBAAmB;EACnB;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA,gBAAgB;;;;EAIhB,iBAAiB;;;EAGjB,gBAAgB;EAChB,SAAS;;;;;;;;;;;KAYC,cAAc,iBAAiB,qBACzC,MAAM,oBAAoB,cACvB,QAAQ;;;;;;;;;;;;;;;;;iBA2BS,aAAa,iBAAiB,kBAClD,MAAM,oBAAoB,YACzB,QAAQ;;;UClFM,uBAAuB,iBAAiB;EACvD,YAAY;EACZ,SAAS;;UAGM,mBAAmB,iBAAiB;EACnD,UAAU;EACV,SAAS;;UAGM,gBAAgB,iBAAiB;;EAEhD,cAAc,YAAY,uBAAuB;;EAEjD,aAAa,YAAY,mBAAmB;;EAE5C,iBAAiB,YAAY,mBAAmB;;EAEhD;;EAEA;;UAGe;EACf;EACA,cAAc;EACd;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;EAEF;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;;UAIa,0BAA0B,iBAAiB;;EAE1D,UAAU;IAAQ;IAAY,OAAO;;;EAErC,UAAU;;;EAGV,QAAQ,eAAe;;EAEvB,QAAQ,gBAAgB;;EAExB,QAAQ;;EAER,gBAAgB,eAAe;;EAE/B,mBAAmB;;EAEnB;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,gBAAgB;;EAEhB,iBAAiB;;;EAGjB,gBAAgB;;;;;;;;;;;;;;;EAehB,UAAU,cAAc;;;;;UAMT;EACf;EACA;EACA;;UAoBe;EACf,cAAc;;EAEd,cAAc,cAAc;;EAE5B,iBAAiB,cAAc;;;;;;;iBAQjB,qBAAqB,OAAO;EAC1C;EACA;EACA;EACA;;UAuBe;EACf,QAAQ,aAAa;;iBAGD,mBAAmB,iBAAiB,kBACxD,MAAM,0BAA0B,YAC/B,QAAQ"}
+{"version":3,"file":"matrix-Ch8JO1pG.d.ts","names":[],"sources":["../src/multishot/types.ts","../src/multishot/judges.ts","../src/multishot/multishot.ts","../src/multishot/matrix.ts"],"mappings":";;;;;UAIiB;EACf;EACA;EACA;EACA,YAAY;IAAQ;IAAY;IAAc,MAAM;;;UAGrC;EACf;EACA;EACA;IAAc;IAAc,MAAM;;EAClC;;UAGe;EACf,YAAY;EACZ,WAAW;EACX;EACA;;;EAGA;;;;;;;;EAQA,iBAAiB;;UAGF;EACf;EACA;IACE;IACA;IACA,YAAY;;;;;;UAOC;EACf;EACA,UAAU,MAAM;EAChB,QAAQ;EACR;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA;IAAY;IAAc;;;UAGX;EACf;IAAW;IAAyB,aAAa;;EACjD;IAAU;IAAwB;;;;EAGlC;;;;EAIA;;;;;;;;KASU,sBACV,KAAK,8BACF,QAAQ;KAED,yBACV,MAAM,yBACN;;EAEE,WAAW;EACX,SAAS;MAER;EAAU;EAAiB;;UAEf;;EAEf;;GAEC;;;;;;;;UASc,eAAe,iBAAiB;;EAE/C,eAAe,SAAS;;;EAGxB,2BAA2B,SAAS;;cAGzB,kCAAkC;WACjB;EAA5B,YAA4B;;cAMjB,gCAAgC;EAC3C,YAAY;;cAMD,iCAAiC;EAC5C,YAAY;;;;;;;;;;;;;;;;;;;iBAyBE,0BAA0B,yBAAyB,SAAS;;;cCvI/D;UAEI;;EAEf;;EAEA;;UAGe,YAAY;;EAE3B;;;EAGA,WAAW;;EAEX;;EAEA,YAAY;;EAEZ;;;EAGA,cAAc,OAAO;;EAErB;;UAGe;;EAEf,OAAO;;EAEP,MAAM;;iBAGc,SAAS,QAC7B,OAAO,YAAY,SACnB,OAAO,SACN,QAAQ;;;iBAsIK,iBAAiB,eAAe;;iBAKhC,iBAAiB,eAAe;;;UCxK/B,oBAAoB,iBAAiB;EACpD,SAAS;EACT,SAAS;;;EAGT,QAAQ,eAAe;;EAEvB,QAAQ;;EAER,gBAAgB,eAAe;;;;EAI/B,mBAAmB;EACnB;EACA;EACA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;EAGA,gBAAgB;;;;EAIhB,iBAAiB;;;EAGjB,gBAAgB;EAChB,SAAS;;;;;;;;;;;KAYC,cAAc,iBAAiB,qBACzC,MAAM,oBAAoB,cACvB,QAAQ;;;;;;;;;;;;;;;;;iBA2BS,aAAa,iBAAiB,kBAClD,MAAM,oBAAoB,YACzB,QAAQ;;;UClFM,uBAAuB,iBAAiB;EACvD,YAAY;EACZ,SAAS;;UAGM,mBAAmB,iBAAiB;EACnD,UAAU;EACV,SAAS;;UAGM,gBAAgB,iBAAiB;;EAEhD,cAAc,YAAY,uBAAuB;;EAEjD,aAAa,YAAY,mBAAmB;;EAE5C,iBAAiB,YAAY,mBAAmB;;EAEhD;;EAEA;;UAGe;EACf;EACA,cAAc;EACd;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;EAEF;IACE,aAAa,MAAM;MAAe;MAAc;;IAChD;;;UAIa,0BAA0B,iBAAiB;;EAE1D,UAAU;IAAQ;IAAY,OAAO;;;EAErC,UAAU;;;EAGV,QAAQ,eAAe;;EAEvB,QAAQ,gBAAgB;;EAExB,QAAQ;;EAER,gBAAgB,eAAe;;EAE/B,mBAAmB;;EAEnB;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;;;EAIA,gBAAgB;;EAEhB,iBAAiB;;;EAGjB,gBAAgB;;;;;;;;;;;;;;;EAehB,UAAU,cAAc;;;;;UAMT;EACf;EACA;EACA;;UAoBe;EACf,cAAc;;EAEd,cAAc,cAAc;;EAE5B,iBAAiB,cAAc;;;;;;;iBAQjB,qBAAqB,OAAO;EAC1C;EACA;EACA;EACA;;UAuBe;EACf,QAAQ,aAAa;;iBAGD,mBAAmB,iBAAiB,kBACxD,MAAM,0BAA0B,YAC/B,QAAQ"}

package/dist/meta-eval/index.d.ts

@@ -1,7 +1,8 @@
 import { f as Run } from "../schema-DID1Cqct.js";
-import { s as TraceStore } from "../store-Cq9oOrI1.js";
 import { a as ContinuousCalibrationResult, n as CandidateScore, o as GoldenItem, r as ContinuousAgreement, t as CalibrationResult } from "../judge-calibration-C5CbMYce.js";
 import { n as SeriesConvergenceResult, o as CorpusAgreementReport, t as SeriesConvergenceOptions } from "../series-convergence-D9WgpXGi.js";
+import { r as LedgerHash } from "../canonical-CFpojCN5.js";
+import { s as TraceStore } from "../store-Cq9oOrI1.js";
 import { a as OutcomeFilter, i as InMemoryOutcomeStore, n as FileSystemOutcomeStore, o as OutcomeStore, r as FileSystemOutcomeStoreOptions, t as DeploymentOutcome } from "../outcome-store-BYHIuO0e.js";
 import { a as rubricPredictiveValidity, i as RubricRanking, n as RubricPredictiveValidityInput, r as RubricPredictiveValidityReport, t as RubricOutcomePair } from "../rubric-predictive-validity-DluJLCKQ.js";
 //#region src/meta-eval/correlation-study.d.ts
@@ -85,6 +86,165 @@ interface CalibrationPair {
 }
 declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
 //#endregion
+//#region src/meta-eval/plants.d.ts
+/**
+ * How a plant item was authored wrong. The class is reported separately in
+ * {@link CatchRateReport.byKind} because a grader is routinely sharp on one
+ * and blind to another.
+ */
+type PlantKind =
+/** A load-bearing value is altered: a number off by one, a comparison flipped. */
+'wrong-value' |
+/** The item carries its own check, and that check passes without testing the claim. */
+'self-certifying' |
+/** The check names an input that does not exist, so it cannot run at all. */
+'unreachable-input' |
+/** A copy of an item already in the set, which is owed a duplicate flag rather than a second grade. */
+'duplicate';
+/** What a working grader owes a seeded item. */
+type PlantExpectation = 'reject' | 'accept';
+interface Plant {
+  /** Name of the plant record. Reported in `missedIds` and `missingIds`. */
+  id: string;
+  kind: PlantKind;
+  /** The seeded item, indistinguishable from a real one once mixed. */
+  item: GoldenItem;
+  /** The verdict a working grader owes this item. */
+  expectedVerdict: PlantExpectation;
+}
+/**
+ * Build one plant record and refuse an incoherent one.
+ *
+ * The refusal that matters is the last: a record whose `expectedVerdict`
+ * disagrees with `item.humanScore` inverts the measurement silently, because
+ * the same item then reads as wrong here and as correct to every calibration
+ * instrument that joins on the id.
+ *
+ * `item` is copied field by field so a later mutation of the caller's object
+ * cannot change what the manifest sealed, and `group` is dropped when it is
+ * absent so the record always has a canonical JSON form.
+ */
+declare function definePlant(input: {
+  id: string;
+  kind: PlantKind;
+  item: GoldenItem;
+  expectedVerdict: PlantExpectation;
+}): Plant;
+interface PlantManifest {
+  /**
+   * Digest over the seeded order, the plants, and the threshold. Publish it
+   * before the grading run: a manifest edited afterwards to match the results
+   * no longer matches its seal, and {@link catchRate} refuses it.
+   */
+  seal: LedgerHash;
+  /** The seed that fixed the mix order. */
+  seed: number;
+  /** The grade at or above which the graded policy's own gate accepts an item. */
+  acceptThreshold: number;
+  /** Every item id in the seeded set, in the order handed out. */
+  itemIds: string[];
+  /** The seeded plants. This is the answer key; keep it out of the graded workspace. */
+  plants: Plant[];
+}
+interface SeededGradingSet {
+  /**
+   * The mixed set in seeded order. Operator-side: it still carries every
+   * item's `humanScore`, so hand the graded policy the payload each `itemId`
+   * names, never this array.
+   */
+  items: GoldenItem[];
+  manifest: PlantManifest;
+}
+interface SeedPlantsOptions {
+  /**
+   * Fixes the mix order. The same dataset, plants, threshold, and seed always
+   * produce the same seeded order and the same seal.
+   */
+  seed?: number;
+  /**
+   * The grade at or above which the graded policy's own gate accepts an item.
+   * Supply the threshold your gate uses; the default suits a judge scoring in
+   * [0, 1] with a pass at the midpoint.
+   */
+  acceptThreshold?: number;
+}
+/**
+ * Mix plants into a grading set and seal which items they are.
+ *
+ * Refusals: a duplicate id anywhere in the mixed set (the join is by id, so a
+ * repeat makes one of the two unscoreable), a plant whose item id collides
+ * with a dataset item (it would shadow real work and grade it as a plant), a
+ * dataset item with no usable label, and a plant whose expectation the run's
+ * `acceptThreshold` contradicts.
+ */
+declare function seedPlants(dataset: readonly GoldenItem[], plants: readonly Plant[], options?: SeedPlantsOptions): SeededGradingSet;
+/**
+ * One grader outcome for one item. `score: null` says the grader ran and
+ * declined to decide — the check never tested the item's defect. Any
+ * `CandidateScore` from a judge run is already a valid outcome.
+ */
+interface PlantOutcome {
+  itemId: string;
+  score: number | null;
+}
+/**
+ * `evaluated` — a rate stands. `incomplete` — a seeded id had no result.
+ * `not_evaluated` — nothing was seeded, or nothing seeded was decided.
+ */
+type CatchRateStatus = 'evaluated' | 'incomplete' | 'not_evaluated';
+interface PlantKindCounts {
+  seeded: number;
+  caught: number;
+  missed: number;
+  indecisive: number;
+  /** caught / (caught + missed), or null when the report is not `evaluated`. */
+  rate: number | null;
+}
+/**
+ * How the same grader treated the unseeded items of the same set. No labels
+ * are needed to read it: a grader that refuses everything scores `rate` 1.0
+ * on reject-plants, and a `rejectionRate` of 1.0 here is what separates that
+ * reflex from discrimination.
+ */
+interface UnseededRejection {
+  n: number;
+  decided: number;
+  rejected: number;
+  /** rejected / decided, or null when nothing unseeded was decided. */
+  rejectionRate: number | null;
+}
+interface CatchRateReport {
+  status: CatchRateStatus;
+  /** Why the status is not `evaluated`. Absent when it is. */
+  reason?: string;
+  seeded: number;
+  caught: number;
+  missed: number;
+  /**
+   * The grader returned a result and declined to decide. Counted apart: it
+   * enters neither side of `rate`.
+   */
+  indecisive: number;
+  /** caught / (caught + missed), or null unless the status is `evaluated`. */
+  rate: number | null;
+  /** One entry per kind actually seeded. A kind nobody seeded is absent, never zero. */
+  byKind: Partial<Record<PlantKind, PlantKindCounts>>;
+  /** Plant ids the grader graded as the seed says it must not. */
+  missedIds: string[];
+  /** Plant ids with no result at all — the reason a status is `incomplete`. */
+  missingIds: string[];
+  unseeded: UnseededRejection;
+}
+/**
+ * Score a grading run against its sealed manifest.
+ *
+ * Refusals: a manifest whose contents no longer match its seal, a result for
+ * an id the manifest never handed out, and a repeated result id. Each says
+ * the results and the manifest describe different runs, and a rate computed
+ * across two runs is a fabrication.
+ */
+declare function catchRate(results: readonly PlantOutcome[], manifest: PlantManifest): CatchRateReport;
+//#endregion
 //#region src/meta-eval/sentinel.d.ts
 declare const SENTINEL_METRIC_NAMES: readonly ['irr', 'calibrationKappa', 'sentinelPassRate'];
 type SentinelMetricName = (typeof SENTINEL_METRIC_NAMES)[number];
@@ -215,5 +375,5 @@ interface EvalHealthStamp {
  */
 declare function evalHealthStamp(report: SentinelReport): EvalHealthStamp;
 //#endregion
-export { CalibrationBin, CalibrationOptions, CalibrationPair, CalibrationReport, CorrelationResult, CorrelationStudyOptions, CorrelationStudyResult, DeploymentOutcome, EvalHealthStamp, EvalMetricSpec, FileSystemOutcomeStore, FileSystemOutcomeStoreOptions, InMemoryOutcomeStore, JudgeSentinelOptions, OutcomeFilter, OutcomePair, OutcomeStore, RubricOutcomePair, RubricPredictiveValidityInput, RubricPredictiveValidityReport, RubricRanking, SentinelMetricName, SentinelMetrics, SentinelReport, SentinelSetOptions, SentinelSnapshot, SentinelStore, SentinelThresholds, SentinelTrend, SnapshotMeta, calibrationCurve, correlationStudy, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, rubricPredictiveValidity, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot };
+export { CalibrationBin, CalibrationOptions, CalibrationPair, CalibrationReport, CatchRateReport, CatchRateStatus, CorrelationResult, CorrelationStudyOptions, CorrelationStudyResult, DeploymentOutcome, EvalHealthStamp, EvalMetricSpec, FileSystemOutcomeStore, FileSystemOutcomeStoreOptions, InMemoryOutcomeStore, JudgeSentinelOptions, OutcomeFilter, OutcomePair, OutcomeStore, Plant, PlantExpectation, PlantKind, PlantKindCounts, PlantManifest, PlantOutcome, RubricOutcomePair, RubricPredictiveValidityInput, RubricPredictiveValidityReport, RubricRanking, SeedPlantsOptions, SeededGradingSet, SentinelMetricName, SentinelMetrics, SentinelReport, SentinelSetOptions, SentinelSnapshot, SentinelStore, SentinelThresholds, SentinelTrend, SnapshotMeta, UnseededRejection, calibrationCurve, catchRate, correlationStudy, definePlant, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, rubricPredictiveValidity, seedPlants, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot };
 //# sourceMappingURL=index.d.ts.map

package/dist/meta-eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/meta-eval/correlation-study.ts","../../src/meta-eval/calibration.ts","../../src/meta-eval/sentinel.ts"],"mappings":";;;;;;;UAkBiB;EACf;;;EAGA,WAAW,KAAK,KAAK,OAAO,eAAe;;UAG5B;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;EAEA;IAAe;IAAe;;;EAE9B;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;;EAEf;;EAEA,gBAAgB;;EAEhB;;EAEA;;;EAGA;;iBAGoB,iBACpB,YAAY,YACZ,cAAc,cACd,aAAa,kBACb,8BACA,UAAS,0BACR,QAAQ;;;UCtDM;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,MAAM;;EAEN;;EAEA;;UAGe;EACf;;EAEA;;EAEA;IAAU;IAAY;;;UAGP;EACf;EACA;;iBAGoB,iBACpB,YAAY,YACZ,cAAc,cACd,YAAY,gBACZ,uBACA,UAAS,qBACR,QAAQ;;;cCDL;KAEM,6BAA6B;UAExB;;EAEf;;EAEA;;EAEA;;UAGe;;EAEf;EACA;;EAEA;;;;;;;EAOA,SAAS;;;UAIM;EACf;EACA;EACA;;;iBAcc,yBAAyB,UAAU,kBAAkB;;;;;;iBAiCrD,wBACd,QAAQ,oBAAoB,6BAC5B,MAAM,eACL;;;;;;;;iBAmBa,sBACd,QAAQ,wBAAwB,qBAChC,MAAM,eACL;UAYc;;EAEf;;;;;;;;;iBAUc,wBACd,QAAQ,kBACR,QAAQ,cACR,MAAM,cACN,UAAS,qBACR;UA4Cc;EACf,OAAO,UAAU,mBAAmB;;EAEpC,QAAQ,mBAAmB,QAAQ;;iBAGrB,sBAAsB,UAAS,qBAA0B;;;;;;;iBAqBzD,kBAAkB,eAAe;UA0ChC;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;EACA,aAAa;;EAEb,cAAc;;UAGC;EACf;EACA,QAAQ;;EAER,OAAO;;EAEP;;EAEA;;EAEA;EACA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;;;;;;EAOA;;iBAKc,oBACd,SAAS,oBACT,MAAM,uBACL;UA6Gc;EACf;EACA;;;;;;;;;;iBAWc,gBAAgB,QAAQ,iBAAiB"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/meta-eval/correlation-study.ts","../../src/meta-eval/calibration.ts","../../src/meta-eval/plants.ts","../../src/meta-eval/sentinel.ts"],"mappings":";;;;;;;;UAkBiB;EACf;;;EAGA,WAAW,KAAK,KAAK,OAAO,eAAe;;UAG5B;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;EAEA;IAAe;IAAe;;;EAE9B;;UAGe;EACf,OAAO;EACP;EACA;;UAGe;;EAEf;;EAEA,gBAAgB;;EAEhB;;EAEA;;;EAGA;;iBAGoB,iBACpB,YAAY,YACZ,cAAc,cACd,aAAa,kBACb,8BACA,UAAS,0BACR,QAAQ;;;UCtDM;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;EACA,MAAM;;EAEN;;EAEA;;UAGe;EACf;;EAEA;;EAEA;IAAU;IAAY;;;UAGP;EACf;EACA;;iBAGoB,iBACpB,YAAY,YACZ,cAAc,cACd,YAAY,gBACZ,uBACA,UAAS,qBACR,QAAQ;;;;;;;;KCDC;;;;;;;;;;KAkBA;UAOK;;EAEf;EACA,MAAM;;EAEN,MAAM;;EAEN,iBAAiB;;;;;;;;;;;;;;iBAeH,YAAY;EAC1B;EACA,MAAM;EACN,MAAM;EACN,iBAAiB;IACf;UAiDa;;;;;;EAMf,MAAM;;EAEN;;EAEA;;EAEA;;EAEA,QAAQ;;UAGO;;;;;;EAMf,OAAO;EACP,UAAU;;UAGK;;;;;EAKf;;;;;;EAMA;;;;;;;;;;;iBAYc,WACd,kBAAkB,cAClB,iBAAiB,SACjB,UAAS,oBACR;;;;;;UAqGc;EACf;EACA;;;;;;KAOU;UAEK;EACf;EACA;EACA;EACA;;EAEA;;;;;;;;UASe;EACf;EACA;EACA;;EAEA;;UAGe;EACf,QAAQ;;EAER;EACA;EACA;EACA;;;;;EAKA;;EAEA;;EAEA,QAAQ,QAAQ,OAAO,WAAW;;EAElC;;EAEA;EACA,UAAU;;;;;;;;;;iBAWI,UACd,kBAAkB,gBAClB,UAAU,gBACT;;;cCpUG;KAEM,6BAA6B;UAExB;;EAEf;;EAEA;;EAEA;;UAGe;;EAEf;EACA;;EAEA;;;;;;;EAOA,SAAS;;;UAIM;EACf;EACA;EACA;;;iBAcc,yBAAyB,UAAU,kBAAkB;;;;;;iBAiCrD,wBACd,QAAQ,oBAAoB,6BAC5B,MAAM,eACL;;;;;;;;iBAmBa,sBACd,QAAQ,wBAAwB,qBAChC,MAAM,eACL;UAYc;;EAEf;;;;;;;;;iBAUc,wBACd,QAAQ,kBACR,QAAQ,cACR,MAAM,cACN,UAAS,qBACR;UA4Cc;EACf,OAAO,UAAU,mBAAmB;;EAEpC,QAAQ,mBAAmB,QAAQ;;iBAGrB,sBAAsB,UAAS,qBAA0B;;;;;;;iBAqBzD,kBAAkB,eAAe;UA0ChC;;EAEf;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf;EACA,aAAa;;EAEb,cAAc;;UAGC;EACf;EACA,QAAQ;;EAER,OAAO;;EAEP;;EAEA;;EAEA;EACA;EACA;;UAGe;EACf,UAAU;EACV;EACA;;;;;;;EAOA;;iBAKc,oBACd,SAAS,oBACT,MAAM,uBACL;UA6Gc;EACf;EACA;;;;;;;;;;iBAWc,gBAAgB,QAAQ,iBAAiB"}

package/dist/meta-eval/index.js

@@ -1,5 +1,7 @@
-import { s as ValidationError } from "../errors-Dngq5h35.js";
+import { n as CaptureIntegrityError, s as ValidationError } from "../errors-Dngq5h35.js";
+import { a as hashCanonical } from "../canonical-DPyQ_rpt.js";
 import { i as makeRng } from "../internal-BMFSR8Ns.js";
+import { t as mulberry32 } from "../random-Dn5fPWkt.js";
 import { a as spearmanR, r as pearsonR } from "../descriptive-1V17A-qa.js";
 import { u as runMetricExtractor } from "../query-BPGMVlbM.js";
 import { t as analyzeSeries } from "../series-convergence-CjO2QdRW.js";
@@ -226,6 +228,289 @@ function bootstrapPearsonCi(xs, ys, iterations, seed) {
 	};
 }
 //#endregion
+//#region src/meta-eval/plants.ts
+/**
+* Plants — seeded known-wrong items that measure the grader, not the work.
+*
+* A grading run reports how the work scored. It cannot report whether the
+* grader would have noticed a wrong answer, because every item it saw was
+* authored in good faith. A plant closes that hole: an item authored wrong by
+* construction is mixed into the live set, graded by the same path as
+* everything else, and the share of plants the grader refused is the catch
+* rate.
+*
+* Measured motive: a sibling lab ran a deliverable gate that accepted any
+* non-empty submission. It produced six false certifications in seventeen
+* deliveries, and no agent lied — the gate never asked a question the format
+* could fail. A catch rate is the number that would have shown it on day one.
+*
+* This module composes existing primitives rather than adding parallel ones:
+*
+* - A plant IS a {@link GoldenItem} from `../judge-calibration`. Its
+*   `humanScore` is the grade a working grader owes the item, so the same
+*   array feeds `calibrateJudge` unchanged.
+* - The grader's output is `CandidateScore[]`, the array `calibrateJudge` and
+*   `snapshotFromSentinelSet` already consume.
+* - "Caught" is `snapshotFromSentinelSet`'s join with the labels inverted:
+*   the grade lands on the side of `acceptThreshold` the seed demands.
+* - The manifest is sealed with `hashCanonical` from `../ledger-core/canonical`,
+*   the digest the sealed-experiment path uses, so the answer key cannot be
+*   revised once the results are in.
+*
+* Blindness has two halves, and this module owns one. It never puts a plant
+* flag on a graded item: `seedPlants` returns the mixed set and a manifest,
+* and only the manifest knows which ids are seeded. Keeping the manifest out
+* of the graded workspace and publishing its `seal` before grading is the
+* caller's half; {@link catchRate} refuses a manifest whose contents no longer
+* match its seal.
+*
+* Refusals, because a catch rate that cannot refuse is not a measurement:
+*
+* - a seeded id with no result makes the report `incomplete`, never a rate
+*   over the results that did come back;
+* - zero seeded plants makes it `not_evaluated`, never 1.0;
+* - a result for an id the manifest never handed out is refused outright.
+*/
+const PLANT_KINDS = [
+	"wrong-value",
+	"self-certifying",
+	"unreachable-input",
+	"duplicate"
+];
+const PLANT_EXPECTATIONS = ["reject", "accept"];
+/** The label boundary a plant record is checked against at definition time. */
+const RECORD_LABEL_BOUNDARY = .5;
+/**
+* Build one plant record and refuse an incoherent one.
+*
+* The refusal that matters is the last: a record whose `expectedVerdict`
+* disagrees with `item.humanScore` inverts the measurement silently, because
+* the same item then reads as wrong here and as correct to every calibration
+* instrument that joins on the id.
+*
+* `item` is copied field by field so a later mutation of the caller's object
+* cannot change what the manifest sealed, and `group` is dropped when it is
+* absent so the record always has a canonical JSON form.
+*/
+function definePlant(input) {
+	const { id, kind, item, expectedVerdict } = input;
+	if (typeof id !== "string" || id.trim() === "") throw new ValidationError("definePlant: id must be a non-empty string");
+	if (!PLANT_KINDS.includes(kind)) throw new ValidationError(`definePlant: plant "${id}" has kind ${JSON.stringify(kind)}; expected one of ${PLANT_KINDS.join(", ")}`);
+	if (!PLANT_EXPECTATIONS.includes(expectedVerdict)) throw new ValidationError(`definePlant: plant "${id}" has expectedVerdict ${JSON.stringify(expectedVerdict)}; expected one of ${PLANT_EXPECTATIONS.join(", ")}`);
+	if (typeof item.itemId !== "string" || item.itemId.trim() === "") throw new ValidationError(`definePlant: plant "${id}" has an empty item.itemId`);
+	if (!Number.isFinite(item.humanScore) || item.humanScore < 0 || item.humanScore > 1) throw new ValidationError(`definePlant: plant "${id}" has humanScore ${item.humanScore}; expected a finite number in [0, 1]`);
+	if (item.group !== void 0 && typeof item.group !== "string") throw new ValidationError(`definePlant: plant "${id}" has a non-string item.group`);
+	if (item.humanScore === RECORD_LABEL_BOUNDARY) throw new ValidationError(`definePlant: plant "${id}" has humanScore ${RECORD_LABEL_BOUNDARY}, which states neither a rejection nor an acceptance`);
+	const labelSays = item.humanScore < RECORD_LABEL_BOUNDARY ? "reject" : "accept";
+	if (labelSays !== expectedVerdict) throw new ValidationError(`definePlant: plant "${id}" expects the grader to ${expectedVerdict} it, but humanScore ${item.humanScore} says ${labelSays}`);
+	return {
+		id,
+		kind,
+		item: {
+			itemId: item.itemId,
+			humanScore: item.humanScore,
+			...item.group === void 0 ? {} : { group: item.group }
+		},
+		expectedVerdict
+	};
+}
+/**
+* Mix plants into a grading set and seal which items they are.
+*
+* Refusals: a duplicate id anywhere in the mixed set (the join is by id, so a
+* repeat makes one of the two unscoreable), a plant whose item id collides
+* with a dataset item (it would shadow real work and grade it as a plant), a
+* dataset item with no usable label, and a plant whose expectation the run's
+* `acceptThreshold` contradicts.
+*/
+function seedPlants(dataset, plants, options = {}) {
+	const seed = options.seed ?? 7;
+	const acceptThreshold = options.acceptThreshold ?? .5;
+	if (!Number.isFinite(seed)) throw new ValidationError(`seedPlants: seed must be a finite number, got ${seed}`);
+	if (!Number.isFinite(acceptThreshold) || acceptThreshold <= 0 || acceptThreshold > 1) throw new ValidationError(`seedPlants: acceptThreshold must be a finite number in (0, 1], got ${acceptThreshold}`);
+	const datasetItems = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const item of dataset) {
+		if (typeof item.itemId !== "string" || item.itemId.trim() === "") throw new ValidationError("seedPlants: a dataset item has an empty itemId");
+		if (!Number.isFinite(item.humanScore)) throw new ValidationError(`seedPlants: dataset item "${item.itemId}" has a non-finite humanScore`);
+		if (seen.has(item.itemId)) throw new ValidationError(`seedPlants: duplicate dataset itemId "${item.itemId}"`);
+		seen.add(item.itemId);
+		datasetItems.push({
+			itemId: item.itemId,
+			humanScore: item.humanScore,
+			...item.group === void 0 ? {} : { group: item.group }
+		});
+	}
+	const sealedPlants = [];
+	const plantIds = /* @__PURE__ */ new Set();
+	for (const plant of plants) {
+		const record = definePlant(plant);
+		if (plantIds.has(record.id)) throw new ValidationError(`seedPlants: duplicate plant id "${record.id}"`);
+		if (seen.has(record.item.itemId)) throw new ValidationError(`seedPlants: plant "${record.id}" reuses itemId "${record.item.itemId}", which is already in the set`);
+		const thresholdSays = record.item.humanScore >= acceptThreshold ? "accept" : "reject";
+		if (thresholdSays !== record.expectedVerdict) throw new ValidationError(`seedPlants: plant "${record.id}" expects the grader to ${record.expectedVerdict} it, but humanScore ${record.item.humanScore} is on the ${thresholdSays} side of acceptThreshold ${acceptThreshold}`);
+		plantIds.add(record.id);
+		seen.add(record.item.itemId);
+		sealedPlants.push(record);
+	}
+	const items = shuffled([...datasetItems, ...sealedPlants.map((plant) => plant.item)], mulberry32(seed));
+	const itemIds = items.map((item) => item.itemId);
+	return {
+		items,
+		manifest: {
+			seal: sealManifest({
+				seed,
+				acceptThreshold,
+				itemIds,
+				plants: sealedPlants
+			}),
+			seed,
+			acceptThreshold,
+			itemIds,
+			plants: sealedPlants
+		}
+	};
+}
+/**
+* Order by one independent uniform key per item, which is a uniform
+* permutation and a pure function of the seed. A comparator that returns a
+* fresh random sign instead is neither: it is not a consistent ordering, so
+* the permutation it produces is biased and depends on the sort algorithm.
+*/
+function shuffled(items, random) {
+	return items.map((item) => ({
+		item,
+		key: random()
+	})).sort((left, right) => left.key - right.key).map((entry) => entry.item);
+}
+function sealManifest(contents) {
+	return hashCanonical({
+		scheme: "agent-eval.plant-manifest.v1",
+		seed: contents.seed,
+		acceptThreshold: contents.acceptThreshold,
+		itemIds: contents.itemIds,
+		plants: contents.plants
+	});
+}
+/**
+* Score a grading run against its sealed manifest.
+*
+* Refusals: a manifest whose contents no longer match its seal, a result for
+* an id the manifest never handed out, and a repeated result id. Each says
+* the results and the manifest describe different runs, and a rate computed
+* across two runs is a fabrication.
+*/
+function catchRate(results, manifest) {
+	const expectedSeal = sealManifest({
+		seed: manifest.seed,
+		acceptThreshold: manifest.acceptThreshold,
+		itemIds: manifest.itemIds,
+		plants: manifest.plants
+	});
+	if (expectedSeal !== manifest.seal) throw new CaptureIntegrityError(`catchRate: manifest contents hash to ${expectedSeal} but the manifest carries seal ${manifest.seal} — the plant set changed after it was sealed`);
+	const handedOut = new Set(manifest.itemIds);
+	const scoreByItemId = /* @__PURE__ */ new Map();
+	for (const result of results) {
+		if (!handedOut.has(result.itemId)) throw new ValidationError(`catchRate: result for "${result.itemId}", which this manifest never handed out`);
+		if (scoreByItemId.has(result.itemId)) throw new ValidationError(`catchRate: duplicate result for "${result.itemId}"`);
+		if (result.score !== null && !Number.isFinite(result.score)) throw new ValidationError(`catchRate: result for "${result.itemId}" has score ${result.score}; expected a finite number or null`);
+		scoreByItemId.set(result.itemId, result.score);
+	}
+	const byKind = {};
+	const missedIds = [];
+	const missingIds = [];
+	let caught = 0;
+	let missed = 0;
+	let indecisive = 0;
+	for (const plant of manifest.plants) {
+		let counts = byKind[plant.kind];
+		if (counts === void 0) {
+			counts = {
+				seeded: 0,
+				caught: 0,
+				missed: 0,
+				indecisive: 0,
+				rate: null
+			};
+			byKind[plant.kind] = counts;
+		}
+		counts.seeded += 1;
+		if (!scoreByItemId.has(plant.item.itemId)) {
+			missingIds.push(plant.id);
+			continue;
+		}
+		const score = scoreByItemId.get(plant.item.itemId) ?? null;
+		if (score === null) {
+			indecisive += 1;
+			counts.indecisive += 1;
+			continue;
+		}
+		if ((score >= manifest.acceptThreshold ? "accept" : "reject") === plant.expectedVerdict) {
+			caught += 1;
+			counts.caught += 1;
+		} else {
+			missed += 1;
+			counts.missed += 1;
+			missedIds.push(plant.id);
+		}
+	}
+	const seeded = manifest.plants.length;
+	const decided = caught + missed;
+	const report = {
+		status: "evaluated",
+		seeded,
+		caught,
+		missed,
+		indecisive,
+		rate: null,
+		byKind,
+		missedIds,
+		missingIds,
+		unseeded: unseededRejection(manifest, scoreByItemId)
+	};
+	if (seeded === 0) return {
+		...report,
+		status: "not_evaluated",
+		reason: "no plant was seeded, so the grader was never asked a question it could fail"
+	};
+	if (missingIds.length > 0) return {
+		...report,
+		status: "incomplete",
+		reason: `${missingIds.length} of ${seeded} seeded plants have no result: ${missingIds.join(", ")}`
+	};
+	if (decided === 0) return {
+		...report,
+		status: "not_evaluated",
+		reason: `all ${seeded} seeded plants are indecisive: no check tested the seeded defect`
+	};
+	for (const counts of Object.values(byKind)) {
+		const kindDecided = counts.caught + counts.missed;
+		counts.rate = kindDecided === 0 ? null : counts.caught / kindDecided;
+	}
+	report.rate = caught / decided;
+	return report;
+}
+function unseededRejection(manifest, scoreByItemId) {
+	const plantItemIds = new Set(manifest.plants.map((plant) => plant.item.itemId));
+	let n = 0;
+	let decided = 0;
+	let rejected = 0;
+	for (const itemId of manifest.itemIds) {
+		if (plantItemIds.has(itemId)) continue;
+		n += 1;
+		const score = scoreByItemId.get(itemId);
+		if (score === void 0 || score === null) continue;
+		decided += 1;
+		if (score < manifest.acceptThreshold) rejected += 1;
+	}
+	return {
+		n,
+		decided,
+		rejected,
+		rejectionRate: decided === 0 ? null : rejected / decided
+	};
+}
+//#endregion
 //#region src/meta-eval/sentinel.ts
 /**
 * Judge sentinel — eval trustworthiness as a continuously measured,
@@ -499,6 +784,6 @@ function evalHealthStamp(report) {
 	};
 }
 //#endregion
-export { FileSystemOutcomeStore, InMemoryOutcomeStore, calibrationCurve, correlationStudy, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, rubricPredictiveValidity, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot };
+export { FileSystemOutcomeStore, InMemoryOutcomeStore, calibrationCurve, catchRate, correlationStudy, definePlant, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, rubricPredictiveValidity, seedPlants, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot };
 
 //# sourceMappingURL=index.js.map