engineering-behavior-observatory 0.2.1

package/dist/src/human-calibration.d.ts

@@ -0,0 +1,167 @@
+import { type BehaviorReview, type AgentSdkBehaviorEvidence } from "./behavior-assertions.js";
+import type { RunManifest } from "./run-bundles.js";
+type DigestString = `sha256:${string}`;
+type Outcome = "passed" | "failed" | "not-run" | "error" | "unavailable";
+type DecisionState = Exclude<BehaviorReview["state"], "proposed">;
+export type ReviewSourceSet = {
+    schemaVersion: "ebo.review-source-set/v1";
+    sources: ReadonlyArray<{
+        bundleRoot: string;
+        assertionPath: string;
+        taskContext: string;
+    }>;
+};
+export type ReviewFilters = {
+    taskIds?: readonly string[];
+    modelIds?: readonly string[];
+    harnessIds?: readonly string[];
+    outcomes?: readonly Outcome[];
+    terminalStates?: readonly string[];
+    categoryIds?: readonly string[];
+    abstentions?: readonly boolean[];
+    confidence?: {
+        min?: number;
+        max?: number;
+    };
+};
+export type ReviewSampleCriteria = {
+    schemaVersion: "ebo.review-sample-criteria/v1";
+    seed: string;
+    strata: ReadonlyArray<{
+        id: string;
+        sampleSize: number;
+        filters: ReviewFilters;
+    }>;
+};
+export type ReviewCandidate = {
+    assertion: {
+        id: string;
+        schemaVersion: "ebo.behavior-assertion/v1";
+        digest: DigestString;
+    };
+    source: {
+        bundleRoot: string;
+        assertionPath: string;
+    };
+    context: {
+        runId: string;
+        attemptId: string;
+        taskId: string;
+        taskContext: string;
+        modelId: string;
+        harnessId: string;
+        terminalState: string;
+        outcome: Outcome;
+        categoryId: string;
+        abstained: boolean;
+        confidence: number | null;
+    };
+};
+export type ReviewSample = {
+    schemaVersion: "ebo.review-sample/v1";
+    createdAt: string;
+    sources: ReviewSourceSet;
+    criteria: ReviewSampleCriteria;
+    population: {
+        sourceCount: number;
+        eligibleAssertionIds: readonly string[];
+        selectedAssertionIds: readonly string[];
+        unavailableStrata: readonly string[];
+        strata: ReadonlyArray<{
+            id: string;
+            eligible: number;
+            selected: number;
+            requested: number;
+        }>;
+    };
+    candidates: readonly ReviewCandidate[];
+};
+export type ReviewDecision = {
+    schemaVersion: "ebo.human-review-decision/v1";
+    id: string;
+    kind: "review" | "adjudication";
+    assertion: ReviewCandidate["assertion"];
+    reviewer: {
+        kind: "human";
+        id: string;
+    };
+    decidedAt: string;
+    state: DecisionState;
+    rationale: string;
+    previousHistory: null | {
+        schemaVersion: "ebo.review-history/v1";
+        digest: DigestString;
+    };
+    adjudicates?: readonly string[];
+};
+export type ReviewHistory = {
+    schemaVersion: "ebo.review-history/v1";
+    selection: {
+        schemaVersion: "ebo.review-sample/v1";
+        digest: DigestString;
+    };
+    decisions: readonly ReviewDecision[];
+};
+type Agreement = {
+    status: "unavailable";
+    denominator: 0;
+    population: string;
+    reason: string;
+} | {
+    status: "available";
+    denominator: number;
+    population: string;
+    agreements: number;
+    disagreements: number;
+    rate: number;
+};
+export type CalibrationSummary = {
+    schemaVersion: "ebo.calibration-summary/v1";
+    selection: {
+        schemaVersion: "ebo.review-sample/v1";
+        digest: DigestString;
+    };
+    history: {
+        schemaVersion: "ebo.review-history/v1";
+        digest: DigestString;
+    };
+    totals: CalibrationCounts;
+    categories: ReadonlyArray<{
+        categoryId: string;
+        counts: CalibrationCounts;
+    }>;
+};
+type CalibrationCounts = {
+    selectedAssertions: number;
+    judgeAbstentions: number;
+    humanDecisionAbstentions: number;
+    disputedAssertions: number;
+    unresolvedAssertions: number;
+    confirmedEligibleAssertions: number;
+    rejectedAssertions: number;
+    judgeHumanAgreement: Agreement;
+    humanHumanAgreement: Agreement;
+    adjudication: {
+        denominator: number;
+        population: "human adjudication decisions";
+        confirmed: number;
+        rejected: number;
+        unresolved: number;
+    };
+};
+export declare function selectReviewSample(sourceSet: ReviewSourceSet, criteria: ReviewSampleCriteria, now?: () => string): Promise<ReviewSample>;
+export declare function writeReviewPacket(selection: ReviewSample, outputRoot: string, now?: () => string): Promise<void>;
+export declare function importReviewDecision(selection: ReviewSample, historyPath: string, decision: ReviewDecision): Promise<{
+    appended: boolean;
+    history: ReviewHistory;
+}>;
+export declare function summarizeCalibration(selection: ReviewSample, history: ReviewHistory): Promise<CalibrationSummary>;
+export declare function validateReviewHistory(selection: ReviewSample, history: ReviewHistory): void;
+export declare function validateReviewSample(selection: ReviewSample): void;
+export declare function revalidateReviewSample(selection: ReviewSample): Promise<void>;
+export declare function terminalVerifierOutcome(capture: AgentSdkBehaviorEvidence["capture"], manifest: RunManifest): Outcome;
+export declare function effectiveReviewOutcome(candidate: ReviewCandidate, decisions: readonly ReviewDecision[]): "confirmed" | "rejected" | "unresolved" | "unreviewed" | "judge-abstained";
+export declare function isDisputedReviewOutcome(candidate: ReviewCandidate, decisions: readonly ReviewDecision[]): boolean;
+export declare function assertCalibrationDestination(sourceRoots: readonly string[], destination: string): void;
+export declare function escapeHtml(value: unknown): string;
+export {};