engineering-behavior-observatory 0.2.1

package/dist/src/aggregation.d.ts

@@ -0,0 +1,183 @@
+import { type BehaviorAssertion } from "./behavior-assertions.js";
+import type { CorpusIndexEntry } from "./corpus.js";
+import { effectiveReviewOutcome, type ReviewHistory, type ReviewSample } from "./human-calibration.js";
+import { type ComparisonReport, type ComparisonRequest } from "./normalization-integrity.js";
+import { type StructuralObservationSet } from "./structural-observations.js";
+export type AggregationDimension = "task" | "model" | "harness" | "trial" | "capture-qualification";
+export type AttemptSelectionPolicy = "all-attempts" | "latest-attempt-per-run";
+export type AggregationRequest = {
+    schemaVersion: "ebo.aggregation-request/v1";
+    sources: {
+        corpusRoot: string;
+        corpusIndex: string;
+        observationSets: ReadonlyArray<{
+            bundleRoot: string;
+            path: string;
+        }>;
+        assertions: ReadonlyArray<{
+            bundleRoot: string;
+            path: string;
+        }>;
+        calibrations: ReadonlyArray<{
+            selection: string;
+            history: string;
+        }>;
+    };
+    groupBy: readonly AggregationDimension[];
+    selectedAttemptPolicy: AttemptSelectionPolicy;
+    recurrence: {
+        minimumOccurrences: number;
+    };
+    comparisons: ReadonlyArray<{
+        id: string;
+        measure: string;
+        left: Partial<Record<AggregationDimension, string>>;
+        right: Partial<Record<AggregationDimension, string>>;
+        matchBy: readonly AggregationDimension[];
+        eligibilityGates: ReadonlyArray<{
+            request: string;
+            report: string;
+        }>;
+    }>;
+};
+export type AggregationInput = {
+    lineage?: {
+        requestDigest?: `sha256:${string}`;
+        corpusIndexDigest?: `sha256:${string}`;
+    };
+    corpusEntries: readonly CorpusIndexEntry[];
+    observationSets: ReadonlyArray<{
+        bundleRoot: string;
+        document: StructuralObservationSet;
+    }>;
+    assertions: ReadonlyArray<{
+        bundleRoot: string;
+        document: BehaviorAssertion;
+    }>;
+    calibrations: ReadonlyArray<{
+        selection: ReviewSample;
+        history: ReviewHistory;
+    }>;
+    comparisons: ReadonlyArray<{
+        id: string;
+        measure: string;
+        left: Partial<Record<AggregationDimension, string>>;
+        right: Partial<Record<AggregationDimension, string>>;
+        matchBy: readonly AggregationDimension[];
+        eligibility: ReadonlyArray<{
+            request: ComparisonRequest;
+            report: ComparisonReport;
+        }>;
+    }>;
+};
+export type AggregateMeasurement = {
+    status: "available" | "unavailable";
+    numerator: {
+        value: number;
+        unit: string;
+    };
+    denominator: {
+        value: number;
+        unit: string;
+    };
+    rate?: number;
+    exclusions: ReadonlyArray<{
+        reason: string;
+        count: number;
+        unit: string;
+    }>;
+    reason?: string;
+};
+export type AggregateMetric = {
+    id: string;
+    population: "run" | "attempt" | "operation" | "assertion" | "reviewed-assertion";
+    claimStatus: "descriptive" | "unavailable";
+    measurement: AggregateMeasurement;
+};
+export type BehaviorAggregate = {
+    behavior: BehaviorAssertion["behavior"];
+    rubric: BehaviorAssertion["rubric"];
+    evaluator: BehaviorAssertion["evaluator"];
+    population: "confirmed-attempt-dimension";
+    assessments: ReadonlyArray<{
+        assessment: "constructive" | "adverse" | "mixed" | "context-dependent";
+        measurement: AggregateMeasurement;
+    }>;
+    assertions: ReadonlyArray<{
+        runId: string;
+        attemptId: string;
+        id: string;
+        digest: `sha256:${string}`;
+        reviewOutcome: ReturnType<typeof effectiveReviewOutcome>;
+        disputed: boolean;
+        included: boolean;
+    }>;
+};
+export type AggregationReport = {
+    schemaVersion: "ebo.aggregation-report/v1";
+    policy: {
+        groupBy: readonly AggregationDimension[];
+        selectedAttemptPolicy: AttemptSelectionPolicy;
+        recurrence: {
+            minimumOccurrences: number;
+            establishesCausality: false;
+            establishesStatisticalSignificance: false;
+        };
+    };
+    sourcePopulation: {
+        uniqueRuns: number;
+        uniqueAttempts: number;
+        selectedAttempts: number;
+        duplicateInputsIgnored: number;
+    };
+    sourceLineage: {
+        requestDigest: `sha256:${string}`;
+        corpusIndexDigest: `sha256:${string}`;
+        manifests: ReadonlyArray<{
+            runId: string;
+            attemptId: string;
+            digest: `sha256:${string}`;
+        }>;
+        observationSets: ReadonlyArray<{
+            runId: string;
+            attemptId: string;
+            digest: `sha256:${string}`;
+        }>;
+        assertions: ReadonlyArray<{
+            runId: string;
+            attemptId: string;
+            id: string;
+            digest: `sha256:${string}`;
+        }>;
+        calibrations: ReadonlyArray<{
+            selectionDigest: `sha256:${string}`;
+            historyDigest: `sha256:${string}`;
+        }>;
+        comparisonGates: ReadonlyArray<{
+            requestDigest: `sha256:${string}`;
+            reportDigest: `sha256:${string}`;
+        }>;
+    };
+    groups: ReadonlyArray<{
+        dimensions: Partial<Record<AggregationDimension, string>>;
+        metrics: readonly AggregateMetric[];
+        behaviors?: readonly BehaviorAggregate[];
+        variations: ReadonlyArray<{
+            measure: "terminal-state" | "verifier-status";
+            distribution: readonly AggregateMetric[];
+            claimStatus: "no-variation" | "case-study" | "recurring-description" | "unavailable";
+        }>;
+    }>;
+    comparisons: ReadonlyArray<{
+        id: string;
+        measure: string;
+        eligibility: readonly ComparisonReport[];
+        matchedDifference: AggregateMeasurement;
+        differingMatchedUnits: number;
+        claimStatus: "no-difference" | "case-study" | "recurring-description" | "unavailable";
+        limitations: readonly string[];
+    }>;
+    limitations: readonly string[];
+};
+export declare function aggregateEvaluation(input: AggregationInput, policy: Pick<AggregationRequest, "groupBy" | "selectedAttemptPolicy" | "recurrence">): Promise<AggregationReport>;
+export declare function selectAggregationAttempts(entries: readonly CorpusIndexEntry[], policy: AttemptSelectionPolicy): CorpusIndexEntry[];