@openpond/evals 0.6.0 → 0.7.0

package/dist/types/learning/admission.d.ts

@@ -0,0 +1,335 @@
+import { z } from "zod";
+import { type RewardBinding, type RewardComposition, type RewardRelease } from "../rewards.js";
+import { type TaskRecord, type TasksetRelease } from "../tasksets.js";
+import { type TaskSchemaIssue } from "../task-schema.js";
+import { type LearningSource, type TaskAdmissionDecision, type TaskBatch, type TaskDefinition, type TaskEvidence, type TaskExampleSubmission } from "./contracts.js";
+export type TaskEvidenceInspection = {
+    evidenceValidity: "valid" | "invalid";
+    taskReady: boolean;
+    observedOutputValid: boolean | null;
+    issues: TaskSchemaIssue[];
+};
+export type TaskFamilySplit = {
+    namespace: string;
+    key: string;
+    kind: "family" | "input";
+    split: TaskExampleSubmission["split"];
+};
+export declare function inspectTaskEvidence(evidence: TaskEvidence, definition: TaskDefinition): TaskEvidenceInspection;
+export declare function validateSourceSubmission(sourceInput: LearningSource, submissionInput: TaskExampleSubmission): TaskExampleSubmission;
+export declare function mapTaskSourceRecord(input: {
+    source: LearningSource;
+    record: unknown;
+    idempotencyKey: string;
+}): {
+    status: "mapped";
+    example: TaskExampleSubmission;
+} | {
+    status: "pending";
+    missing: string[];
+};
+export declare function readTaskJsonPointer(value: unknown, pointer: string): unknown;
+export declare function taskRecordFromEvidence(evidence: TaskEvidence, definition: TaskDefinition): TaskRecord;
+export declare function taskAttemptEvidence(evidence: TaskEvidence, output: Record<string, unknown>): {
+    output: Record<string, unknown>;
+    runtimeEventRefs: string[];
+    artifactRefs: string[];
+};
+export declare function assertAdmissionDecision(input: {
+    decision: TaskAdmissionDecision;
+    evidence: TaskEvidence;
+    definition: TaskDefinition;
+    purpose: TaskBatch["purpose"];
+}): void;
+export declare function assertGradeIdentity(grade: RewardComposition, evidence: TaskEvidence, definition: TaskDefinition, output: Record<string, unknown>): void;
+export declare function taskFamilyReservations(evidence: TaskEvidence, definition: TaskDefinition): TaskFamilySplit[];
+export declare function sealTaskBatch(input: {
+    id: string;
+    definition: TaskDefinition;
+    binding: RewardBinding;
+    rewards: RewardRelease[];
+    purpose: TaskBatch["purpose"];
+    evidence: TaskEvidence[];
+    decisions: TaskAdmissionDecision[];
+    priorSplits: TaskFamilySplit[];
+    actorId: string;
+    now: string;
+}): TaskBatch;
+export declare const TaskBatchPackageMetadataSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.taskBatchPackage.v1">;
+    batch: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        revision: z.ZodNumber;
+    }, z.core.$strict>;
+    definition: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.taskDefinition.v1">;
+        id: z.ZodString;
+        revision: z.ZodNumber;
+        name: z.ZodString;
+        description: z.ZodString;
+        instructions: z.ZodString;
+        category: z.ZodEnum<{
+            custom: "custom";
+            structured: "structured";
+            question_answering: "question_answering";
+            coding: "coding";
+            tool_workflow: "tool_workflow";
+        }>;
+        familyNamespace: z.ZodString;
+        inputSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        outputSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        rewardBinding: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodNumber;
+        }, z.core.$strict>;
+        harness: z.ZodNullable<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>>;
+        execution: z.ZodObject<{
+            policy: z.ZodObject<{
+                policyVisibleFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                privilegedFields: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                hiddenGraderRefs: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                connectedAppScopes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            }, z.core.$strict>;
+            environment: z.ZodObject<{
+                protocolVersion: z.ZodLiteral<"openpond.environment.v1">;
+                kind: z.ZodEnum<{
+                    text: "text";
+                    agent: "agent";
+                    work: "work";
+                    custom_program: "custom_program";
+                }>;
+                entrypoint: z.ZodString;
+                stateful: z.ZodBoolean;
+                deterministicSeeds: z.ZodBoolean;
+                lifecycle: z.ZodArray<z.ZodEnum<{
+                    create: "create";
+                    reset: "reset";
+                    step: "step";
+                    collect: "collect";
+                    destroy: "destroy";
+                }>>;
+                networkPolicy: z.ZodEnum<{
+                    none: "none";
+                    declared_read_only: "declared_read_only";
+                    declared_scoped: "declared_scoped";
+                }>;
+                defaultTimeoutMs: z.ZodNumber;
+            }, z.core.$strict>;
+            environmentRelease: z.ZodOptional<z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>>;
+            tools: z.ZodArray<z.ZodObject<{
+                name: z.ZodString;
+                description: z.ZodString;
+                inputSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+                inputSchemaHash: z.ZodString;
+                sideEffect: z.ZodEnum<{
+                    read: "read";
+                    write: "write";
+                }>;
+                timeoutMs: z.ZodNumber;
+            }, z.core.$strict>>;
+            capabilities: z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                required: z.ZodBoolean;
+                scopes: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                portability: z.ZodEnum<{
+                    portable: "portable";
+                    host_adapter: "host_adapter";
+                    local_only: "local_only";
+                    hosted_only: "hosted_only";
+                }>;
+            }, z.core.$strict>>;
+            verifierSetRelease: z.ZodOptional<z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>>;
+        }, z.core.$strict>;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    binding: z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.rewardBinding.v1">;
+        id: z.ZodString;
+        revision: z.ZodNumber;
+        name: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+        recipeRef: z.ZodOptional<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodNumber;
+        }, z.core.$strict>>;
+        sources: z.ZodArray<z.ZodObject<{
+            graderId: z.ZodString;
+            reward: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+                revision: z.ZodNumber;
+            }, z.core.$strict>;
+            role: z.ZodEnum<{
+                evaluation: "evaluation";
+                training: "training";
+            }>;
+            normalization: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"identity">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"linear">;
+                minimum: z.ZodNumber;
+                maximum: z.ZodNumber;
+                direction: z.ZodEnum<{
+                    higher: "higher";
+                    lower: "lower";
+                }>;
+            }, z.core.$strict>], "kind">;
+            weight: z.ZodNumber;
+            required: z.ZodBoolean;
+            hardGate: z.ZodBoolean;
+            privileged: z.ZodBoolean;
+            fixtureRefs: z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>>;
+        }, z.core.$strict>>;
+        aggregation: z.ZodLiteral<"weighted_mean">;
+        unscorable: z.ZodLiteral<"exclude_optional_require_all_required">;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    rewards: z.ZodArray<z.ZodObject<{
+        schemaVersion: z.ZodLiteral<"openpond.rewardRelease.v1">;
+        id: z.ZodString;
+        revision: z.ZodNumber;
+        name: z.ZodString;
+        description: z.ZodString;
+        implementation: z.ZodUnion<readonly [z.ZodObject<{
+            kind: z.ZodEnum<{
+                content: "content";
+                schema: "schema";
+                artifact: "artifact";
+                runtime_event: "runtime_event";
+                state: "state";
+            }>;
+            config: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"model_judge">;
+            rubricRef: z.ZodObject<{
+                id: z.ZodString;
+                path: z.ZodString;
+                contentHash: z.ZodString;
+                sizeBytes: z.ZodNumber;
+                mediaType: z.ZodString;
+                visibility: z.ZodEnum<{
+                    policy: "policy";
+                    verifier: "verifier";
+                    host_private: "host_private";
+                }>;
+            }, z.core.$strict>;
+            calibrationStatus: z.ZodEnum<{
+                pending: "pending";
+                passed: "passed";
+                failed: "failed";
+            }>;
+            model: z.ZodOptional<z.ZodObject<{
+                providerId: z.ZodString;
+                modelId: z.ZodString;
+                revision: z.ZodNullable<z.ZodString>;
+            }, z.core.$strict>>;
+            temperature: z.ZodOptional<z.ZodNumber>;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"custom_verifier">;
+            networkPolicy: z.ZodLiteral<"none">;
+            verifierRef: z.ZodObject<{
+                id: z.ZodString;
+                path: z.ZodString;
+                contentHash: z.ZodString;
+                sizeBytes: z.ZodNumber;
+                mediaType: z.ZodString;
+                visibility: z.ZodEnum<{
+                    policy: "policy";
+                    verifier: "verifier";
+                    host_private: "host_private";
+                }>;
+            }, z.core.$strict>;
+            exportName: z.ZodOptional<z.ZodString>;
+            timeoutMs: z.ZodNumber;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"human">;
+            rubricRef: z.ZodObject<{
+                id: z.ZodString;
+                path: z.ZodString;
+                contentHash: z.ZodString;
+                sizeBytes: z.ZodNumber;
+                mediaType: z.ZodString;
+                visibility: z.ZodEnum<{
+                    policy: "policy";
+                    verifier: "verifier";
+                    host_private: "host_private";
+                }>;
+            }, z.core.$strict>;
+            reviewerRole: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"learned_model">;
+            modelVersion: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            inputContract: z.ZodObject<{
+                id: z.ZodString;
+                path: z.ZodString;
+                contentHash: z.ZodString;
+                sizeBytes: z.ZodNumber;
+                mediaType: z.ZodString;
+                visibility: z.ZodEnum<{
+                    policy: "policy";
+                    verifier: "verifier";
+                    host_private: "host_private";
+                }>;
+            }, z.core.$strict>;
+        }, z.core.$strict>]>;
+        rawScore: z.ZodObject<{
+            minimum: z.ZodNumber;
+            maximum: z.ZodNumber;
+        }, z.core.$strict>;
+        assets: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            path: z.ZodString;
+            contentHash: z.ZodString;
+            sizeBytes: z.ZodNumber;
+            mediaType: z.ZodString;
+            visibility: z.ZodEnum<{
+                policy: "policy";
+                verifier: "verifier";
+                host_private: "host_private";
+            }>;
+        }, z.core.$strict>>;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    admissions: z.ZodArray<z.ZodObject<{
+        taskId: z.ZodString;
+        evidence: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodNumber;
+        }, z.core.$strict>;
+        decision: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodNumber;
+        }, z.core.$strict>;
+        supervisedTarget: z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodJSONSchema>>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export declare function compileTaskBatch(input: {
+    batch: TaskBatch;
+    definition: TaskDefinition;
+    binding: RewardBinding;
+    rewards: RewardRelease[];
+    evidence: TaskEvidence[];
+    decisions: TaskAdmissionDecision[];
+}): TasksetRelease;
+export declare function taskBatchPackageMetadata(release: TasksetRelease): z.infer<typeof TaskBatchPackageMetadataSchema>;
+//# sourceMappingURL=admission.d.ts.map

package/dist/types/learning/admission.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"admission.d.ts","sourceRoot":"","sources":["../../../../../src/learning/admission.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,EAAsF,KAAK,aAAa,EAAE,KAAK,iBAAiB,EAAE,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AACnL,OAAO,EAAuE,KAAK,UAAU,EAAE,KAAK,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAC3I,OAAO,EAA4C,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACnG,OAAO,EAAwS,KAAK,cAAc,EAAE,KAAK,qBAAqB,EAAE,KAAK,SAAS,EAAE,KAAK,cAAc,EAAE,KAAK,YAAY,EAAE,KAAK,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAE3c,MAAM,MAAM,sBAAsB,GAAG;IACnC,gBAAgB,EAAE,OAAO,GAAG,SAAS,CAAC;IACtC,SAAS,EAAE,OAAO,CAAC;IACnB,mBAAmB,EAAE,OAAO,GAAG,IAAI,CAAC;IACpC,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B,CAAC;AACF,MAAM,MAAM,eAAe,GAAG;IAAE,SAAS,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC;IAAC,KAAK,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAA;CAAE,CAAC;AAElI,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,sBAAsB,CAe9G;AAED,wBAAgB,wBAAwB,CAAC,WAAW,EAAE,cAAc,EAAE,eAAe,EAAE,qBAAqB,GAAG,qBAAqB,CAWnI;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,cAAc,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,qBAAqB,CAAA;CAAE,GAAG;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAmB/M;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAU5E;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,UAAU,CAUrG;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;sBAChD,MAAM,EAAE;;EAClD;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,qBAAqB,CAAC;IAAC,QAAQ,EAAE,YAAY,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,CAAA;CAAE,GAAG,IAAI,CAkB3K;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,iBAAiB,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAGvJ;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,GAAG,eAAe,EAAE,CAM5G;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,CAAC;IAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,SAAS,EAAE,qBAAqB,EAAE,CAAC;IAAC,WAAW,EAAE,eAAe,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAuBvS;AAED,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAOhC,CAAC;AAEZ,wBAAgB,gBAAgB,CAAC,KAAK,EAAE;IAAE,KAAK,EAAE,SAAS,CAAC;IAAC,UAAU,EAAE,cAAc,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAC;IAAC,QAAQ,EAAE,YAAY,EAAE,CAAC;IAAC,SAAS,EAAE,qBAAqB,EAAE,CAAA;CAAE,GAAG,cAAc,CAuBxN;AAED,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAQhH"}

package/dist/types/learning/assets.d.ts

@@ -0,0 +1,49 @@
+import { z } from "zod";
+import { type ImmutableAssetRef } from "@openpond/harness";
+/** Small authored source files. Large binary evidence uses the host asset API. */
+export declare const LearningTextAssetContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.learningTextAsset.v1">;
+    id: z.ZodString;
+    revision: z.ZodLiteral<1>;
+    asset: z.ZodObject<{
+        id: z.ZodString;
+        path: z.ZodString;
+        contentHash: z.ZodString;
+        sizeBytes: z.ZodNumber;
+        mediaType: z.ZodString;
+        visibility: z.ZodEnum<{
+            policy: "policy";
+            verifier: "verifier";
+            host_private: "host_private";
+        }>;
+    }, z.core.$strict>;
+    text: z.ZodString;
+}, z.core.$strict>;
+export declare const LearningTextAssetSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.learningTextAsset.v1">;
+    id: z.ZodString;
+    revision: z.ZodLiteral<1>;
+    asset: z.ZodObject<{
+        id: z.ZodString;
+        path: z.ZodString;
+        contentHash: z.ZodString;
+        sizeBytes: z.ZodNumber;
+        mediaType: z.ZodString;
+        visibility: z.ZodEnum<{
+            policy: "policy";
+            verifier: "verifier";
+            host_private: "host_private";
+        }>;
+    }, z.core.$strict>;
+    text: z.ZodString;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type LearningTextAsset = z.infer<typeof LearningTextAssetSchema>;
+export declare function createLearningTextAsset(input: {
+    text: string;
+    path: string;
+    mediaType: string;
+    visibility: ImmutableAssetRef["visibility"];
+}): LearningTextAsset;
+export declare function verifyLearningTextAsset(asset: LearningTextAsset, reference: ImmutableAssetRef): string;
+//# sourceMappingURL=assets.d.ts.map

package/dist/types/learning/assets.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"assets.d.ts","sourceRoot":"","sources":["../../../../../src/learning/assets.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAoF,KAAK,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAI7I,kFAAkF;AAClF,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;kBAUzC,CAAC;AACH,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;kBAAyF,CAAC;AAC9H,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AAExE,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,iBAAiB,CAAC,YAAY,CAAC,CAAA;CAAE,GAAG,iBAAiB,CAUhK;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,iBAAiB,GAAG,MAAM,CAItG"}