@openpond/evals 0.1.0

package/dist/types/conformance.d.ts

@@ -0,0 +1,573 @@
+export declare const genericToolConformance: {
+    snapshot: {
+        schemaVersion: "openpond.agentSnapshot.v1";
+        id: string;
+        profileRelease: {
+            id: string;
+            contentHash: string;
+        } | null;
+        instructions: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        }[];
+        skills: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        }[];
+        agents: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        }[];
+        toolDeclarations: {
+            name: string;
+            description: string;
+            inputSchema: Record<string, unknown>;
+            inputSchemaHash: string;
+            sideEffect: "read" | "write";
+            timeoutMs: number;
+        }[];
+        capabilityRequirements: {
+            id: string;
+            required: boolean;
+            scopes: string[];
+        }[];
+        dependencyLock: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        };
+        portability: {
+            portable: boolean;
+            blockers: string[];
+            localOnlyAssetRefs: string[];
+            hostPrivateAssetRefs: string[];
+        };
+        metadata: Record<string, unknown>;
+        contentHash: string;
+    };
+    harness: {
+        schemaVersion: "openpond.harnessRelease.v1";
+        id: string;
+        agentSnapshot: {
+            id: string;
+            contentHash: string;
+        };
+        program: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        };
+        environment: {
+            protocolVersion: "openpond.environment.v1";
+            kind: "text" | "agent" | "work" | "custom_program";
+            entrypoint: string;
+            stateful: boolean;
+            deterministicSeeds: boolean;
+            lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
+            networkPolicy: "none" | "declared_read_only" | "declared_scoped";
+            defaultTimeoutMs: number;
+        };
+        tools: {
+            name: string;
+            description: string;
+            inputSchema: Record<string, unknown>;
+            inputSchemaHash: string;
+            sideEffect: "read" | "write";
+            timeoutMs: number;
+        }[];
+        lifecycle: {
+            create: true;
+            reset: true;
+            step: true;
+            collect: true;
+            destroy: true;
+            resetScope: "task" | "attempt";
+        };
+        graderInterface: {
+            visibleEvidence: string[];
+            privilegedEvidence: string[];
+            privateVerifierIsolation: boolean;
+        };
+        policy: {
+            policyVisibleFields: string[];
+            privilegedFields: string[];
+            hiddenGraderRefs: string[];
+            connectedAppScopes: string[];
+        };
+        files: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        }[];
+        metadata: Record<string, unknown>;
+        contentHash: string;
+    };
+    taskset: {
+        schemaVersion: "openpond.tasksetRelease.v1";
+        id: string;
+        revision: number;
+        harnessRelease: {
+            id: string;
+            contentHash: string;
+        };
+        policy: {
+            policyVisibleFields: string[];
+            privilegedFields: string[];
+            hiddenGraderRefs: string[];
+            connectedAppScopes: string[];
+        };
+        environment: {
+            protocolVersion: "openpond.environment.v1";
+            kind: "text" | "agent" | "work" | "custom_program";
+            entrypoint: string;
+            stateful: boolean;
+            deterministicSeeds: boolean;
+            lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
+            networkPolicy: "none" | "declared_read_only" | "declared_scoped";
+            defaultTimeoutMs: number;
+        };
+        tools: {
+            name: string;
+            description: string;
+            inputSchema: Record<string, unknown>;
+            inputSchemaHash: string;
+            sideEffect: "read" | "write";
+            timeoutMs: number;
+        }[];
+        capabilities: {
+            id: string;
+            required: boolean;
+            scopes: string[];
+            portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
+        }[];
+        tasks: {
+            id: string;
+            clusterKey: string;
+            split: "train" | "validation" | "test" | "frozen_eval";
+            input: Record<string, unknown>;
+            expectedOutput: Record<string, unknown> | null;
+            policyVisibleContext: Record<string, unknown>;
+            privilegedContextRef: string | null;
+            artifactRefs: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            }[];
+            tags: string[];
+        }[];
+        graders: ({
+            id: string;
+            version: string;
+            weight: number;
+            hardGate: boolean;
+            rewardEligible: boolean;
+            privileged: boolean;
+            kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
+            config: Record<string, unknown>;
+        } | {
+            id: string;
+            version: string;
+            weight: number;
+            hardGate: boolean;
+            rewardEligible: boolean;
+            privileged: boolean;
+            kind: "model_judge";
+            rubricRef: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            };
+            calibrationStatus: "pending" | "passed" | "failed";
+        } | {
+            id: string;
+            version: string;
+            weight: number;
+            hardGate: boolean;
+            rewardEligible: boolean;
+            privileged: boolean;
+            kind: "custom_verifier";
+            verifierRef: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            };
+            timeoutMs: number;
+            networkPolicy: "none";
+        } | {
+            id: string;
+            version: string;
+            weight: number;
+            hardGate: boolean;
+            rewardEligible: boolean;
+            privileged: boolean;
+            kind: "human";
+            rubricRef: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            };
+            reviewerRole: string;
+        })[];
+        metadata: Record<string, unknown>;
+        contentHash: string;
+    };
+    manifest: {
+        schemaVersion: "openpond.runManifest.v1";
+        id: string;
+        harnessRelease: {
+            id: string;
+            contentHash: string;
+        };
+        tasksetRelease: {
+            id: string;
+            contentHash: string;
+        };
+        model: {
+            provider: string;
+            model: string;
+            revision: string | null;
+            artifactHash: string | null;
+            tokenizerRevision: string | null;
+            chatTemplateHash: string | null;
+        };
+        runtimeTarget: {
+            adapterId: string;
+            placement: "local" | "remote" | "colocated" | "provider_native";
+            runtimeVersion: string;
+            capabilityReceipt: string;
+        };
+        limits: {
+            maxTurns: number;
+            timeoutMs: number;
+            maxOutputBytes: number;
+            maximumSpendUsd: number | null;
+        };
+        approval: {
+            id: string;
+            contentHash: string;
+        } | null;
+        createdAt: string;
+        metadata: Record<string, unknown>;
+        contentHash: string;
+    };
+};
+export declare const marketingPortfolioConformance: {
+    snapshot: {
+        schemaVersion: "openpond.agentSnapshot.v1";
+        id: string;
+        profileRelease: {
+            id: string;
+            contentHash: string;
+        } | null;
+        instructions: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        }[];
+        skills: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        }[];
+        agents: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        }[];
+        toolDeclarations: {
+            name: string;
+            description: string;
+            inputSchema: Record<string, unknown>;
+            inputSchemaHash: string;
+            sideEffect: "read" | "write";
+            timeoutMs: number;
+        }[];
+        capabilityRequirements: {
+            id: string;
+            required: boolean;
+            scopes: string[];
+        }[];
+        dependencyLock: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        };
+        portability: {
+            portable: boolean;
+            blockers: string[];
+            localOnlyAssetRefs: string[];
+            hostPrivateAssetRefs: string[];
+        };
+        metadata: Record<string, unknown>;
+        contentHash: string;
+    };
+    harness: {
+        schemaVersion: "openpond.harnessRelease.v1";
+        id: string;
+        agentSnapshot: {
+            id: string;
+            contentHash: string;
+        };
+        program: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        };
+        environment: {
+            protocolVersion: "openpond.environment.v1";
+            kind: "text" | "agent" | "work" | "custom_program";
+            entrypoint: string;
+            stateful: boolean;
+            deterministicSeeds: boolean;
+            lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
+            networkPolicy: "none" | "declared_read_only" | "declared_scoped";
+            defaultTimeoutMs: number;
+        };
+        tools: {
+            name: string;
+            description: string;
+            inputSchema: Record<string, unknown>;
+            inputSchemaHash: string;
+            sideEffect: "read" | "write";
+            timeoutMs: number;
+        }[];
+        lifecycle: {
+            create: true;
+            reset: true;
+            step: true;
+            collect: true;
+            destroy: true;
+            resetScope: "task" | "attempt";
+        };
+        graderInterface: {
+            visibleEvidence: string[];
+            privilegedEvidence: string[];
+            privateVerifierIsolation: boolean;
+        };
+        policy: {
+            policyVisibleFields: string[];
+            privilegedFields: string[];
+            hiddenGraderRefs: string[];
+            connectedAppScopes: string[];
+        };
+        files: {
+            id: string;
+            path: string;
+            contentHash: string;
+            sizeBytes: number;
+            mediaType: string;
+            visibility: "policy" | "verifier" | "host_private";
+        }[];
+        metadata: Record<string, unknown>;
+        contentHash: string;
+    };
+    taskset: {
+        schemaVersion: "openpond.tasksetRelease.v1";
+        id: string;
+        revision: number;
+        harnessRelease: {
+            id: string;
+            contentHash: string;
+        };
+        policy: {
+            policyVisibleFields: string[];
+            privilegedFields: string[];
+            hiddenGraderRefs: string[];
+            connectedAppScopes: string[];
+        };
+        environment: {
+            protocolVersion: "openpond.environment.v1";
+            kind: "text" | "agent" | "work" | "custom_program";
+            entrypoint: string;
+            stateful: boolean;
+            deterministicSeeds: boolean;
+            lifecycle: ("create" | "reset" | "step" | "collect" | "destroy")[];
+            networkPolicy: "none" | "declared_read_only" | "declared_scoped";
+            defaultTimeoutMs: number;
+        };
+        tools: {
+            name: string;
+            description: string;
+            inputSchema: Record<string, unknown>;
+            inputSchemaHash: string;
+            sideEffect: "read" | "write";
+            timeoutMs: number;
+        }[];
+        capabilities: {
+            id: string;
+            required: boolean;
+            scopes: string[];
+            portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
+        }[];
+        tasks: {
+            id: string;
+            clusterKey: string;
+            split: "train" | "validation" | "test" | "frozen_eval";
+            input: Record<string, unknown>;
+            expectedOutput: Record<string, unknown> | null;
+            policyVisibleContext: Record<string, unknown>;
+            privilegedContextRef: string | null;
+            artifactRefs: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            }[];
+            tags: string[];
+        }[];
+        graders: ({
+            id: string;
+            version: string;
+            weight: number;
+            hardGate: boolean;
+            rewardEligible: boolean;
+            privileged: boolean;
+            kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
+            config: Record<string, unknown>;
+        } | {
+            id: string;
+            version: string;
+            weight: number;
+            hardGate: boolean;
+            rewardEligible: boolean;
+            privileged: boolean;
+            kind: "model_judge";
+            rubricRef: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            };
+            calibrationStatus: "pending" | "passed" | "failed";
+        } | {
+            id: string;
+            version: string;
+            weight: number;
+            hardGate: boolean;
+            rewardEligible: boolean;
+            privileged: boolean;
+            kind: "custom_verifier";
+            verifierRef: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            };
+            timeoutMs: number;
+            networkPolicy: "none";
+        } | {
+            id: string;
+            version: string;
+            weight: number;
+            hardGate: boolean;
+            rewardEligible: boolean;
+            privileged: boolean;
+            kind: "human";
+            rubricRef: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "policy" | "verifier" | "host_private";
+            };
+            reviewerRole: string;
+        })[];
+        metadata: Record<string, unknown>;
+        contentHash: string;
+    };
+    manifest: {
+        schemaVersion: "openpond.runManifest.v1";
+        id: string;
+        harnessRelease: {
+            id: string;
+            contentHash: string;
+        };
+        tasksetRelease: {
+            id: string;
+            contentHash: string;
+        };
+        model: {
+            provider: string;
+            model: string;
+            revision: string | null;
+            artifactHash: string | null;
+            tokenizerRevision: string | null;
+            chatTemplateHash: string | null;
+        };
+        runtimeTarget: {
+            adapterId: string;
+            placement: "local" | "remote" | "colocated" | "provider_native";
+            runtimeVersion: string;
+            capabilityReceipt: string;
+        };
+        limits: {
+            maxTurns: number;
+            timeoutMs: number;
+            maxOutputBytes: number;
+            maximumSpendUsd: number | null;
+        };
+        approval: {
+            id: string;
+            contentHash: string;
+        } | null;
+        createdAt: string;
+        metadata: Record<string, unknown>;
+        contentHash: string;
+    };
+};
+//# sourceMappingURL=conformance.d.ts.map

package/dist/types/conformance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"conformance.d.ts","sourceRoot":"","sources":["../../src/conformance.ts"],"names":[],"mappings":"AASA,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAEjC,CAAC;AAEH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAGxC,CAAC"}

package/dist/types/graders.d.ts

@@ -0,0 +1,70 @@
+import { z } from "zod";
+import type { GraderSpec, TaskRecord } from "./tasksets.js";
+export declare const GraderEvidenceContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
+    graderId: z.ZodString;
+    graderVersion: z.ZodString;
+    score: z.ZodNullable<z.ZodNumber>;
+    passed: z.ZodBoolean;
+    rewardEligible: z.ZodBoolean;
+    failureClass: z.ZodNullable<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>>;
+    feedback: z.ZodArray<z.ZodString>;
+    visibleEvidenceRefs: z.ZodArray<z.ZodString>;
+    privilegedEvidenceRefs: z.ZodArray<z.ZodString>;
+}, z.core.$strict>;
+export declare const GraderEvidenceSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
+    graderId: z.ZodString;
+    graderVersion: z.ZodString;
+    score: z.ZodNullable<z.ZodNumber>;
+    passed: z.ZodBoolean;
+    rewardEligible: z.ZodBoolean;
+    failureClass: z.ZodNullable<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>>;
+    feedback: z.ZodArray<z.ZodString>;
+    visibleEvidenceRefs: z.ZodArray<z.ZodString>;
+    privilegedEvidenceRefs: z.ZodArray<z.ZodString>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type AttemptEvidence = {
+    output: Record<string, unknown>;
+    runtimeEventRefs: string[];
+    artifactRefs: string[];
+    infrastructureError?: string | null;
+};
+export type ModelJudgeRunner = (input: {
+    grader: Extract<GraderSpec, {
+        kind: "model_judge";
+    }>;
+    task: TaskRecord;
+    evidence: AttemptEvidence;
+}) => Promise<Omit<GraderEvidence, "schemaVersion" | "graderId" | "graderVersion" | "contentHash">>;
+export type CustomVerifierRunner = (input: {
+    grader: Extract<GraderSpec, {
+        kind: "custom_verifier";
+    }>;
+    task: TaskRecord;
+    evidence: AttemptEvidence;
+}) => Promise<Omit<GraderEvidence, "schemaVersion" | "graderId" | "graderVersion" | "contentHash">>;
+export declare function gradeEvidence(input: {
+    task: TaskRecord;
+    evidence: AttemptEvidence;
+    graders: GraderSpec[];
+    modelJudge?: ModelJudgeRunner;
+    customVerifier?: CustomVerifierRunner;
+}): Promise<GraderEvidence[]>;
+export type GraderEvidence = z.infer<typeof GraderEvidenceSchema>;
+//# sourceMappingURL=graders.d.ts.map

package/dist/types/graders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graders.d.ts","sourceRoot":"","sources":["../../src/graders.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EAA2B,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAErF,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;kBAW7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,MAAM,MAAM,eAAe,GAAG;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,aAAa,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAC/O,MAAM,MAAM,oBAAoB,GAAG,CAAC,KAAK,EAAE;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,iBAAiB,CAAA;KAAE,CAAC,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,eAAe,CAAA;CAAE,KAAK,OAAO,CAAC,IAAI,CAAC,cAAc,EAAE,eAAe,GAAG,UAAU,GAAG,eAAe,GAAG,aAAa,CAAC,CAAC,CAAC;AAEvP,wBAAsB,aAAa,CAAC,KAAK,EAAE;IACzC,IAAI,EAAE,UAAU,CAAC;IACjB,QAAQ,EAAE,eAAe,CAAC;IAC1B,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB5B;AA4CD,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC"}