@openpond/evals 0.1.0

package/dist/types/runs.d.ts

@@ -0,0 +1,293 @@
+import { z } from "zod";
+export declare const ModelRefSchema: z.ZodObject<{
+    provider: z.ZodString;
+    model: z.ZodString;
+    revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+}, z.core.$strict>;
+export declare const RuntimeTargetBindingSchema: z.ZodObject<{
+    adapterId: z.ZodString;
+    placement: z.ZodEnum<{
+        local: "local";
+        remote: "remote";
+        colocated: "colocated";
+        provider_native: "provider_native";
+    }>;
+    runtimeVersion: z.ZodString;
+    capabilityReceipt: z.ZodString;
+}, z.core.$strict>;
+export declare const RunLimitsSchema: z.ZodObject<{
+    maxTurns: z.ZodNumber;
+    timeoutMs: z.ZodNumber;
+    maxOutputBytes: z.ZodNumber;
+    maximumSpendUsd: z.ZodNullable<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const RunManifestContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.runManifest.v1">;
+    id: z.ZodString;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    model: z.ZodObject<{
+        provider: z.ZodString;
+        model: z.ZodString;
+        revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    }, z.core.$strict>;
+    runtimeTarget: z.ZodObject<{
+        adapterId: z.ZodString;
+        placement: z.ZodEnum<{
+            local: "local";
+            remote: "remote";
+            colocated: "colocated";
+            provider_native: "provider_native";
+        }>;
+        runtimeVersion: z.ZodString;
+        capabilityReceipt: z.ZodString;
+    }, z.core.$strict>;
+    limits: z.ZodObject<{
+        maxTurns: z.ZodNumber;
+        timeoutMs: z.ZodNumber;
+        maxOutputBytes: z.ZodNumber;
+        maximumSpendUsd: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    approval: z.ZodNullable<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>;
+export declare const RunManifestSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.runManifest.v1">;
+    id: z.ZodString;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    model: z.ZodObject<{
+        provider: z.ZodString;
+        model: z.ZodString;
+        revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    }, z.core.$strict>;
+    runtimeTarget: z.ZodObject<{
+        adapterId: z.ZodString;
+        placement: z.ZodEnum<{
+            local: "local";
+            remote: "remote";
+            colocated: "colocated";
+            provider_native: "provider_native";
+        }>;
+        runtimeVersion: z.ZodString;
+        capabilityReceipt: z.ZodString;
+    }, z.core.$strict>;
+    limits: z.ZodObject<{
+        maxTurns: z.ZodNumber;
+        timeoutMs: z.ZodNumber;
+        maxOutputBytes: z.ZodNumber;
+        maximumSpendUsd: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    approval: z.ZodNullable<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export declare const AttemptReceiptContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.attemptReceipt.v1">;
+    id: z.ZodString;
+    runManifest: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    taskId: z.ZodString;
+    seed: z.ZodString;
+    terminal: z.ZodBoolean;
+    failureClass: z.ZodNullable<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>>;
+    outputHash: z.ZodNullable<z.ZodString>;
+    traceHash: z.ZodString;
+    artifactRefs: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+    }, z.core.$strict>>;
+    graderEvidenceRefs: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+    }, z.core.$strict>>;
+    startedAt: z.ZodString;
+    completedAt: z.ZodString;
+    latencyMs: z.ZodNumber;
+    costUsd: z.ZodNullable<z.ZodNumber>;
+    legacyAttemptRef: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>;
+export declare const AttemptReceiptSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.attemptReceipt.v1">;
+    id: z.ZodString;
+    runManifest: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    taskId: z.ZodString;
+    seed: z.ZodString;
+    terminal: z.ZodBoolean;
+    failureClass: z.ZodNullable<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>>;
+    outputHash: z.ZodNullable<z.ZodString>;
+    traceHash: z.ZodString;
+    artifactRefs: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+    }, z.core.$strict>>;
+    graderEvidenceRefs: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+    }, z.core.$strict>>;
+    startedAt: z.ZodString;
+    completedAt: z.ZodString;
+    latencyMs: z.ZodNumber;
+    costUsd: z.ZodNullable<z.ZodNumber>;
+    legacyAttemptRef: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export declare const EvaluationResultContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.evaluationResult.v1">;
+    id: z.ZodString;
+    runManifest: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    model: z.ZodObject<{
+        provider: z.ZodString;
+        model: z.ZodString;
+        revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    }, z.core.$strict>;
+    receiptRefs: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    attemptCount: z.ZodNumber;
+    rewardEligibleCount: z.ZodNumber;
+    terminalCount: z.ZodNumber;
+    meanScore: z.ZodNullable<z.ZodNumber>;
+    failureCounts: z.ZodRecord<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>, z.ZodNumber>;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strict>;
+export declare const EvaluationResultSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.evaluationResult.v1">;
+    id: z.ZodString;
+    runManifest: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    harnessRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    tasksetRelease: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    model: z.ZodObject<{
+        provider: z.ZodString;
+        model: z.ZodString;
+        revision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        artifactHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        tokenizerRevision: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        chatTemplateHash: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+    }, z.core.$strict>;
+    receiptRefs: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>>;
+    attemptCount: z.ZodNumber;
+    rewardEligibleCount: z.ZodNumber;
+    terminalCount: z.ZodNumber;
+    meanScore: z.ZodNullable<z.ZodNumber>;
+    failureCounts: z.ZodRecord<z.ZodEnum<{
+        policy_failure: "policy_failure";
+        grader_failure: "grader_failure";
+        environment_failure: "environment_failure";
+        infrastructure_failure: "infrastructure_failure";
+        timeout: "timeout";
+        cancelled: "cancelled";
+    }>, z.ZodNumber>;
+    metadata: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export declare function createRunManifest(input: z.input<typeof RunManifestContentSchema>): RunManifest;
+export declare function createAttemptReceipt(input: z.input<typeof AttemptReceiptContentSchema>): AttemptReceipt;
+export declare function verifyAttemptReceipt(receipt: AttemptReceipt): boolean;
+export declare function aggregateEvaluationReceipts(input: {
+    id: string;
+    manifest: RunManifest;
+    receipts: AttemptReceipt[];
+    metadata?: Record<string, unknown>;
+}): EvaluationResult;
+export declare function assertComparableRunManifests(base: RunManifest, candidate: RunManifest): void;
+export declare function rewardEligibleReceipts(receipts: AttemptReceipt[]): AttemptReceipt[];
+export type ModelRef = z.infer<typeof ModelRefSchema>;
+export type RuntimeTargetBinding = z.infer<typeof RuntimeTargetBindingSchema>;
+export type RunLimits = z.infer<typeof RunLimitsSchema>;
+export type RunManifest = z.infer<typeof RunManifestSchema>;
+export type AttemptReceipt = z.infer<typeof AttemptReceiptSchema>;
+export type EvaluationResult = z.infer<typeof EvaluationResultSchema>;
+//# sourceMappingURL=runs.d.ts.map

package/dist/types/runs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runs.d.ts","sourceRoot":"","sources":["../../src/runs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,eAAO,MAAM,cAAc;;;;;;;kBAOhB,CAAC;AAEZ,eAAO,MAAM,0BAA0B;;;;;;;;;;kBAK5B,CAAC;AAEZ,eAAO,MAAM,eAAe;;;;;kBAKjB,CAAC;AAEZ,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAW1B,CAAC;AACZ,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAA+E,CAAC;AAE9G,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkB7B,CAAC;AACZ,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAkF,CAAC;AAEpH,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAc/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AAExH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,GAAG,WAAW,CAG9F;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,GAAG,cAAc,CAGvG;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAKrE;AAED,wBAAgB,2BAA2B,CAAC,KAAK,EAAE;IACjD,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,WAAW,CAAC;IACtB,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAgCnB;AAED,wBAAgB,4BAA4B,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,WAAW,GAAG,IAAI,CAO5F;AAED,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CASnF;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AACtD,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC5D,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAClE,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC"}

package/dist/types/sha256.d.ts

@@ -0,0 +1,2 @@
+export declare function sha256Hex(value: string | Uint8Array): string;
+//# sourceMappingURL=sha256.d.ts.map

package/dist/types/sha256.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sha256.d.ts","sourceRoot":"","sources":["../../src/sha256.ts"],"names":[],"mappings":"AAgCA,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAgE5D"}