@openpond/evals 0.11.2 → 0.11.3

package/dist/index.js

@@ -14,6 +14,7 @@ export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./profile-evaluations.js";
 export * from "./profile-evaluation-comparisons.js";
+export * from "./profile-evaluation-execution.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";
 export * from "./telemetry.js";

package/dist/profile-evaluation-execution.js

@@ -0,0 +1,76 @@
+import { ImmutableArtifactRefSchema, contentHash } from "@openpond/harness";
+import { gradeTaskEvidence } from "./graders.js";
+import { aggregateTasksetRunReceipts, assertTasksetMetricSource } from "./metrics.js";
+import { createAttemptReceipt } from "./runs.js";
+import { assertProfileEvaluationRunAdmission } from "./taskset-run-contract.js";
+import { policyTaskView } from "./tasksets.js";
+/** The installation owns execution, tools, approvals and storage. This runner
+ * owns frozen population dispatch and private grading for both Desktop and web. */
+export async function executeProfileEvaluationRun(input) {
+    assertProfileEvaluationRunAdmission(input.manifest, input.taskset, input.catalog);
+    assertTasksetMetricSource(input.taskset, input.metricSource);
+    const source = input.manifest.profileEvaluation;
+    const definition = input.catalog.definitions.find((item) => item.id === source.definitionId);
+    const tasks = new Map(input.taskset.tasks.map((task) => [task.id, task]));
+    const receipts = [];
+    const grades = [];
+    for (const member of input.manifest.population) {
+        input.signal?.throwIfAborted();
+        const task = tasks.get(member.taskId);
+        const execution = await input.execute({
+            task: policyTaskView(task), seed: member.seed, source,
+            ...(input.signal ? { signal: input.signal } : {}),
+        });
+        input.signal?.throwIfAborted();
+        const grade = await gradeTaskEvidence({
+            task, evidence: execution.evidence, graders: input.taskset.graders,
+            ...(input.modelJudge ? { modelJudge: input.modelJudge } : {}),
+            ...(input.customVerifier ? { customVerifier: input.customVerifier } : {}),
+            ...(input.signal ? { signal: input.signal } : {}),
+        });
+        const gradeRef = ImmutableArtifactRefSchema.parse(await input.saveGrade(grade));
+        if (gradeRef.contentHash !== grade.contentHash)
+            throw new Error("Stored Profile evaluation grade differs from its graded evidence.");
+        const failureClass = execution.failureClass ?? grade.failureClass;
+        const receipt = createAttemptReceipt({
+            schemaVersion: "openpond.attemptReceipt.v1",
+            id: member.receiptId,
+            runManifest: { id: input.manifest.id, contentHash: input.manifest.contentHash },
+            taskId: member.taskId,
+            seed: member.seed,
+            terminal: execution.terminal,
+            failureClass,
+            outputHash: execution.evidence.infrastructureError ? null : contentHash(execution.evidence.output),
+            traceHash: execution.traceHash,
+            artifactRefs: execution.artifactRefs,
+            graderEvidenceRefs: [gradeRef],
+            startedAt: execution.startedAt,
+            completedAt: execution.completedAt,
+            latencyMs: execution.latencyMs,
+            costUsd: execution.costUsd,
+            metadata: {
+                gradingRole: "evaluation",
+                score: grade.score,
+                passed: grade.passed,
+                rewardEligible: grade.rewardEligible,
+                sourceRevision: source.sourceRevision,
+                definitionHash: source.definitionHash,
+                environmentHash: source.environmentHash,
+            },
+        });
+        await input.saveReceipt(receipt);
+        grades.push(grade);
+        receipts.push(receipt);
+    }
+    const metric = await aggregateTasksetRunReceipts({
+        manifest: input.manifest, taskset: input.taskset, receipts,
+        ...(input.metricSource !== undefined ? { source: input.metricSource } : {}),
+        ...(input.signal ? { signal: input.signal } : {}),
+    }, input.metricExecutor);
+    const passRate = grades.filter((grade) => grade.passed).length / grades.length;
+    return {
+        receipts, grades, metric, passRate,
+        passed: passRate >= definition.criterion.minimumPassRate
+            && (!definition.criterion.requireComplete || receipts.every((receipt) => receipt.terminal)),
+    };
+}

package/dist/taskset-run-contract.js

@@ -2,7 +2,7 @@ import { z } from "zod";
 import { assertContentHash, contentHash, ImmutableReleaseRefSchema, MetadataSchema, ModelRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
 import { RunLimitsSchema, RuntimeTargetBindingSchema, verifyAttemptReceipt } from "./runs.js";
 import { TasksetMetricPolicySchema } from "./metric-policy.js";
-import { ProfileEvaluationRunSourceSchema } from "./profile-evaluations.js";
+import { ProfileEvaluationCatalogSchema, ProfileEvaluationRunSourceSchema } from "./profile-evaluations.js";
 import { assertTasksetRelease } from "./tasksets.js";
 /** A fixture check owns no model identity and cannot become a model evaluation. */
 export const TasksetRunPolicySchema = z.discriminatedUnion("kind", [
@@ -75,6 +75,39 @@ export function assertTasksetRunRelease(manifest, taskset) {
         || manifest.execution.policyHash !== contentHash(taskset.policy)))
         throw new Error("Taskset-owned execution differs from its pinned release.");
 }
+/** Resolve the verifier-only definition before dispatch. A Profile run must
+ * evaluate exactly its declared frozen cases and seeds through its release. */
+export function assertProfileEvaluationRunAdmission(manifest, taskset, catalogInput) {
+    assertTasksetRunRelease(manifest, taskset);
+    const catalog = ProfileEvaluationCatalogSchema.parse(catalogInput);
+    const source = manifest.profileEvaluation;
+    if (!source || manifest.execution.kind !== "harness" || manifest.policy.kind !== "model") {
+        throw new Error("Profile evaluation requires fresh model execution through its bound Harness.");
+    }
+    if (source.catalogHash !== contentHash(catalog))
+        throw new Error("Profile evaluation catalog differs from its pinned source.");
+    const definition = catalog.definitions.find((entry) => entry.id === source.definitionId);
+    if (!definition || source.definitionHash !== contentHash(definition)
+        || contentHash(source.target) !== contentHash(definition.target)) {
+        throw new Error("Profile evaluation definition differs from its pinned source.");
+    }
+    if (definition.tasksetRelease.id !== taskset.id || definition.tasksetRelease.contentHash !== taskset.contentHash) {
+        throw new Error("Profile evaluation Taskset differs from its pinned definition.");
+    }
+    if (definition.split === "train")
+        throw new Error("Profile evaluation cannot execute training cases.");
+    const tasks = new Map(taskset.tasks.map((task) => [task.id, task]));
+    if (definition.taskIds.some((id) => tasks.get(id)?.split !== definition.split)) {
+        throw new Error("Profile evaluation case is absent from its declared Taskset split.");
+    }
+    const expected = new Set(definition.taskIds.flatMap((taskId) => definition.seeds.map((seed) => JSON.stringify([taskId, seed]))));
+    const actual = new Set(manifest.population.map(({ taskId, seed }) => JSON.stringify([taskId, seed])));
+    if (manifest.population.some((member) => member.fixtureId !== null)
+        || actual.size !== expected.size || manifest.population.length !== expected.size
+        || [...actual].some((member) => !expected.has(member))) {
+        throw new Error("Profile evaluation population differs from its declared cases and seeds.");
+    }
+}
 /** Complete runs cannot drop failed members, add convenient attempts or relabel
  * another task/seed's evidence. Ordering is always the admitted population's. */
 export function orderTasksetRunReceipts(manifest, receipts) {

package/dist/types/index.d.ts

@@ -14,6 +14,7 @@ export * from "./model-improvement-qualification.js";
 export * from "./preferences.js";
 export * from "./profile-evaluations.js";
 export * from "./profile-evaluation-comparisons.js";
+export * from "./profile-evaluation-execution.js";
 export * from "./review-conformance.js";
 export * from "./rollouts.js";
 export * from "./telemetry.js";

package/dist/types/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qBAAqB,CAAC;AACpC,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,yBAAyB,CAAC;AACxC,cAAc,WAAW,CAAC;AAC1B,cAAc,cAAc,CAAC;AAC7B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,sCAAsC,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,0BAA0B,CAAC;AACzC,cAAc,qCAAqC,CAAC;AACpD,cAAc,mCAAmC,CAAC;AAClD,cAAc,yBAAyB,CAAC;AACxC,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC"}

package/dist/types/profile-evaluation-execution.d.ts

@@ -0,0 +1,46 @@
+import { type ImmutableArtifactRef } from "@openpond/harness";
+import { type AttemptEvidence, type CustomVerifierRunner, type ModelJudgeRunner, type TaskGrade } from "./graders.js";
+import { type TasksetMetricExecutor, type TasksetMetricResult } from "./metrics.js";
+import { type AttemptReceipt } from "./runs.js";
+import { type TasksetRunManifest } from "./taskset-run-contract.js";
+import { policyTaskView, type TasksetRelease } from "./tasksets.js";
+import type { ProfileEvaluationCatalog, ProfileEvaluationRunSource } from "./profile-evaluations.js";
+type PolicyTask = ReturnType<typeof policyTaskView>;
+/** The installation owns execution, tools, approvals and storage. This runner
+ * owns frozen population dispatch and private grading for both Desktop and web. */
+export declare function executeProfileEvaluationRun(input: {
+    manifest: TasksetRunManifest;
+    taskset: TasksetRelease;
+    catalog: ProfileEvaluationCatalog;
+    execute: (member: {
+        task: PolicyTask;
+        seed: string;
+        source: ProfileEvaluationRunSource;
+        signal?: AbortSignal;
+    }) => Promise<{
+        evidence: AttemptEvidence;
+        traceHash: string;
+        artifactRefs: ImmutableArtifactRef[];
+        startedAt: string;
+        completedAt: string;
+        latencyMs: number;
+        costUsd: number | null;
+        terminal: boolean;
+        failureClass?: AttemptReceipt["failureClass"];
+    }>;
+    saveGrade: (grade: TaskGrade) => Promise<ImmutableArtifactRef>;
+    saveReceipt: (receipt: AttemptReceipt) => Promise<void>;
+    modelJudge?: ModelJudgeRunner;
+    customVerifier?: CustomVerifierRunner;
+    metricSource?: string;
+    metricExecutor?: TasksetMetricExecutor;
+    signal?: AbortSignal;
+}): Promise<{
+    receipts: AttemptReceipt[];
+    grades: TaskGrade[];
+    metric: TasksetMetricResult;
+    passRate: number;
+    passed: boolean;
+}>;
+export {};
+//# sourceMappingURL=profile-evaluation-execution.d.ts.map

package/dist/types/profile-evaluation-execution.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profile-evaluation-execution.d.ts","sourceRoot":"","sources":["../../../../src/profile-evaluation-execution.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEvG,OAAO,EAAqB,KAAK,eAAe,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAE,MAAM,cAAc,CAAC;AACzI,OAAO,EAA0D,KAAK,qBAAqB,EAAE,KAAK,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAC5I,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAuC,KAAK,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AACzG,OAAO,EAAE,cAAc,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAErG,KAAK,UAAU,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAEpD;mFACmF;AACnF,wBAAsB,2BAA2B,CAAC,KAAK,EAAE;IACvD,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,OAAO,EAAE,cAAc,CAAC;IACxB,OAAO,EAAE,wBAAwB,CAAC;IAClC,OAAO,EAAE,CAAC,MAAM,EAAE;QAChB,IAAI,EAAE,UAAU,CAAC;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,MAAM,EAAE,0BAA0B,CAAC;QACnC,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,KAAK,OAAO,CAAC;QACZ,QAAQ,EAAE,eAAe,CAAC;QAC1B,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,oBAAoB,EAAE,CAAC;QACrC,SAAS,EAAE,MAAM,CAAC;QAClB,WAAW,EAAE,MAAM,CAAC;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;QACvB,QAAQ,EAAE,OAAO,CAAC;QAClB,YAAY,CAAC,EAAE,cAAc,CAAC,cAAc,CAAC,CAAC;KAC/C,CAAC,CAAC;IACH,SAAS,EAAE,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC/D,WAAW,EAAE,CAAC,OAAO,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACxD,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,oBAAoB,CAAC;IACtC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IACV,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,MAAM,EAAE,mBAAmB,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,OAAO,CAAC;CACjB,CAAC,CAkED"}

package/dist/types/taskset-run-contract.d.ts

@@ -1,6 +1,7 @@
 import { z } from "zod";
 import { type AttemptReceipt } from "./runs.js";
 import { type TasksetMetricPolicy } from "./metric-policy.js";
+import { type ProfileEvaluationCatalog } from "./profile-evaluations.js";
 import { type TasksetRelease } from "./tasksets.js";
 /** A fixture check owns no model identity and cannot become a model evaluation. */
 export declare const TasksetRunPolicySchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
@@ -274,6 +275,9 @@ export declare function tasksetRunMetricPolicy(taskset: TasksetRelease): Taskset
 /** Admission verifies this before dispatch. Package byte verification belongs
  * to the package loader; this validates the release and population semantics. */
 export declare function assertTasksetRunRelease(manifest: TasksetRunManifest, taskset: TasksetRelease): void;
+/** Resolve the verifier-only definition before dispatch. A Profile run must
+ * evaluate exactly its declared frozen cases and seeds through its release. */
+export declare function assertProfileEvaluationRunAdmission(manifest: TasksetRunManifest, taskset: TasksetRelease, catalogInput: ProfileEvaluationCatalog): void;
 /** Complete runs cannot drop failed members, add convenient attempts or relabel
  * another task/seed's evidence. Ordering is always the admitted population's. */
 export declare function orderTasksetRunReceipts(manifest: TasksetRunManifest, receipts: AttemptReceipt[]): AttemptReceipt[];

package/dist/types/taskset-run-contract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"taskset-run-contract.d.ts","sourceRoot":"","sources":["../../../../src/taskset-run-contract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACnH,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAEzF,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,mFAAmF;AACnF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;4BAGjC,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;4BAGpC,CAAC;AACH,eAAO,MAAM,sBAAsB;;;;;kBAKxB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AACvH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,mBAAmB,CAKnF;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAcnG;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAclH"}
+{"version":3,"file":"taskset-run-contract.d.ts","sourceRoot":"","sources":["../../../../src/taskset-run-contract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAqE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AACnH,OAAO,EAA6B,KAAK,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAoE,KAAK,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AAC3I,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAE1E,mFAAmF;AACnF,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;4BAGjC,CAAC;AACH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;4BAGpC,CAAC;AACH,eAAO,MAAM,sBAAsB;;;;;kBAKxB,CAAC;AAEZ,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAuB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAiF,CAAC;AACvH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,cAAc,GAAG,mBAAmB,CAKnF;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,OAAO,EAAE,cAAc,GAAG,IAAI,CAcnG;AAED;+EAC+E;AAC/E,wBAAgB,mCAAmC,CACjD,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE,cAAc,EACvB,YAAY,EAAE,wBAAwB,GACrC,IAAI,CA4BN;AAED;iFACiF;AACjF,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,cAAc,EAAE,GAAG,cAAc,EAAE,CAclH"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openpond/evals",
-  "version": "0.11.2",
+  "version": "0.11.3",
   "description": "Portable Tasksets, graders, evaluation runs, receipts, conformance fixtures, and Work evidence for OpenPond Harnesses",
   "license": "MIT",
   "type": "module",