@openpond/evals 0.7.1 → 0.7.2

package/dist/learning/authoring-service.js

@@ -0,0 +1,49 @@
+import { assertBoundedTaskJson } from "../task-schema.js";
+import { AuthoringDraftSchema } from "./authoring.js";
+import { LearningDomainError } from "./errors.js";
+import { learningRef, sameLearningRef, sealLearningContent } from "./contracts.js";
+import { requireLearningRelease } from "./repository.js";
+export async function saveAuthoringDraft(tx, input, now) {
+    // A 100-record page remains below the common 16 MiB response limit.
+    assertBoundedTaskJson(input.draft, 131_072);
+    const previous = await tx.get("draft", input.draft.id);
+    if (previous && (previous.status !== "draft" || previous.targetId !== input.draft.targetId || previous.targetKind !== input.draft.targetKind || (previous.baseRelease === null ? input.draft.baseRelease !== null : input.draft.baseRelease === null || !sameLearningRef(previous.baseRelease, input.draft.baseRelease))))
+        throw new LearningDomainError("authoring_draft_identity_conflict", 409);
+    if (input.draft.baseRelease) {
+        if (input.draft.baseRelease.id !== input.draft.targetId)
+            throw new LearningDomainError("authoring_draft_base_mismatch", 422);
+        await requireLearningRelease(tx, input.draft.targetKind, input.draft.baseRelease);
+    }
+    const draft = AuthoringDraftSchema.parse(sealLearningContent({ ...input.draft, schemaVersion: "openpond.authoringDraft.v1", revision: input.expectedRevision + 1, status: "draft", publishedRelease: null, createdAt: previous?.createdAt ?? now, updatedAt: now }));
+    await tx.put("draft", draft, input.expectedRevision, { parentId: draft.targetKind, status: draft.status });
+    return { kind: "draft", id: draft.id, revision: draft.revision };
+}
+export async function archiveAuthoringDraft(tx, ref, now) {
+    const draft = await currentDraft(tx, ref);
+    const { contentHash: _hash, ...content } = draft;
+    const archived = AuthoringDraftSchema.parse(sealLearningContent({ ...content, revision: draft.revision + 1, status: "archived", updatedAt: now }));
+    await tx.put("draft", archived, draft.revision, { parentId: draft.targetKind, status: archived.status });
+    return { kind: "draft", id: archived.id, revision: archived.revision };
+}
+export async function finalizeAuthoringDraft(tx, input, published, now) {
+    const finalization = input.finalizeDraft;
+    if (!finalization)
+        return null;
+    const draft = await currentDraft(tx, finalization.draft);
+    if (draft.targetKind !== finalization.targetKind || draft.targetId !== finalization.release.id || finalization.release.revision !== (draft.baseRelease?.revision ?? 0) + 1 || !published.some(value => value.kind === finalization.targetKind && value.id === finalization.release.id && value.revision === finalization.release.revision))
+        throw new LearningDomainError("authoring_draft_publication_mismatch", 422);
+    await requireLearningRelease(tx, finalization.targetKind, finalization.release);
+    const { contentHash: _hash, ...content } = draft;
+    const completed = AuthoringDraftSchema.parse(sealLearningContent({ ...content, revision: draft.revision + 1, status: "published", publishedRelease: finalization.release, updatedAt: now }));
+    await tx.put("draft", completed, draft.revision, { parentId: draft.targetKind, status: completed.status });
+    return { kind: "draft", id: completed.id, revision: completed.revision };
+}
+async function currentDraft(tx, ref) {
+    const draft = await requireLearningRelease(tx, "draft", ref);
+    const current = await tx.get("draft", ref.id);
+    if (!current || !sameLearningRef(learningRef(current), ref))
+        throw new LearningDomainError("authoring_draft_revision_stale", 409);
+    if (draft.status !== "draft")
+        throw new LearningDomainError("authoring_draft_closed", 409);
+    return draft;
+}

package/dist/learning/authoring.js

@@ -0,0 +1,37 @@
+import { z } from "zod";
+import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
+import { RewardBindingSourceSchema, RewardReleaseRefSchema } from "../rewards.js";
+import { LearningRevisionRefSchema } from "./contracts.js";
+/** Strings intentionally preserve incomplete JSON/code until explicit publication. */
+export const RewardAuthoringFieldsSchema = z.object({
+    name: z.string().max(500), description: z.string().max(10_000),
+    kind: z.enum(["custom_verifier", "state", "content", "schema", "artifact", "runtime_event", "model_judge", "learned_model", "human"]),
+    fields: z.string(), outputField: z.string(), expectedField: z.string(), expectedValue: z.string(),
+    schema: z.string(), reference: z.string(), events: z.string(), code: z.string(), exportName: z.string(), timeout: z.string(),
+    rubric: z.string(), providerId: z.string(), modelId: z.string(), modelRevision: z.string(), temperature: z.string(),
+    reviewerRole: z.string(), learnedId: z.string(), learnedHash: z.string(), inputContract: z.string(), minimum: z.string(), maximum: z.string(),
+}).strict();
+export const TaskFormatAuthoringFieldsSchema = z.object({
+    name: z.string().max(500), description: z.string().max(10_000), instructions: z.string().max(20_000),
+    input: z.string(), output: z.string(), familyNamespace: z.string(),
+    sources: z.array(RewardBindingSourceSchema).max(100), recipeRef: RewardReleaseRefSchema.optional(),
+}).strict();
+export const CombinedRewardAuthoringFieldsSchema = z.object({
+    name: z.string().max(500), description: z.string().max(10_000), sources: z.array(RewardBindingSourceSchema).max(100),
+}).strict();
+export const AuthoringTargetKindSchema = z.enum(["definition", "reward", "binding"]);
+const base = z.object({ id: ReleaseIdSchema, targetId: ReleaseIdSchema, baseRelease: LearningRevisionRefSchema.nullable(), editorVersion: z.literal("openpond.modelsEditor.v1") });
+export const AuthoringDraftInputSchema = z.discriminatedUnion("targetKind", [
+    base.extend({ targetKind: z.literal("definition"), fields: TaskFormatAuthoringFieldsSchema }).strict(),
+    base.extend({ targetKind: z.literal("reward"), fields: RewardAuthoringFieldsSchema }).strict(),
+    base.extend({ targetKind: z.literal("binding"), fields: CombinedRewardAuthoringFieldsSchema }).strict(),
+]);
+const history = {
+    schemaVersion: z.literal("openpond.authoringDraft.v1"), revision: z.number().int().positive(),
+    status: z.enum(["draft", "archived", "published"]), publishedRelease: LearningRevisionRefSchema.nullable(),
+    createdAt: ReleaseTimestampSchema, updatedAt: ReleaseTimestampSchema,
+};
+export const AuthoringDraftContentSchema = z.discriminatedUnion("targetKind", [AuthoringDraftInputSchema.options[0].extend(history).strict(), AuthoringDraftInputSchema.options[1].extend(history).strict(), AuthoringDraftInputSchema.options[2].extend(history).strict()]);
+export const AuthoringDraftSchema = z.discriminatedUnion("targetKind", [AuthoringDraftContentSchema.options[0].extend({ contentHash: ReleaseHashSchema }).strict(), AuthoringDraftContentSchema.options[1].extend({ contentHash: ReleaseHashSchema }).strict(), AuthoringDraftContentSchema.options[2].extend({ contentHash: ReleaseHashSchema }).strict()]);
+/** Exact draft and resulting release are finalized in the publication transaction. */
+export const AuthoringDraftFinalizationSchema = z.object({ draft: LearningRevisionRefSchema, targetKind: AuthoringTargetKindSchema, release: LearningRevisionRefSchema }).strict();

package/dist/learning/index.js

@@ -7,3 +7,4 @@ export * from "./service.js";
 export * from "./grade-worker.js";
 export * from "./transport.js";
 export * from "./errors.js";
+export * from "./authoring.js";

package/dist/learning/operations.js

@@ -2,9 +2,10 @@ import { z } from "zod";
 import { ReleaseIdSchema } from "@openpond/harness";
 import { LearningJsonObjectSchema, LearningRevisionRefSchema, LearningSourceContentSchema, TaskDefinitionContentSchema, TaskExampleSubmissionSchema, TaskFeedbackSubmissionSchema, LearningPolicyContentSchema } from "./contracts.js";
 import { RewardBindingContentSchema, RewardReleaseContentSchema } from "../rewards.js";
+import { AuthoringDraftInputSchema, AuthoringDraftFinalizationSchema } from "./authoring.js";
 import { LearningTextAssetContentSchema } from "./assets.js";
 const command = z.object({ operationId: ReleaseIdSchema });
-const publish = command.extend({ action: z.literal("publish"), expectedRevision: z.number().int().nonnegative() });
+const publish = command.extend({ action: z.literal("publish"), expectedRevision: z.number().int().nonnegative(), finalizeDraft: AuthoringDraftFinalizationSchema.optional() });
 export const PublishLearningResourceCommandSchema = z.discriminatedUnion("kind", [
     publish.extend({ kind: z.literal("asset"), content: LearningTextAssetContentSchema }).strict(),
     publish.extend({ kind: z.literal("definition"), content: TaskDefinitionContentSchema }).strict(),
@@ -15,7 +16,8 @@ export const PublishLearningResourceCommandSchema = z.discriminatedUnion("kind",
 ]);
 export const PublishLearningResourcesCommandSchema = command.extend({
     action: z.literal("publish_resources"),
-    resources: z.array(z.union(PublishLearningResourceCommandSchema.options.map((schema) => schema.omit({ operationId: true, action: true })))).min(1).max(20),
+    finalizeDraft: AuthoringDraftFinalizationSchema.optional(),
+    resources: z.array(z.union(PublishLearningResourceCommandSchema.options.map((schema) => schema.omit({ operationId: true, action: true, finalizeDraft: true })))).min(1).max(20),
 }).strict();
 export const SubmitTaskExampleCommandSchema = command.extend({ action: z.literal("submit_example"), example: TaskExampleSubmissionSchema }).strict();
 export const SubmitTaskFeedbackCommandSchema = command.extend({ action: z.literal("submit_feedback"), feedback: TaskFeedbackSubmissionSchema }).strict();
@@ -30,4 +32,6 @@ export const ReviewTaskEvidenceCommandSchema = command.extend({
 }).strict();
 export const SealTaskBatchCommandSchema = command.extend({ action: z.literal("seal_batch"), batchId: ReleaseIdSchema, taskDefinition: LearningRevisionRefSchema, purpose: z.enum(["supervised_training", "reward_training", "evaluation"]), evidence: z.array(LearningRevisionRefSchema).min(1).max(10_000), decisions: z.array(LearningRevisionRefSchema).min(1).max(10_000) }).strict();
 export const CancelTaskGradeCommandSchema = command.extend({ action: z.literal("cancel_grade"), gradeId: ReleaseIdSchema, expectedRevision: z.number().int().positive() }).strict();
-export const LearningCommandSchema = z.union([PublishLearningResourceCommandSchema, PublishLearningResourcesCommandSchema, SubmitTaskExampleCommandSchema, SubmitTaskFeedbackCommandSchema, ApplyTaskCorrectionCommandSchema, ResolveTaskFeedbackCommandSchema, QueueTaskGradeCommandSchema, ReviewTaskEvidenceCommandSchema, SealTaskBatchCommandSchema, CancelTaskGradeCommandSchema]);
+export const SaveAuthoringDraftCommandSchema = command.extend({ action: z.literal("save_draft"), expectedRevision: z.number().int().nonnegative(), draft: AuthoringDraftInputSchema }).strict();
+export const ArchiveAuthoringDraftCommandSchema = command.extend({ action: z.literal("archive_draft"), draft: LearningRevisionRefSchema }).strict();
+export const LearningCommandSchema = z.union([SaveAuthoringDraftCommandSchema, ArchiveAuthoringDraftCommandSchema, PublishLearningResourceCommandSchema, PublishLearningResourcesCommandSchema, SubmitTaskExampleCommandSchema, SubmitTaskFeedbackCommandSchema, ApplyTaskCorrectionCommandSchema, ResolveTaskFeedbackCommandSchema, QueueTaskGradeCommandSchema, ReviewTaskEvidenceCommandSchema, SealTaskBatchCommandSchema, CancelTaskGradeCommandSchema]);

package/dist/learning/repository.js

@@ -4,8 +4,10 @@ import { assertLearningContentHash, LearningSourceSchema, TaskAdmissionDecisionS
 import { RewardBindingSchema, RewardReleaseSchema } from "../rewards.js";
 import { TasksetReleaseSchema } from "../tasksets.js";
 import { LearningDomainError } from "./errors.js";
+import { AuthoringDraftSchema } from "./authoring.js";
 import { LearningTextAssetSchema } from "./assets.js";
 export const learningResourceSchemas = {
+    draft: AuthoringDraftSchema,
     asset: LearningTextAssetSchema,
     definition: TaskDefinitionSchema, reward: RewardReleaseSchema, binding: RewardBindingSchema,
     source: LearningSourceSchema, evidence: TaskEvidenceSchema, feedback: TaskFeedbackSchema,

package/dist/learning/service.js

@@ -1,3 +1,4 @@
+import { saveAuthoringDraft, archiveAuthoringDraft, finalizeAuthoringDraft } from "./authoring-service.js";
 import { LearningDomainError } from "./errors.js";
 import { contentHash } from "@openpond/harness";
 import { createRewardBinding, createRewardRelease, resolveBoundRewards } from "../rewards.js";
@@ -27,6 +28,12 @@ export function createLearningService(repository, options = {}) {
             }
             let pointers;
             switch (input.action) {
+                case "save_draft":
+                    pointers = [await saveAuthoringDraft(transaction, input, now())];
+                    break;
+                case "archive_draft":
+                    pointers = [await archiveAuthoringDraft(transaction, input.draft, now())];
+                    break;
                 case "publish":
                     pointers = [await publish(transaction, input)];
                     break;
@@ -61,6 +68,11 @@ export function createLearningService(repository, options = {}) {
                     pointers = await seal(transaction, input, context.actor.id);
                     break;
             }
+            if (input.action === "publish" || input.action === "publish_resources") {
+                const finalized = await finalizeAuthoringDraft(transaction, input, pointers, now());
+                if (finalized)
+                    pointers.push(finalized);
+            }
             await transaction.saveOperation(operationId, { requestHash, resources: pointers });
             return { operationId: input.operationId, resources: await Promise.all(pointers.map((pointer) => requireLearningResource(transaction, pointer.kind, pointer.id, pointer.revision))) };
         });

package/dist/types/learning/authoring-service.d.ts

@@ -0,0 +1,25 @@
+import { type AuthoringDraftInput } from "./authoring.js";
+import { type LearningRevisionRef } from "./contracts.js";
+import { type LearningResourcePointer, type LearningTransaction } from "./repository.js";
+import type { LearningCommand } from "./operations.js";
+export declare function saveAuthoringDraft(tx: LearningTransaction, input: {
+    draft: AuthoringDraftInput;
+    expectedRevision: number;
+}, now: string): Promise<{
+    kind: "draft";
+    id: string;
+    revision: number;
+}>;
+export declare function archiveAuthoringDraft(tx: LearningTransaction, ref: LearningRevisionRef, now: string): Promise<{
+    kind: "draft";
+    id: string;
+    revision: number;
+}>;
+export declare function finalizeAuthoringDraft(tx: LearningTransaction, input: Extract<LearningCommand, {
+    action: "publish" | "publish_resources";
+}>, published: LearningResourcePointer[], now: string): Promise<{
+    kind: "draft";
+    id: string;
+    revision: number;
+} | null>;
+//# sourceMappingURL=authoring-service.d.ts.map

package/dist/types/learning/authoring-service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"authoring-service.d.ts","sourceRoot":"","sources":["../../../../../src/learning/authoring-service.ts"],"names":[],"mappings":"AACA,OAAO,EAAwB,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAEhF,OAAO,EAAqD,KAAK,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAC7G,OAAO,EAA0B,KAAK,uBAAuB,EAAE,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACjH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAEvD,wBAAsB,kBAAkB,CAAC,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE;IAAE,KAAK,EAAE,mBAAmB,CAAC;IAAC,gBAAgB,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,EAAE,MAAM;;;;GAY7I;AAED,wBAAsB,qBAAqB,CAAC,EAAE,EAAE,mBAAmB,EAAE,GAAG,EAAE,mBAAmB,EAAE,GAAG,EAAE,MAAM;;;;GAMzG;AAED,wBAAsB,sBAAsB,CAAC,EAAE,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,CAAC,eAAe,EAAE;IAAE,MAAM,EAAE,SAAS,GAAG,mBAAmB,CAAA;CAAE,CAAC,EAAE,SAAS,EAAE,uBAAuB,EAAE,EAAE,GAAG,EAAE,MAAM;;;;UAUpM"}