@openpond/evals 0.12.3 → 0.12.5

package/dist/learning/index.js

@@ -25,6 +25,7 @@ export * from "./reward-fixture-executor.js";
 export * from "./iteration-reservation-contracts.js";
 export * from "./iteration-dispatch-contracts.js";
 export * from "./iteration-dispatch-worker.js";
+export * from "./iteration-dispatch-deferral.js";
 export * from "./iteration-candidate-decision.js";
 export { resolveLearningTrainingParent } from "./iteration-training-parent.js";
 export * from "./schedule-contracts.js";

package/dist/learning/iteration-dispatch-commands.js

@@ -22,7 +22,7 @@ export async function commandLearningIterationDispatch(tx, input, now) {
     const resumed = LearningIterationDispatchSchema.parse({ ...dispatch, revision: dispatch.revision + 1,
         state: dispatch.execution ? "submitted" : dispatch.submission ? "prepared" : "preparing",
         cancelRequestedAt: cancelling ? dispatch.cancelRequestedAt ?? now : null,
-        consecutiveFailures: 0, nextAttemptAt: null, lastError: null, updatedAt: now });
+        consecutiveFailures: 0, nextAttemptAt: null, lastError: null, admission: null, updatedAt: now });
     // Preserve an in-flight lease. Its owner observes cancellation on the next
     // transition, and a replacement worker may claim only after that lease ends.
     await tx.put("dispatch", resumed, dispatch.revision, { parentId: iteration.id, status: resumed.state });

package/dist/learning/iteration-dispatch-contracts.js

@@ -8,6 +8,24 @@ export const LearningIterationSubmissionSchema = z.object({
     protocol: ReleaseIdSchema,
     payload: LearningJsonObjectSchema,
 }).strict();
+/** Normal capacity/calendar waits are not failed execution attempts. */
+export const LearningIterationAdmissionSchema = z.discriminatedUnion("status", [
+    z.object({ status: z.literal("ready") }).strict(),
+    z.object({ status: z.literal("deferred"), retryAt: ReleaseTimestampSchema,
+        reason: z.string().trim().min(1).max(2_000) }).strict(),
+]);
+/** The execution owner has durably rejected/fenced this exact attempt before
+ * creating an execution or spending. A timeout/absence read is not this proof. */
+export const LearningIterationSubmissionDeferralSchema = z.object({
+    scope: ReleaseIdSchema,
+    dispatchId: ReleaseIdSchema,
+    submissionHash: z.string().regex(/^[a-f0-9]{64}$/),
+    leaseGeneration: z.number().int().positive(),
+    status: z.literal("deferred"),
+    retryAt: ReleaseTimestampSchema,
+    reason: z.string().trim().min(1).max(2_000),
+    receipt: ImmutableReleaseRefSchema,
+}).strict();
 export const LearningIterationDispatchSchema = z.object({
     schemaVersion: z.literal("openpond.learningIterationDispatch.v1"),
     id: ReleaseIdSchema,
@@ -28,6 +46,8 @@ export const LearningIterationDispatchSchema = z.object({
     leaseExpiresAt: ReleaseTimestampSchema.nullable(),
     consecutiveFailures: z.number().int().nonnegative(),
     nextAttemptAt: ReleaseTimestampSchema.nullable(),
+    admission: LearningIterationAdmissionSchema.nullable().optional(),
+    lastSubmissionDeferral: LearningIterationSubmissionDeferralSchema.optional(),
     lastError: z.string().max(20_000).nullable(),
     createdAt: ReleaseTimestampSchema,
     updatedAt: ReleaseTimestampSchema,
@@ -37,6 +57,13 @@ export const LearningIterationDispatchSchema = z.object({
     }
     if (value.submissionStartedAt && !value.submission)
         context.addIssue({ code: "custom", path: ["submission"], message: "A started dispatch requires its persisted submission." });
+    if (value.admission?.status === "deferred" && (value.submissionStartedAt || value.state !== "prepared" || value.nextAttemptAt !== value.admission.retryAt)) {
+        context.addIssue({ code: "custom", path: ["admission"], message: "Admission deferral requires an unstarted prepared dispatch and its retry time." });
+    }
+    if (value.lastSubmissionDeferral && (value.lastSubmissionDeferral.dispatchId !== value.id ||
+        value.lastSubmissionDeferral.submissionHash !== value.submissionHash || value.lastSubmissionDeferral.leaseGeneration >= value.leaseGeneration)) {
+        context.addIssue({ code: "custom", path: ["lastSubmissionDeferral"], message: "A retained deferral must identify an earlier fenced attempt of this submission." });
+    }
     if (value.execution && !value.submissionStartedAt)
         context.addIssue({ code: "custom", path: ["execution"], message: "An execution requires a started dispatch." });
     if (["prepared", "submitted"].includes(value.state) && !value.submission)

package/dist/learning/iteration-dispatch-deferral.js

@@ -0,0 +1,28 @@
+import { LearningIterationSchema } from "./contracts.js";
+import { LearningDomainError } from "./errors.js";
+import { LearningIterationDispatchSchema, LearningIterationSubmissionDeferralSchema } from "./iteration-dispatch-contracts.js";
+/** Call under the dispatch's scope lock. The execution owner must first fence
+ * this attempt and prove no execution/charge exists. When the owner shares this
+ * repository, commit this mutation in its native admission transaction instead:
+ * late submits then fail the same generation fence as lease replacement.
+ */
+export async function deferLearningIterationSubmission(tx, scope, current, iteration, value, now) {
+    const outcome = LearningIterationSubmissionDeferralSchema.parse(value);
+    if (outcome.scope !== scope || outcome.dispatchId !== current.id || outcome.submissionHash !== current.submissionHash ||
+        outcome.leaseGeneration !== current.leaseGeneration || !current.submissionStartedAt || current.execution ||
+        current.state !== "prepared" || !current.leaseOwner ||
+        !current.leaseExpiresAt || Date.parse(current.leaseExpiresAt) <= Date.parse(now) ||
+        iteration.id !== current.iterationId || (iteration.status !== "dispatching" && !(current.cancelRequestedAt && iteration.status === "cancelling")))
+        throw new LearningDomainError("learning_submission_deferral_fenced", 409);
+    const retryAt = new Date(Math.max(Date.parse(outcome.retryAt), Date.parse(now) + 1_000)).toISOString();
+    const dispatch = LearningIterationDispatchSchema.parse({ ...current, revision: current.revision + 1,
+        submissionStartedAt: null, leaseGeneration: current.leaseGeneration + 1, leaseOwner: null, leaseExpiresAt: null,
+        admission: current.cancelRequestedAt ? null : { status: "deferred", reason: outcome.reason, retryAt },
+        nextAttemptAt: current.cancelRequestedAt ? null : retryAt, lastSubmissionDeferral: outcome,
+        consecutiveFailures: 0, lastError: null, updatedAt: now });
+    await tx.put("dispatch", dispatch, current.revision, { parentId: current.iterationId, status: dispatch.state });
+    if (iteration.failure)
+        await tx.put("iteration", LearningIterationSchema.parse({ ...iteration,
+            revision: iteration.revision + 1, failure: null, updatedAt: now }), iteration.revision, { parentId: current.chainId, status: iteration.status });
+    return dispatch;
+}

package/dist/learning/iteration-dispatch-worker.js

@@ -2,8 +2,10 @@ import { contentHash } from "@openpond/harness";
 import { LearningIterationSchema } from "./contracts.js";
 import { LearningDomainError } from "./errors.js";
 import { LearningChainSchema, LearningIterationReservationSchema } from "./iteration-reservation-contracts.js";
-import { LearningIterationDispatchSchema, LearningIterationObservationSchema, LearningIterationSubmissionSchema, } from "./iteration-dispatch-contracts.js";
+import { LearningIterationAdmissionSchema, LearningIterationDispatchSchema, LearningIterationObservationSchema, LearningIterationSubmissionSchema, } from "./iteration-dispatch-contracts.js";
 import { requireLearningRelease, requireLearningResource } from "./repository.js";
+import { deferLearningIterationSubmission } from "./iteration-dispatch-deferral.js";
+import { LearningIterationSubmissionDeferralSchema } from "./iteration-dispatch-contracts.js";
 const closed = new Set(["accepted", "rejected", "failed", "cancelled", "completed_without_candidate"]);
 export function createLearningIterationWorker(repository, executor, options) {
     const now = options.now ?? (() => new Date().toISOString());
@@ -20,7 +22,7 @@ export function createLearningIterationWorker(repository, executor, options) {
             const stamp = now();
             if (current?.state === "settled" || (current?.state === "blocked" && !current.cancelRequestedAt)
                 || (current?.leaseExpiresAt && Date.parse(current.leaseExpiresAt) > Date.parse(stamp))
-                || (current?.nextAttemptAt && Date.parse(current.nextAttemptAt) > Date.parse(stamp)))
+                || (!current?.cancelRequestedAt && current?.nextAttemptAt && Date.parse(current.nextAttemptAt) > Date.parse(stamp)))
                 return null;
             const policy = await requireLearningRelease(tx, "policy", iteration.policy);
             if (policy.executionOwner !== options.executionOwner)
@@ -80,7 +82,7 @@ export function createLearningIterationWorker(repository, executor, options) {
         });
         await tx.put("iteration", updated, iteration.revision, { parentId: current.chainId, status });
         await saveDispatch(tx, current, { state: terminal ? "settled" : "submitted", execution: outcome?.execution ?? current.execution,
-            terminalReceipt: outcome?.receipt ?? null, leaseOwner: null, leaseExpiresAt: null,
+            terminalReceipt: outcome?.receipt ?? null, leaseOwner: null, leaseExpiresAt: null, admission: null,
             nextAttemptAt: terminal ? null : new Date(Date.parse(now()) + 5_000).toISOString(), consecutiveFailures: 0, lastError: null });
         if (terminal) {
             const reservation = await requireLearningResource(tx, "reservation", iteration.id);
@@ -124,6 +126,25 @@ export function createLearningIterationWorker(repository, executor, options) {
                     return read();
                 dispatch = prepared;
             }
+            if (!dispatch.submissionStartedAt && !dispatch.cancelRequestedAt && executor.admit) {
+                const checkedAt = now();
+                const admission = LearningIterationAdmissionSchema.parse(await bounded(input => executor.admit(input)));
+                if (admission.status === "deferred") {
+                    if (Date.parse(admission.retryAt) <= Date.parse(checkedAt))
+                        throw new LearningDomainError("learning_admission_retry_time_invalid", 409);
+                    await fenced(scope, dispatch, async (tx, current, iteration) => {
+                        if (current.cancelRequestedAt)
+                            return settle(tx, current, iteration, null);
+                        const retryAt = new Date(Math.max(Date.parse(admission.retryAt), Date.parse(now()) + 1_000)).toISOString();
+                        await saveDispatch(tx, current, { admission: { ...admission, retryAt }, nextAttemptAt: retryAt,
+                            consecutiveFailures: 0, lastError: null, leaseOwner: null, leaseExpiresAt: null });
+                        if (iteration.failure)
+                            await tx.put("iteration", LearningIterationSchema.parse({ ...iteration,
+                                revision: iteration.revision + 1, failure: null, updatedAt: now() }), iteration.revision, { parentId: current.chainId, status: iteration.status });
+                    });
+                    return read();
+                }
+            }
             const starting = await fenced(scope, dispatch, async (tx, current, iteration) => {
                 if (current.cancelRequestedAt && !current.submissionStartedAt) {
                     await settle(tx, current, iteration, null);
@@ -134,7 +155,7 @@ export function createLearningIterationWorker(repository, executor, options) {
                     await saveDispatch(tx, current, { leaseOwner: null, leaseExpiresAt: null });
                     return null;
                 }
-                return saveDispatch(tx, current, { submissionStartedAt: current.submissionStartedAt ?? now() });
+                return saveDispatch(tx, current, { submissionStartedAt: current.submissionStartedAt ?? now(), admission: null });
             });
             if (!starting)
                 return read();
@@ -147,6 +168,14 @@ export function createLearningIterationWorker(repository, executor, options) {
                 dispatch = refreshed;
                 observed = dispatch.cancelRequestedAt ? await bounded(input => executor.cancel(input)) : await bounded(input => executor.submit(input));
             }
+            if (observed.status === "deferred") {
+                const outcome = LearningIterationSubmissionDeferralSchema.parse(observed);
+                if (outcome.scope !== scope || outcome.dispatchId !== dispatch.id || outcome.submissionHash !== dispatch.submissionHash ||
+                    outcome.leaseGeneration !== dispatch.leaseGeneration)
+                    throw new LearningDomainError("learning_execution_identity_mismatch", 409);
+                await fenced(scope, dispatch, (tx, current, iteration) => deferLearningIterationSubmission(tx, scope, current, iteration, outcome, now()));
+                return read();
+            }
             const outcome = LearningIterationObservationSchema.parse(observed);
             if (outcome.scope !== scope || outcome.dispatchId !== dispatch.id || outcome.submissionHash !== dispatch.submissionHash
                 || (dispatch.execution && contentHash(outcome.execution) !== contentHash(dispatch.execution)))
@@ -165,7 +194,7 @@ export function createLearningIterationWorker(repository, executor, options) {
                 const blocked = failures > acquired.policy.limits.maxRetries;
                 const message = (error instanceof Error ? error.message : "Learning dispatch failed.").slice(0, 20_000);
                 await saveDispatch(tx, current, { state: blocked ? "blocked" : current.state, consecutiveFailures: failures,
-                    lastError: message, leaseOwner: null, leaseExpiresAt: null,
+                    lastError: message, leaseOwner: null, leaseExpiresAt: null, admission: null,
                     nextAttemptAt: blocked ? null : new Date(Date.parse(now()) + Math.min(60_000, 1_000 * 2 ** Math.min(failures, 6))).toISOString() });
                 const updated = LearningIterationSchema.parse({ ...iteration, revision: iteration.revision + 1, retryCount: iteration.retryCount + 1,
                     failure: { code: blocked ? "learning_dispatch_attention_required" : "learning_dispatch_retry_pending", message }, updatedAt: now() });

package/dist/tool-learning/actions.js

@@ -0,0 +1,62 @@
+import { z } from "zod";
+import { TOOL_LEARNING_LIMITS, ToolLearningHashSchema, boundedToolLearningSchema, toolLearningText, } from "./primitives.js";
+const Text = toolLearningText(TOOL_LEARNING_LIMITS.actionBytes);
+const RelativePath = z
+    .string()
+    .min(1)
+    .max(1_000)
+    .refine((value) => !value.startsWith("/") &&
+    !value.includes("\\") &&
+    !/^[A-Za-z]:/u.test(value) &&
+    value.split("/").every((part) => part !== "" && part !== "." && part !== ".."), "Use a workspace-relative artifact path.");
+export const ToolLearningActionSchema = boundedToolLearningSchema(z.discriminatedUnion("kind", [
+    z.object({ kind: z.literal("python"), code: Text }).strict(),
+    z
+        .object({ kind: z.literal("shell"), command: Text, shell: z.string().min(1).max(120) })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("tool_arguments"),
+        arguments: z.record(z.string().min(1).max(200), z.json()),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("edit"),
+        patch: Text,
+        baseFiles: z
+            .array(z
+            .object({ path: RelativePath, contentHash: ToolLearningHashSchema.nullable() })
+            .strict())
+            .min(1)
+            .max(32),
+    })
+        .strict(),
+]), TOOL_LEARNING_LIMITS.actionBytes);
+export const ToolLearningRepairSchema = z.discriminatedUnion("kind", [
+    z
+        .object({
+        kind: z.literal("replacement"),
+        expectedActionHash: ToolLearningHashSchema,
+        environmentFingerprint: ToolLearningHashSchema,
+        action: ToolLearningActionSchema,
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("patch"),
+        expectedActionHash: ToolLearningHashSchema,
+        environmentFingerprint: ToolLearningHashSchema,
+        files: z
+            .array(z
+            .object({
+            path: RelativePath,
+            expectedContentHash: ToolLearningHashSchema,
+            patch: Text,
+        })
+            .strict())
+            .min(1)
+            .max(16),
+    })
+        .strict(),
+]);

package/dist/tool-learning/consultation.js

@@ -0,0 +1,198 @@
+import { z } from "zod";
+import { ToolLearningActionSchema } from "./actions.js";
+import { TOOL_LEARNING_LIMITS, ToolLearningEnvironmentSchema, ToolLearningHashSchema as Hash, ToolLearningIdSchema as Id, ToolLearningRefSchema as Ref, ToolLearningScopeSchema, ToolLearningTimestampSchema as Timestamp, boundedToolLearningSchema, toolLearningContentHash, toolLearningText, } from "./primitives.js";
+export const ToolLearningCapabilitySchema = z
+    .object({
+    id: Id,
+    application: z.string().min(1).max(200),
+    versions: z.array(z.string().min(1).max(200)).min(1).max(32),
+    tools: z.array(z.string().min(1).max(200)).min(1).max(32),
+    mode: z.literal("recovery"),
+    routingInstruction: toolLearningText(2000),
+    qualification: z.discriminatedUnion("kind", [
+        z.object({ kind: z.literal("integration"), receipt: Ref }).strict(),
+        z
+            .object({
+            kind: z.literal("learned"),
+            receipt: Ref,
+            trainingRun: Ref,
+            adapter: Ref,
+            evaluation: Ref,
+            activation: Ref,
+        })
+            .strict(),
+    ]),
+})
+    .strict();
+const Manifest = z
+    .object({
+    schemaVersion: z.literal("openpond.toolLearningManifest.v1"),
+    installationId: Id,
+    scope: ToolLearningScopeSchema,
+    revision: z.number().int().positive(),
+    contentHash: Hash,
+    issuedAt: Timestamp,
+    expiresAt: Timestamp,
+    modelRelease: Ref,
+    responseSchema: z.literal("openpond.toolLearningDecision.v1"),
+    capabilities: z.array(ToolLearningCapabilitySchema).max(TOOL_LEARNING_LIMITS.capabilities),
+})
+    .strict()
+    .superRefine((manifest, context) => {
+    const { contentHash, ...body } = manifest;
+    if (toolLearningContentHash(body) !== contentHash)
+        context.addIssue({
+            code: "custom",
+            path: ["contentHash"],
+            message: "Manifest content hash mismatch.",
+        });
+    const lifetime = Date.parse(manifest.expiresAt) - Date.parse(manifest.issuedAt);
+    if (lifetime <= 0 || lifetime > TOOL_LEARNING_LIMITS.manifestSeconds * 1000)
+        context.addIssue({
+            code: "custom",
+            path: ["expiresAt"],
+            message: "Manifest expiry exceeds the supported lifetime.",
+        });
+    if (new Set(manifest.capabilities.map((entry) => entry.id)).size !== manifest.capabilities.length)
+        context.addIssue({
+            code: "custom",
+            path: ["capabilities"],
+            message: "Capability IDs must be unique.",
+        });
+});
+export const ToolLearningManifestSchema = boundedToolLearningSchema(Manifest, TOOL_LEARNING_LIMITS.consultationBytes);
+export const ToolLearningConsultationRequestSchema = boundedToolLearningSchema(z
+    .object({
+    schemaVersion: z.literal("openpond.toolLearningConsultation.v1"),
+    operationId: Id,
+    installationId: Id,
+    scope: ToolLearningScopeSchema,
+    sessionId: Id,
+    callId: Id,
+    capabilityId: Id,
+    manifestHash: Hash,
+    modelRelease: Ref,
+    mode: z.literal("recovery"),
+    intent: toolLearningText(2000),
+    action: ToolLearningActionSchema,
+    actionHash: Hash,
+    environment: ToolLearningEnvironmentSchema,
+    contextHash: Hash,
+    failure: z
+        .object({
+        message: toolLearningText(TOOL_LEARNING_LIMITS.resultBytes),
+        observation: Ref,
+    })
+        .strict(),
+    round: z.number().int().min(1).max(TOOL_LEARNING_LIMITS.consultationRounds),
+})
+    .strict()
+    .superRefine((request, context) => {
+    if (toolLearningContentHash(request.action) !== request.actionHash)
+        context.addIssue({
+            code: "custom",
+            path: ["actionHash"],
+            message: "Consultation action hash mismatch.",
+        });
+    if (toolLearningContentHash({
+        intent: request.intent,
+        environment: request.environment,
+        failure: request.failure,
+    }) !== request.contextHash)
+        context.addIssue({
+            code: "custom",
+            path: ["contextHash"],
+            message: "Consultation context hash mismatch.",
+        });
+}), TOOL_LEARNING_LIMITS.consultationBytes);
+/** Only model-authored task content; transport failures never become proceed. */
+export const ToolLearningDecisionSchema = z.discriminatedUnion("decision", [
+    z.object({ decision: z.literal("proceed") }).strict(),
+    z
+        .object({
+        decision: z.literal("revise"),
+        issues: z
+            .array(z
+            .object({
+            category: z.string().min(1).max(120),
+            feedback: toolLearningText(2000),
+        })
+            .strict())
+            .min(1)
+            .max(8),
+    })
+        .strict(),
+    z
+        .object({
+        decision: z.literal("needs_context"),
+        missing: z
+            .array(z
+            .object({
+            field: z.string().min(1).max(120),
+            reason: toolLearningText(500),
+        })
+            .strict())
+            .min(1)
+            .max(8),
+    })
+        .strict(),
+]);
+const ResponseIdentity = {
+    operationId: Id,
+    installationId: Id,
+    capabilityId: Id,
+    manifestHash: Hash,
+    modelRelease: Ref,
+    actionHash: Hash,
+    contextHash: Hash,
+};
+export const ToolLearningConsultationResponseSchema = boundedToolLearningSchema(z.discriminatedUnion("state", [
+    z
+        .object({
+        schemaVersion: z.literal("openpond.toolLearningDecision.v1"),
+        ...ResponseIdentity,
+        state: z.literal("completed"),
+        decision: ToolLearningDecisionSchema,
+        usageReceipt: Ref,
+        completedAt: Timestamp,
+    })
+        .strict(),
+    z
+        .object({
+        schemaVersion: z.literal("openpond.toolLearningDecision.v1"),
+        ...ResponseIdentity,
+        state: z.literal("unavailable"),
+        reason: z.enum([
+            "disabled",
+            "revoked",
+            "expired_manifest",
+            "unsupported_scope",
+            "budget_exhausted",
+            "timeout",
+            "invalid_output",
+            "cancelled",
+            "dispatch_uncertain",
+        ]),
+        usageReceipt: Ref.nullable(),
+    })
+        .strict(),
+]), TOOL_LEARNING_LIMITS.consultationBytes);
+/** Invoke before exposing a result to the agent; even valid JSON may be stale. */
+export function assertToolLearningConsultationResponse(requestValue, responseValue) {
+    const request = ToolLearningConsultationRequestSchema.parse(requestValue);
+    const response = ToolLearningConsultationResponseSchema.parse(responseValue);
+    for (const key of [
+        "operationId",
+        "installationId",
+        "capabilityId",
+        "manifestHash",
+        "actionHash",
+        "contextHash",
+    ])
+        if (request[key] !== response[key])
+            throw new Error(`tool_learning_consultation_${key}_mismatch`);
+    if (request.modelRelease.id !== response.modelRelease.id ||
+        request.modelRelease.contentHash !== response.modelRelease.contentHash)
+        throw new Error("tool_learning_consultation_model_mismatch");
+    return response;
+}