@openpond/evals 0.12.3 → 0.12.4

package/LEARNING.md

@@ -394,3 +394,25 @@ revision changes and budget reservations, resolve authoritative execution/gradin
 receipts, isolate holdouts from training, bind the exact candidate at runtime,
 verify cleanup, and enforce the verdict inside the candidate-decision transaction.
 No customer serving is activated by an acceptance group.
+
+### Bounded tool episodes
+
+The explicit `@openpond/evals/tool-learning` entry describes bounded attempt,
+observation, recovery-link, verification, installation, and recovery-consultation
+contracts. An event is sealed over validated sanitized bytes; the observation
+references the exact attempted action. Execution failure, validation failure,
+incomplete observation, uncertain effects, and verified outcome stay distinct.
+A producer-provided verifier reference is a claim that intake must resolve against
+its authentic execution owner.
+
+Collection acknowledgements explicitly have `learningEnrollment: false`. The
+capture policy requires separate review for model updates and disables synthetic
+variants. Retaining a successful command is not evidence of the requested outcome,
+a qualified training target, or permission to spend. Reuse the existing task
+review, immutable supervised batch, and publication owners for actual admission.
+
+Recovery consultation binds its action/context, active manifest, selected model,
+and usage receipt. Integration qualification advertises an existing-model route;
+learned qualification additionally requires retained training/evaluation/activation
+lineage. Neither installation nor receipt of evidence establishes learned efficacy.
+These protocols do not install client hooks or start background services.

package/dist/tool-learning/actions.js

@@ -0,0 +1,62 @@
+import { z } from "zod";
+import { TOOL_LEARNING_LIMITS, ToolLearningHashSchema, boundedToolLearningSchema, toolLearningText, } from "./primitives.js";
+const Text = toolLearningText(TOOL_LEARNING_LIMITS.actionBytes);
+const RelativePath = z
+    .string()
+    .min(1)
+    .max(1_000)
+    .refine((value) => !value.startsWith("/") &&
+    !value.includes("\\") &&
+    !/^[A-Za-z]:/u.test(value) &&
+    value.split("/").every((part) => part !== "" && part !== "." && part !== ".."), "Use a workspace-relative artifact path.");
+export const ToolLearningActionSchema = boundedToolLearningSchema(z.discriminatedUnion("kind", [
+    z.object({ kind: z.literal("python"), code: Text }).strict(),
+    z
+        .object({ kind: z.literal("shell"), command: Text, shell: z.string().min(1).max(120) })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("tool_arguments"),
+        arguments: z.record(z.string().min(1).max(200), z.json()),
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("edit"),
+        patch: Text,
+        baseFiles: z
+            .array(z
+            .object({ path: RelativePath, contentHash: ToolLearningHashSchema.nullable() })
+            .strict())
+            .min(1)
+            .max(32),
+    })
+        .strict(),
+]), TOOL_LEARNING_LIMITS.actionBytes);
+export const ToolLearningRepairSchema = z.discriminatedUnion("kind", [
+    z
+        .object({
+        kind: z.literal("replacement"),
+        expectedActionHash: ToolLearningHashSchema,
+        environmentFingerprint: ToolLearningHashSchema,
+        action: ToolLearningActionSchema,
+    })
+        .strict(),
+    z
+        .object({
+        kind: z.literal("patch"),
+        expectedActionHash: ToolLearningHashSchema,
+        environmentFingerprint: ToolLearningHashSchema,
+        files: z
+            .array(z
+            .object({
+            path: RelativePath,
+            expectedContentHash: ToolLearningHashSchema,
+            patch: Text,
+        })
+            .strict())
+            .min(1)
+            .max(16),
+    })
+        .strict(),
+]);

package/dist/tool-learning/consultation.js

@@ -0,0 +1,198 @@
+import { z } from "zod";
+import { ToolLearningActionSchema } from "./actions.js";
+import { TOOL_LEARNING_LIMITS, ToolLearningEnvironmentSchema, ToolLearningHashSchema as Hash, ToolLearningIdSchema as Id, ToolLearningRefSchema as Ref, ToolLearningScopeSchema, ToolLearningTimestampSchema as Timestamp, boundedToolLearningSchema, toolLearningContentHash, toolLearningText, } from "./primitives.js";
+export const ToolLearningCapabilitySchema = z
+    .object({
+    id: Id,
+    application: z.string().min(1).max(200),
+    versions: z.array(z.string().min(1).max(200)).min(1).max(32),
+    tools: z.array(z.string().min(1).max(200)).min(1).max(32),
+    mode: z.literal("recovery"),
+    routingInstruction: toolLearningText(2000),
+    qualification: z.discriminatedUnion("kind", [
+        z.object({ kind: z.literal("integration"), receipt: Ref }).strict(),
+        z
+            .object({
+            kind: z.literal("learned"),
+            receipt: Ref,
+            trainingRun: Ref,
+            adapter: Ref,
+            evaluation: Ref,
+            activation: Ref,
+        })
+            .strict(),
+    ]),
+})
+    .strict();
+const Manifest = z
+    .object({
+    schemaVersion: z.literal("openpond.toolLearningManifest.v1"),
+    installationId: Id,
+    scope: ToolLearningScopeSchema,
+    revision: z.number().int().positive(),
+    contentHash: Hash,
+    issuedAt: Timestamp,
+    expiresAt: Timestamp,
+    modelRelease: Ref,
+    responseSchema: z.literal("openpond.toolLearningDecision.v1"),
+    capabilities: z.array(ToolLearningCapabilitySchema).max(TOOL_LEARNING_LIMITS.capabilities),
+})
+    .strict()
+    .superRefine((manifest, context) => {
+    const { contentHash, ...body } = manifest;
+    if (toolLearningContentHash(body) !== contentHash)
+        context.addIssue({
+            code: "custom",
+            path: ["contentHash"],
+            message: "Manifest content hash mismatch.",
+        });
+    const lifetime = Date.parse(manifest.expiresAt) - Date.parse(manifest.issuedAt);
+    if (lifetime <= 0 || lifetime > TOOL_LEARNING_LIMITS.manifestSeconds * 1000)
+        context.addIssue({
+            code: "custom",
+            path: ["expiresAt"],
+            message: "Manifest expiry exceeds the supported lifetime.",
+        });
+    if (new Set(manifest.capabilities.map((entry) => entry.id)).size !== manifest.capabilities.length)
+        context.addIssue({
+            code: "custom",
+            path: ["capabilities"],
+            message: "Capability IDs must be unique.",
+        });
+});
+export const ToolLearningManifestSchema = boundedToolLearningSchema(Manifest, TOOL_LEARNING_LIMITS.consultationBytes);
+export const ToolLearningConsultationRequestSchema = boundedToolLearningSchema(z
+    .object({
+    schemaVersion: z.literal("openpond.toolLearningConsultation.v1"),
+    operationId: Id,
+    installationId: Id,
+    scope: ToolLearningScopeSchema,
+    sessionId: Id,
+    callId: Id,
+    capabilityId: Id,
+    manifestHash: Hash,
+    modelRelease: Ref,
+    mode: z.literal("recovery"),
+    intent: toolLearningText(2000),
+    action: ToolLearningActionSchema,
+    actionHash: Hash,
+    environment: ToolLearningEnvironmentSchema,
+    contextHash: Hash,
+    failure: z
+        .object({
+        message: toolLearningText(TOOL_LEARNING_LIMITS.resultBytes),
+        observation: Ref,
+    })
+        .strict(),
+    round: z.number().int().min(1).max(TOOL_LEARNING_LIMITS.consultationRounds),
+})
+    .strict()
+    .superRefine((request, context) => {
+    if (toolLearningContentHash(request.action) !== request.actionHash)
+        context.addIssue({
+            code: "custom",
+            path: ["actionHash"],
+            message: "Consultation action hash mismatch.",
+        });
+    if (toolLearningContentHash({
+        intent: request.intent,
+        environment: request.environment,
+        failure: request.failure,
+    }) !== request.contextHash)
+        context.addIssue({
+            code: "custom",
+            path: ["contextHash"],
+            message: "Consultation context hash mismatch.",
+        });
+}), TOOL_LEARNING_LIMITS.consultationBytes);
+/** Only model-authored task content; transport failures never become proceed. */
+export const ToolLearningDecisionSchema = z.discriminatedUnion("decision", [
+    z.object({ decision: z.literal("proceed") }).strict(),
+    z
+        .object({
+        decision: z.literal("revise"),
+        issues: z
+            .array(z
+            .object({
+            category: z.string().min(1).max(120),
+            feedback: toolLearningText(2000),
+        })
+            .strict())
+            .min(1)
+            .max(8),
+    })
+        .strict(),
+    z
+        .object({
+        decision: z.literal("needs_context"),
+        missing: z
+            .array(z
+            .object({
+            field: z.string().min(1).max(120),
+            reason: toolLearningText(500),
+        })
+            .strict())
+            .min(1)
+            .max(8),
+    })
+        .strict(),
+]);
+const ResponseIdentity = {
+    operationId: Id,
+    installationId: Id,
+    capabilityId: Id,
+    manifestHash: Hash,
+    modelRelease: Ref,
+    actionHash: Hash,
+    contextHash: Hash,
+};
+export const ToolLearningConsultationResponseSchema = boundedToolLearningSchema(z.discriminatedUnion("state", [
+    z
+        .object({
+        schemaVersion: z.literal("openpond.toolLearningDecision.v1"),
+        ...ResponseIdentity,
+        state: z.literal("completed"),
+        decision: ToolLearningDecisionSchema,
+        usageReceipt: Ref,
+        completedAt: Timestamp,
+    })
+        .strict(),
+    z
+        .object({
+        schemaVersion: z.literal("openpond.toolLearningDecision.v1"),
+        ...ResponseIdentity,
+        state: z.literal("unavailable"),
+        reason: z.enum([
+            "disabled",
+            "revoked",
+            "expired_manifest",
+            "unsupported_scope",
+            "budget_exhausted",
+            "timeout",
+            "invalid_output",
+            "cancelled",
+            "dispatch_uncertain",
+        ]),
+        usageReceipt: Ref.nullable(),
+    })
+        .strict(),
+]), TOOL_LEARNING_LIMITS.consultationBytes);
+/** Invoke before exposing a result to the agent; even valid JSON may be stale. */
+export function assertToolLearningConsultationResponse(requestValue, responseValue) {
+    const request = ToolLearningConsultationRequestSchema.parse(requestValue);
+    const response = ToolLearningConsultationResponseSchema.parse(responseValue);
+    for (const key of [
+        "operationId",
+        "installationId",
+        "capabilityId",
+        "manifestHash",
+        "actionHash",
+        "contextHash",
+    ])
+        if (request[key] !== response[key])
+            throw new Error(`tool_learning_consultation_${key}_mismatch`);
+    if (request.modelRelease.id !== response.modelRelease.id ||
+        request.modelRelease.contentHash !== response.modelRelease.contentHash)
+        throw new Error("tool_learning_consultation_model_mismatch");
+    return response;
+}

package/dist/tool-learning/episodes.js

@@ -0,0 +1,284 @@
+import { z } from "zod";
+import { ToolLearningActionSchema } from "./actions.js";
+import { TOOL_LEARNING_LIMITS, ToolLearningAssetSchema, ToolLearningEnvironmentSchema, ToolLearningHashSchema as Hash, ToolLearningIdSchema as Id, ToolLearningRefSchema as Ref, ToolLearningSanitizationSchema, ToolLearningTimestampSchema as Timestamp, boundedToolLearningSchema, toolLearningContentHash, toolLearningText, } from "./primitives.js";
+export const ToolEpisodeAttemptSchema = z
+    .object({
+    intent: toolLearningText(2_000),
+    tool: z
+        .object({
+        name: z.string().min(1).max(200),
+        version: z.string().max(200).nullable(),
+        server: z.string().max(200).nullable(),
+    })
+        .strict(),
+    action: ToolLearningActionSchema,
+    actionHash: Hash,
+    environment: ToolLearningEnvironmentSchema,
+    contextAssets: z.array(ToolLearningAssetSchema).max(TOOL_LEARNING_LIMITS.assetsPerEpisode),
+    selection: z.discriminatedUnion("kind", [
+        z.object({ kind: z.literal("eligible_attempt") }).strict(),
+        z
+            .object({
+            kind: z.literal("unchanged_control"),
+            numerator: z.number().int().positive(),
+            denominator: z.number().int().positive(),
+            policyHash: Hash,
+        })
+            .strict(),
+    ]),
+})
+    .strict()
+    .superRefine((attempt, context) => {
+    if (toolLearningContentHash(attempt.action) !== attempt.actionHash)
+        context.addIssue({
+            code: "custom",
+            path: ["actionHash"],
+            message: "Action bytes differ from their hash.",
+        });
+    if (attempt.selection.kind === "unchanged_control" &&
+        attempt.selection.numerator > attempt.selection.denominator)
+        context.addIssue({
+            code: "custom",
+            path: ["selection"],
+            message: "Sampling probability cannot exceed one.",
+        });
+});
+export const ToolEpisodeFailureKindSchema = z.enum([
+    "validation_failure",
+    "execution_failure",
+    "process_failure",
+    "timeout",
+    "cancelled",
+    "infrastructure_failure",
+    "authorization_denied",
+    "outcome_failure",
+    "unknown_failure",
+]);
+const ObservationBase = {
+    attempt: Ref,
+    actionHash: Hash,
+    execution: z.enum(["not_executed", "executed", "unknown"]),
+    sideEffects: z.enum(["none", "possible", "observed", "unknown"]),
+    resultExcerpt: toolLearningText(TOOL_LEARNING_LIMITS.resultBytes),
+    resultHash: Hash.nullable(),
+    assets: z.array(ToolLearningAssetSchema).max(TOOL_LEARNING_LIMITS.assetsPerEpisode),
+};
+export const ToolEpisodeObservationSchema = z
+    .discriminatedUnion("outcome", [
+    z
+        .object({
+        ...ObservationBase,
+        outcome: z.literal("succeeded"),
+        exitCode: z.literal(0).nullable(),
+    })
+        .strict(),
+    z
+        .object({
+        ...ObservationBase,
+        outcome: z.literal("failed"),
+        classification: ToolEpisodeFailureKindSchema,
+        errorCode: z.string().min(1).max(200),
+        message: toolLearningText(2_000),
+        exitCode: z.number().int().min(-255).max(255).nullable(),
+    })
+        .strict(),
+    z
+        .object({
+        ...ObservationBase,
+        outcome: z.literal("incomplete"),
+        reason: z.enum([
+            "completion_not_observed",
+            "observer_restarted",
+            "transport_interrupted",
+            "coverage_gap",
+        ]),
+    })
+        .strict(),
+])
+    .superRefine((observation, context) => {
+    if (observation.execution === "not_executed" && observation.sideEffects !== "none")
+        context.addIssue({
+            code: "custom",
+            path: ["sideEffects"],
+            message: "An unexecuted action cannot claim tool effects.",
+        });
+    if (observation.outcome === "succeeded" && observation.execution !== "executed")
+        context.addIssue({
+            code: "custom",
+            path: ["execution"],
+            message: "Successful execution requires an observed execution.",
+        });
+    if (observation.outcome !== "failed")
+        return;
+    if (observation.classification === "validation_failure" &&
+        observation.execution !== "not_executed")
+        context.addIssue({
+            code: "custom",
+            path: ["execution"],
+            message: "Pre-execution validation failure must be unexecuted.",
+        });
+    if (observation.classification === "process_failure" &&
+        (observation.execution !== "executed" ||
+            observation.exitCode === null ||
+            observation.exitCode === 0))
+        context.addIssue({
+            code: "custom",
+            path: ["exitCode"],
+            message: "Process failure requires a nonzero observed exit.",
+        });
+});
+export const ToolEpisodeRecoveryLinkSchema = z
+    .object({
+    failedAttempt: Ref,
+    failedObservation: Ref,
+    replacementAttempt: Ref,
+    goalId: Id,
+    basis: z.enum(["explicit_relation", "artifact_lineage", "unverified"]),
+    attributionEvidence: z.array(Ref).max(16),
+})
+    .strict()
+    .superRefine((link, context) => {
+    if (link.failedAttempt.id === link.replacementAttempt.id)
+        context.addIssue({
+            code: "custom",
+            message: "A recovery must reference a different attempt.",
+        });
+    if (link.basis !== "unverified" && link.attributionEvidence.length === 0)
+        context.addIssue({
+            code: "custom",
+            path: ["attributionEvidence"],
+            message: "Recovery attribution needs retained evidence.",
+        });
+});
+/** A producer supplies a claim/reference. Intake must resolve its authentic owner
+ * receipt; possession of an append credential never grants verifier authority. */
+export const ToolEpisodeVerificationSchema = z
+    .object({
+    attempt: Ref,
+    actionHash: Hash,
+    outcome: z.enum(["passed", "failed", "unknown"]),
+    verifierRelease: Ref.nullable(),
+    verifierReceipt: Ref.nullable(),
+    environmentFingerprint: Hash,
+    resetStateHash: Hash.nullable(),
+    outputArtifacts: z.array(ToolLearningAssetSchema).max(TOOL_LEARNING_LIMITS.assetsPerEpisode),
+    reason: toolLearningText(2_000),
+})
+    .strict()
+    .superRefine((verification, context) => {
+    if (verification.outcome !== "unknown" &&
+        (!verification.verifierRelease || !verification.verifierReceipt))
+        context.addIssue({
+            code: "custom",
+            message: "A verification outcome requires an exact verifier and execution receipt.",
+        });
+});
+const Event = z
+    .object({
+    schemaVersion: z.literal("openpond.toolEpisodeEvent.v1"),
+    operationId: Id,
+    installationId: Id,
+    episodeId: Id,
+    eventId: Id,
+    sessionId: Id,
+    parentSessionId: Id.nullable(),
+    turnId: Id.nullable(),
+    callId: Id,
+    occurredAt: Timestamp,
+    sequence: z.number().int().nonnegative(),
+    adapterVersion: z.string().min(1).max(200),
+    sanitization: ToolLearningSanitizationSchema,
+    contentHash: Hash,
+})
+    .strict();
+export const ToolEpisodeEventSchema = boundedToolLearningSchema(z
+    .discriminatedUnion("eventKind", [
+    Event.extend({ eventKind: z.literal("attempt"), payload: ToolEpisodeAttemptSchema }).strict(),
+    Event.extend({
+        eventKind: z.literal("observation"),
+        payload: ToolEpisodeObservationSchema,
+    }).strict(),
+    Event.extend({
+        eventKind: z.literal("recovery_link"),
+        payload: ToolEpisodeRecoveryLinkSchema,
+    }).strict(),
+    Event.extend({
+        eventKind: z.literal("verification"),
+        payload: ToolEpisodeVerificationSchema,
+    }).strict(),
+    Event.extend({
+        eventKind: z.literal("lineage"),
+        payload: z
+            .object({
+            relation: z.enum(["fork", "split", "reopen"]),
+            parentEpisode: Ref,
+            parentSessionId: Id,
+            reason: toolLearningText(1_000),
+        })
+            .strict(),
+    }).strict(),
+])
+    .superRefine((event, context) => {
+    const { contentHash, ...body } = event;
+    if (toolLearningContentHash(body) !== contentHash)
+        context.addIssue({
+            code: "custom",
+            path: ["contentHash"],
+            message: "Episode event content hash mismatch.",
+        });
+}), TOOL_LEARNING_LIMITS.eventBytes);
+export function sealToolEpisodeEvent(value) {
+    const hash = toolLearningContentHash(value);
+    if (!value ||
+        typeof value !== "object" ||
+        Array.isArray(value) ||
+        Object.hasOwn(value, "contentHash"))
+        throw new Error("tool_episode_unsigned_envelope_required");
+    return ToolEpisodeEventSchema.parse({ ...value, contentHash: hash });
+}
+export const ToolEpisodeAppendSchema = boundedToolLearningSchema(z
+    .object({
+    schemaVersion: z.literal("openpond.toolEpisodeAppend.v1"),
+    operationId: Id,
+    installationId: Id,
+    events: z.array(ToolEpisodeEventSchema).min(1).max(TOOL_LEARNING_LIMITS.eventsPerAppend),
+})
+    .strict()
+    .superRefine((append, context) => {
+    if (append.events.some((event) => event.installationId !== append.installationId))
+        context.addIssue({
+            code: "custom",
+            path: ["events"],
+            message: "Append events must belong to the exact installation.",
+        });
+    if (new Set(append.events.map((event) => event.eventId)).size !== append.events.length)
+        context.addIssue({
+            code: "custom",
+            path: ["events"],
+            message: "Append event IDs must be unique.",
+        });
+}), TOOL_LEARNING_LIMITS.eventBytes * TOOL_LEARNING_LIMITS.eventsPerAppend);
+export const ToolEpisodeAppendReceiptSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.toolEpisodeAppendReceipt.v1"),
+    operationId: Id,
+    requestHash: Hash,
+    installationId: Id,
+    teamId: Id,
+    projectId: Id,
+    receivedAt: Timestamp,
+    events: z
+        .array(z
+        .object({
+        eventId: Id,
+        contentHash: Hash,
+        state: z.enum(["retained", "already_retained", "privacy_rejected"]),
+        receiptId: Id,
+        reason: z.string().max(200).nullable(),
+    })
+        .strict())
+        .min(1)
+        .max(TOOL_LEARNING_LIMITS.eventsPerAppend),
+    learningEnrollment: z.literal(false),
+})
+    .strict();

package/dist/tool-learning/index.js

@@ -0,0 +1,6 @@
+export * from "./primitives.js";
+export * from "./actions.js";
+export * from "./episodes.js";
+export * from "./installations.js";
+export * from "./consultation.js";
+export * from "./policy.js";