@openpond/evals 0.11.19 → 0.11.21

package/dist/connected-evidence/collector-coverage-contracts.js

@@ -0,0 +1,160 @@
+import { z } from "zod";
+import { contentHash } from "@openpond/harness";
+const Id = z.string().trim().min(1).max(500);
+const Hash = z.string().regex(/^[a-f0-9]{64}$/);
+const Timestamp = z.iso.datetime();
+const Count = z.number().int().nonnegative().max(1_000_000);
+export const CollectorCoverageSessionSchema = z
+    .object({
+    sourceKeyHash: Hash,
+    nativeRevisionHash: Hash.nullable(),
+    state: z.enum(["reading", "admitted", "skipped", "failed"]),
+    reason: z
+        .enum([
+        "no_eligible_boundaries",
+        "native_session_not_ready",
+        "source_read_failed",
+        "source_revision_unavailable",
+    ])
+        .nullable(),
+    unknownTimeBoundaries: Count,
+    normalized: z
+        .array(z
+        .object({
+        sessionHash: Hash,
+        boundaries: z
+            .array(z
+            .object({ id: Id, inputHash: Hash, revisionHash: Hash, occurredAt: Timestamp })
+            .strict())
+            .max(5_000),
+    })
+        .strict())
+        .max(1_000),
+})
+    .strict();
+export const CollectorCoverageManifestSchema = z
+    .object({
+    schemaVersion: z.literal("openpond.collectorCoverageManifest.v1"),
+    jobId: Id,
+    connectionId: Id,
+    connectionRevision: z.number().int().positive(),
+    machineId: Id,
+    sourceInstanceId: Id,
+    from: Timestamp.nullable(),
+    to: Timestamp,
+    startedAt: Timestamp,
+    finishedAt: Timestamp.nullable(),
+    listingComplete: z.boolean(),
+    pendingOperations: Count,
+    sessions: z.array(CollectorCoverageSessionSchema).max(10_000),
+})
+    .strict()
+    .superRefine((value, ctx) => {
+    if (value.from !== null && Date.parse(value.from) >= Date.parse(value.to))
+        ctx.addIssue({
+            code: "custom",
+            message: "Collector source window is empty.",
+        });
+    if (value.finishedAt !== null &&
+        Date.parse(value.finishedAt) < Date.parse(value.startedAt))
+        ctx.addIssue({
+            code: "custom",
+            message: "Collector finish time precedes its start.",
+        });
+    if (new Set(value.sessions.map((session) => session.sourceKeyHash)).size !==
+        value.sessions.length)
+        ctx.addIssue({
+            code: "custom",
+            message: "Collector source inventory is duplicated.",
+        });
+    let boundaries = 0;
+    for (const session of value.sessions)
+        for (const normalized of session.normalized) {
+            boundaries += normalized.boundaries.length;
+            if (new Set(normalized.boundaries.map((boundary) => boundary.id)).size !==
+                normalized.boundaries.length)
+                ctx.addIssue({
+                    code: "custom",
+                    message: "Normalized source boundaries are duplicated.",
+                });
+            for (const boundary of normalized.boundaries)
+                if (Date.parse(boundary.occurredAt) >= Date.parse(value.to) ||
+                    (value.from !== null &&
+                        Date.parse(boundary.occurredAt) < Date.parse(value.from)))
+                    ctx.addIssue({
+                        code: "custom",
+                        message: "Eligible boundary is outside the declared event-time window.",
+                    });
+        }
+    if (boundaries > 20_000)
+        ctx.addIssue({
+            code: "custom",
+            message: "Collector coverage exceeds its bounded boundary inventory.",
+        });
+});
+export const CollectorCoverageSummarySchema = z
+    .object({
+    jobId: Id,
+    manifestHash: Hash,
+    sourceInventoryHash: Hash,
+    eligibleBoundaryHash: Hash,
+    from: Timestamp.nullable(),
+    to: Timestamp,
+    finishedAt: Timestamp.nullable(),
+    conversations: Count,
+    eligibleTurns: Count,
+    admittedSessions: Count,
+    skippedSessions: Count,
+    failedSessions: Count,
+    unknownTimeBoundaries: Count,
+    pendingOperations: Count,
+    listingComplete: z.boolean(),
+    complete: z.boolean(),
+})
+    .strict();
+export function hashCollectorEligibleBoundaries(input) {
+    const boundaries = new Map(input.map((boundary) => [
+        JSON.stringify([boundary.id, boundary.inputHash, boundary.revisionHash]),
+        { id: boundary.id, inputHash: boundary.inputHash, revisionHash: boundary.revisionHash },
+    ]));
+    const ordered = [...boundaries]
+        .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+        .map(([, boundary]) => boundary);
+    return { hash: contentHash(ordered), count: ordered.length };
+}
+export function summarizeCollectorCoverage(raw) {
+    const manifest = CollectorCoverageManifestSchema.parse(raw);
+    const boundaries = hashCollectorEligibleBoundaries(manifest.sessions.flatMap((session) => session.normalized.flatMap((normalized) => normalized.boundaries)));
+    const unknownTimeBoundaries = manifest.sessions.reduce((sum, session) => sum + session.unknownTimeBoundaries, 0);
+    return CollectorCoverageSummarySchema.parse({
+        jobId: manifest.jobId,
+        manifestHash: contentHash(manifest),
+        sourceInventoryHash: contentHash(manifest.sessions
+            .map((session) => ({
+            sourceKeyHash: session.sourceKeyHash,
+            nativeRevisionHash: session.nativeRevisionHash,
+            snapshots: session.normalized.map((normalized) => normalized.sessionHash),
+        }))
+            .sort((a, b) => a.sourceKeyHash.localeCompare(b.sourceKeyHash))),
+        eligibleBoundaryHash: boundaries.hash,
+        from: manifest.from,
+        to: manifest.to,
+        finishedAt: manifest.finishedAt,
+        conversations: manifest.sessions.length,
+        eligibleTurns: boundaries.count,
+        admittedSessions: manifest.sessions.filter((session) => session.state === "admitted").length,
+        skippedSessions: manifest.sessions.filter((session) => session.state === "skipped").length,
+        failedSessions: manifest.sessions.filter((session) => session.state === "failed").length,
+        unknownTimeBoundaries,
+        pendingOperations: manifest.pendingOperations,
+        listingComplete: manifest.listingComplete,
+        complete: manifest.finishedAt !== null &&
+            manifest.listingComplete &&
+            manifest.pendingOperations === 0 &&
+            unknownTimeBoundaries === 0 &&
+            manifest.sessions.every((session) => session.nativeRevisionHash !== null &&
+                (session.state === "admitted" ||
+                    (session.state === "skipped" &&
+                        session.reason === "no_eligible_boundaries"))),
+    });
+}

package/dist/connected-evidence/contracts.js

@@ -3,7 +3,7 @@ import { contentHash } from "@openpond/harness";
 export const CONNECTED_EVIDENCE_VERSION = "openpond.connectedEvidence.v1";
 export const CONNECTED_NORMALIZER_VERSION = "connected-evidence-1";
 export const CONNECTED_EVIDENCE_LIMITS = { files: 100, sourceBytes: 32 * 1024 * 1024, decodedBytes: 64 * 1024 * 1024,
-    lineBytes: 2 * 1024 * 1024, events: 100_000, sessions: 1_000, boundaries: 5_000, evaluatorBytes: 2 * 1024 * 1024 };
+    lineBytes: 64 * 1024 * 1024, events: 100_000, sessions: 1_000, boundaries: 5_000, evaluatorBytes: 2 * 1024 * 1024 };
 const Id = z.string().min(1).max(500);
 const Hash = z.string().regex(/^[a-f0-9]{64}$/);
 const Time = z.string().datetime().nullable();

package/dist/connected-evidence/imports.js

@@ -1,6 +1,6 @@
 import { contentHash } from "@openpond/harness";
 import { CONNECTED_EVIDENCE_LIMITS, CONNECTED_NORMALIZER_VERSION, } from "./contracts.js";
-import { connectedJsonLines, normalizeConnectedSession } from "./normalize.js";
+import { connectedJsonLines, normalizeConnectedSession, ConnectedSessionWithoutUserRequestError } from "./normalize.js";
 import { parseGrokBuildSession } from "./sources/grok-build.js";
 import { parsePiSession } from "./sources/pi.js";
 import { parseOpenCodeSession } from "./sources/opencode.js";
@@ -55,6 +55,7 @@ export function previewAgentImport(input) {
         catch (error) {
             issues.push({
                 file,
+                ...(error instanceof ConnectedSessionWithoutUserRequestError ? { code: "no_user_request" } : {}),
                 message: error instanceof Error ? error.message : "Unsupported session.",
             });
         }

package/dist/connected-evidence/index.js

@@ -3,3 +3,4 @@ export * from "./normalize.js";
 export * from "./imports.js";
 export * from "./usage-summary.js";
 export * from "./python-correctness.js";
+export * from "./collector-coverage-contracts.js";

package/dist/connected-evidence/normalize.js

@@ -1,5 +1,8 @@
-import { contentHash } from "@openpond/harness";
+import { contentHash, contentHashArrayPrefixes } from "@openpond/harness";
 import { CONNECTED_EVIDENCE_LIMITS, CONNECTED_EVIDENCE_VERSION, CONNECTED_NORMALIZER_VERSION, ConnectedSessionSchema, hasRecordedConnectedAnswer } from "./contracts.js";
+export class ConnectedSessionWithoutUserRequestError extends Error {
+    constructor() { super("The session has no retained user request."); }
+}
 export function connectedTimestamp(value, units = "milliseconds") {
     if (typeof value !== "number" && typeof value !== "string")
         return null;
@@ -17,10 +20,12 @@ export function normalizeConnectedSession(input) {
         throw new Error("connected_duplicate_event");
     const starts = events.flatMap((event, index) => event.kind === "message" && event.role === "user" ? [index] : []);
     if (!starts.length)
-        throw new Error("The session has no retained user request.");
+        throw new ConnectedSessionWithoutUserRequestError();
     if (starts.length + (input.includeConversation === false ? 0 : 1) > CONNECTED_EVIDENCE_LIMITS.boundaries)
         throw new Error("connected_boundary_limit");
     const familyKey = `family-${contentHash([input.origin, ...(input.acquisition ? [input.acquisition.sourceInstanceId] : []), input.parentSessionId ?? input.sessionId]).slice(0, 32)}`;
+    const inputHashes = contentHashArrayPrefixes(events, starts.map(start => start + 1));
+    const firstCompaction = events.findIndex(event => event.kind === "compaction");
     const boundaries = starts.map((start, position) => {
         const end = starts[position + 1] ?? events.length;
         const observed = events.slice(start + 1, end);
@@ -32,12 +37,12 @@ export function normalizeConnectedSession(input) {
         const terminal = [...observed].reverse().find(event => event.kind === "terminal");
         const status = terminal && typeof terminal.content === "object" && terminal.content !== null && !Array.isArray(terminal.content)
             ? terminal.content.status : null;
-        const compacted = events.slice(0, start + 1).some(event => event.kind === "compaction");
+        const compacted = firstCompaction >= 0 && firstCompaction <= start;
         const unknown = observed.some(event => event.kind === "unknown");
         return {
             id: `case-${contentHash([input.origin, ...(input.acquisition ? [input.acquisition.sourceInstanceId] : []), input.sessionId, input.branchId ?? null, events[start].id]).slice(0, 40)}`,
             familyKey, projection: "turn", requestEventId: events[start].id, start, end,
-            inputHash: contentHash(events.slice(0, start + 1)), outputHash: answer ? contentHash(answer.content) : null,
+            inputHash: inputHashes[position], outputHash: answer ? contentHash(answer.content) : null,
             revisionHash: contentHash(observed), terminal: status === "completed" || status === "failed" || status === "cancelled" ? status : "unknown",
             coverage: { answer: Boolean(answer), context: input.contextComplete === false ? "unknown" : compacted ? "compacted" : "retained",
                 process: !tools.length ? unknown ? "partial" : "absent" : incomplete || unknown ? "partial" : "retained",
@@ -69,6 +74,11 @@ export function normalizeConnectedSession(input) {
     return session;
 }
 export function connectedJsonLines(file) {
+    // Native tool results and image blocks can occupy one large JSONL record.
+    // Bound the whole decoded input before parsing, rather than rejecting valid
+    // evidence solely because its exporter uses a single physical line.
+    if (new TextEncoder().encode(file.text).length > CONNECTED_EVIDENCE_LIMITS.decodedBytes)
+        throw new Error("Decoded source files exceed 64 MiB.");
     const lines = file.text.replace(/^\uFEFF/u, "").split(/\r?\n/u);
     if (lines.length > CONNECTED_EVIDENCE_LIMITS.events + 1)
         throw new Error("connected_event_limit");

package/dist/execution-contracts.js

@@ -83,7 +83,7 @@ export const VerifierSetReleaseContentSchema = z.object({
     schemaVersion: z.literal("openpond.verifierSetRelease.v1"),
     id: ReleaseIdSchema,
     revision: z.number().int().positive(),
-    graders: z.array(GraderSpecSchema).min(1).max(1_000),
+    graders: z.array(GraderSpecSchema).max(1_000),
     isolation: z.object({
         processBoundary: z.enum(["same_process", "isolated_process", "container"]),
         networkPolicy: z.literal("none"),

package/dist/javascript-isolate-node.js

@@ -1,11 +1,12 @@
 import { Worker } from "node:worker_threads";
-import { javascriptVerifierWorkerSource } from "./javascript-verifier-worker-source.js";
 import { validateJavaScriptIsolateInput } from "./javascript-isolate.js";
 import { assertBoundedTaskJson } from "./task-schema.js";
 /** Both environment steps and graders settle only after their worker stops. */
 export async function executeJavaScriptIsolateInWorker(input) {
     input.signal?.throwIfAborted();
     validateJavaScriptIsolateInput(input);
+    const { javascriptVerifierWorkerSource } = await import("./javascript-verifier-worker-source.js");
+    input.signal?.throwIfAborted();
     const signal = input.signal;
     const workerData = { source: input.source, exportName: input.exportName, value: input.value, timeoutMs: input.timeoutMs, maxResultBytes: input.maxResultBytes, deterministic: input.deterministic === true, errorPrefix: input.errorPrefix };
     const error = (suffix) => new Error(`${input.errorPrefix}_${suffix}`);