@openpond/evals 0.12.0 → 0.12.1

package/dist/artifact-verification.js

@@ -143,6 +143,8 @@ function structuralFailures(required, entry) {
     }
     if (entry.parseStatus === "failed")
         failures.push(`Required output ${required.path} could not be parsed.`);
+    if (required.submissionMode === "structured_json" && (entry.schemaStatus !== "passed" || entry.parseStatus !== "passed" || entry.metadata.structuredSchemaHash !== required.schemaRef?.contentHash || entry.metadata.structuredArtifactHash !== entry.artifact?.contentHash))
+        failures.push(`Required output ${required.path} lacks bound structured validation evidence.`);
     if (entry.schemaStatus === "failed")
         failures.push(`Required output ${required.path} failed schema validation.`);
     return failures;

package/dist/connected-evidence/sources/claude-branches.js

@@ -48,5 +48,38 @@ export function claudeGraph(rows) {
             return { leafId: leaf.uuid, chainHash: selected.chainHash, title: title.slice(0, 180), updatedAt: connectedTimestamp(last?.timestamp), messages: messages.length };
         });
     }
-    return { sessionId, leaves, select, branches };
+    function latestLeafId() {
+        // Match Claude's session browser: walk terminal bookkeeping back to a
+        // message, then prefer the last main-chain message in file order. Wall
+        // clock timestamps and later sidechain/meta records do not choose the chat.
+        const candidates = new Set();
+        for (const leaf of leaves) {
+            let current = leaf;
+            const seen = new Set();
+            while (current) {
+                const id = current.uuid;
+                if (seen.has(id))
+                    throw new Error("Claude Code branch contains a parent cycle.");
+                seen.add(id);
+                if (current.type === "user" || current.type === "assistant") {
+                    candidates.add(id);
+                    break;
+                }
+                current = typeof current.parentUuid === "string" ? byId.get(current.parentUuid) : undefined;
+            }
+        }
+        let latest, fallback;
+        for (const row of nodes) {
+            const id = row.uuid;
+            if (!candidates.has(id))
+                continue;
+            fallback = id;
+            if (!row.isSidechain && !row.teamName && !row.isMeta)
+                latest = id;
+        }
+        if (!latest && !fallback)
+            throw new Error("Claude Code transcript has no conversation message.");
+        return (latest ?? fallback);
+    }
+    return { sessionId, leaves, select, branches, latestLeafId };
 }

package/dist/connected-evidence/sources/claude-code.js

@@ -30,7 +30,7 @@ export function parseClaudeSession(file, rows, leafId) {
             if (block.type === "tool_use")
                 events.push(event({ ...base, id, kind: "tool_call", callId: typeof block.id === "string" ? block.id : null, content: json(block) }));
             else if (block.type === "tool_result")
-                events.push(event({ ...base, id, kind: "tool_result", role: "tool", callId: typeof block.tool_use_id === "string" ? block.tool_use_id : null, content: json(block) }));
+                events.push(event({ ...base, id, kind: "tool_result", role: "tool", callId: typeof block.tool_use_id === "string" ? block.tool_use_id : null, content: json({ ...block, ...(row.tool_use_result !== undefined ? { nativeToolResult: row.tool_use_result } : {}) }) }));
             else
                 events.push(event({ ...base, id, kind: "unknown", content: json(block) }));
         }

package/dist/native-conversations/collector-acquisition.js

@@ -7,6 +7,7 @@ import { CollectorQueueFullError } from "./collector-snapshots.js";
 import { COLLECTOR_DEFAULTS } from "./collector-contracts.js";
 import { CollectorCoverage } from "./collector-coverage.js";
 import { CollectorSourceCache } from "./collector-source-cache.js";
+import { nativeSessionSelectionKey } from "./selection.js";
 /** One pass over a frozen listing per page. Unchanged revisions are cheap; a
  * failed source is tried again only in a later explicitly launched job. */
 export async function acquireCollectorSource(input) {
@@ -14,23 +15,30 @@ export async function acquireCollectorSource(input) {
     const cache = new CollectorSourceCache(store);
     let cursor, processed = 0, discovered = 0;
     const failures = [];
+    const selection = connection.selectedSessionKeys ? new Set(connection.selectedSessionKeys) : null;
+    const found = new Set();
+    let enumerated = 0;
     do {
         await check();
         progress({ phase: "discovering" });
         // File mtimes and native activity summaries are not event-time authority.
         // Enumerate the source, then apply the consented window to native events.
         const page = await listSessions(connection.source, { cursor, limit: 50 });
-        discovered += page.items.length;
-        if (discovered > 10_000)
-            throw new Error("Source inventory exceeds 10,000 sessions. Select a narrower import window.");
-        for (const native of page.items) {
+        enumerated += page.items.length;
+        if (enumerated > 10_000)
+            throw new Error("Source inventory exceeds 10,000 sessions. Select a narrower source directory.");
+        const items = page.items.filter(native => !selection || selection.has(nativeSessionSelectionKey(native)));
+        for (const native of items)
+            found.add(nativeSessionSelectionKey(native));
+        discovered += items.length;
+        for (const native of items) {
             const scanKey = JSON.stringify([native.path, native.nativeSessionId]);
             store.coverage.record(connection.id, { sourceKeyHash: CollectorCoverage.sourceKey(scanKey),
                 nativeRevisionHash: native.storageRevision ? contentHash([native.storageRevision, agentImportNormalizerRevision(connection.source.source)]) : null,
                 state: "reading", reason: null, unknownTimeBoundaries: 0, normalized: [] });
         }
         progress({ discovered });
-        for (const native of page.items) {
+        for (const native of items) {
             await check();
             // Do not parse another transcript while any retained admission is blocked.
             await drain();
@@ -139,7 +147,10 @@ export async function acquireCollectorSource(input) {
         cursor = page.nextCursor ?? undefined;
     } while (cursor);
     store.progress.discovered(connection.id);
-    store.coverage.finish(connection.id, true);
+    // A chosen subset cannot certify coverage of the entire source window.
+    store.coverage.finish(connection.id, selection === null);
+    if (selection && [...selection].some(key => !found.has(key)))
+        throw new Error("A selected conversation is no longer available. Review the upload selection.");
     if (failures.length)
         throw new Error(`${failures.length} session(s) need attention. ${failures[0]}`);
 }

package/dist/native-conversations/collector-source-cache.js

@@ -12,7 +12,7 @@ export class CollectorSourceCache {
     }
     selection(connection, until, revision) {
         return contentHash([connection.source, connection.projectId, connection.teamId,
-            connection.apiBaseUrl, connection.account ?? null, connection.since, until, revision]);
+            connection.apiBaseUrl, connection.account ?? null, connection.since, connection.selectedSessionKeys ?? null, until, revision]);
     }
     read(connection, until, sessionKey, revision) {
         const row = this.store.database.prepare("SELECT value FROM source_read_cache WHERE connection_id=? AND session_key=? AND selection_hash=?").get(connection.id, sessionKey, this.selection(connection, until, revision));

package/dist/native-conversations/collector-store.js

@@ -61,6 +61,20 @@ export class CollectorStore {
                 this.database.exec("COMMIT");
                 return;
             }
+            if (prior && JSON.stringify(prior.selectedSessionKeys) !== JSON.stringify(connection.selectedSessionKeys)) {
+                const owner = JSON.parse(this.setting("owner") || "{}");
+                if (owner.pid) {
+                    let alive = true;
+                    try {
+                        process.kill(owner.pid, 0);
+                    }
+                    catch (error) {
+                        alive = error.code !== "ESRCH";
+                    }
+                    if (alive)
+                        throw new Error("Cancel or finish the running import before changing its conversation selection.");
+                }
+            }
             const identityChanged = !!prior &&
                 (prior.source.instanceId !== connection.source.instanceId ||
                     prior.projectId !== connection.projectId ||
@@ -69,7 +83,8 @@ export class CollectorStore {
                     prior.account !== connection.account);
             if (identityChanged)
                 connection = { ...connection, destinations: undefined };
-            if (prior && (identityChanged || prior.since !== connection.since)) {
+            if (prior && (identityChanged || prior.since !== connection.since ||
+                JSON.stringify(prior.selectedSessionKeys) !== JSON.stringify(connection.selectedSessionKeys))) {
                 this.database.prepare("DELETE FROM settings WHERE key=?").run(`coverage:${connection.id}`);
                 this.database.prepare("DELETE FROM coverage_sessions WHERE connection_id=?").run(connection.id);
                 // Scope changes fence queued evidence too: do not upload a wider old
@@ -81,7 +96,16 @@ export class CollectorStore {
                 this.database.prepare("DELETE FROM settings WHERE key=?").run(`syncAcknowledged:${connection.id}`);
                 connection = { ...connection, completedSyncRevision: 0 };
                 this.error(connection.id, null);
-                for (const table of ["source_scans", "checkpoints", "pending"])
+                if (!identityChanged && prior.since === connection.since) {
+                    // Preserve acknowledged boundaries when adding conversations, but
+                    // reacquire every queued boundary whose snapshot is being discarded.
+                    this.database.prepare(`DELETE FROM checkpoints WHERE connection_id=? AND EXISTS (
+            SELECT 1 FROM pending p, json_each(p.payload,'$.boundaryIds') b
+            WHERE p.connection_id=checkpoints.connection_id
+              AND json_extract(p.payload,'$.sessionKey')=checkpoints.session_key
+              AND b.value=checkpoints.boundary_id)`).run(connection.id);
+                }
+                for (const table of identityChanged || prior.since !== connection.since ? ["source_scans", "checkpoints", "pending"] : ["source_scans", "pending"])
                     this.database
                         .prepare(`DELETE FROM ${table} WHERE connection_id=?`)
                         .run(connection.id);
@@ -270,6 +294,7 @@ export class CollectorStore {
                         .get(String(row.id))?.bytes ?? 0),
                     lastAdmissionAt: this.setting(`lastAdmission:${row.id}`),
                     since: config.since,
+                    selectedConversations: config.selectedSessionKeys?.length ?? null,
                     schedule,
                     nextRunAt: schedule && config.state === "active" ? nextScheduledSync(schedule) : null,
                     lastSuccessfulSyncAt: this.setting(`lastSuccess:${row.id}`),

package/dist/native-conversations/collector.js

@@ -36,7 +36,7 @@ export async function runCollector(input) {
             }
         }
         const now = new Date();
-        selected = store.connections().filter(connection => connection.state !== "disconnected" &&
+        selected = store.connections().filter(connection => connection.state !== "disconnected" && connection.selectedSessionKeys?.length !== 0 &&
             (!input.connectionIds || input.connectionIds.includes(connection.id)) &&
             (input.trigger !== "scheduled" || (connection.state === "active" && scheduledSyncDue(store.schedule(connection.id), store.setting(`scheduledMinute:${connection.id}`), now))));
         if (input.trigger === "scheduled")
@@ -173,13 +173,14 @@ export async function runCollector(input) {
                 for (let pass = 0; pass < 3; pass++) {
                     await check();
                     const revision = connection.revision, requested = connection.requestedSyncRevision ?? 0;
+                    const selection = JSON.stringify(connection.selectedSessionKeys);
                     phase = "source";
                     new CollectorSourceCache(store).seed(connection, until, store.coverage.manifest(connection.id));
                     store.coverage.begin(connection, `${nonce}-${pass}`, run.startedAt, until);
                     const checkAcquisition = async () => {
                         await check();
                         phase = "source";
-                        if (connection.revision !== revision)
+                        if (connection.revision !== revision || JSON.stringify(connection.selectedSessionKeys) !== selection)
                             throw new CollectorScopeChangedError("Source settings changed during acquisition.");
                     };
                     try {
@@ -193,7 +194,7 @@ export async function runCollector(input) {
                     }
                     errors.set(connection.id, "source", null);
                     await drain();
-                    await input.transport.publishCoverage(connection, store.coverage.finish(connection.id, true), signal);
+                    await input.transport.publishCoverage(connection, store.coverage.finish(connection.id, connection.selectedSessionKeys === undefined), signal);
                     connection = retained(connection);
                     if (connection.revision !== revision) {
                         if (pass === 2)

package/dist/native-conversations/history.js

@@ -293,6 +293,13 @@ export async function readSession(source, session, input = {}) {
         const graph = claudeGraph(connectedJsonLines(file));
         if (graph.sessionId !== session.nativeSessionId)
             throw new Error("Native session changed identity.");
+        if (input.latestBranch) {
+            if (input.branchLeafId || input.expectedBranchRevision || input.branchAnchor)
+                throw new Error("Choose either the latest conversation or an explicit branch.");
+            // Desktop inspection follows the native conversation. Imports keep their
+            // existing explicit branch authority and never enable this option.
+            files = [{ ...file, text: graph.select(graph.latestLeafId()).chain.map(row => JSON.stringify(row)).join("\n") + "\n" }];
+        }
         if (input.branchAnchor) {
             if (input.branchLeafId || input.expectedBranchRevision)
                 throw new Error("Choose either a frozen branch snapshot or a followed branch anchor.");

package/dist/native-conversations/index.js

@@ -2,6 +2,7 @@
 export * from "./contracts.js";
 export * from "./discovery.js";
 export * from "./history.js";
+export * from "./selection.js";
 export * from "./collector-contracts.js";
 export * from "./collector-store.js";
 export * from "./collector.js";

package/dist/native-conversations/selection.js

@@ -0,0 +1,57 @@
+import { contentHash } from "@openpond/harness";
+import { z } from "zod";
+import { listSessions } from "./history.js";
+export const NativeUploadSelectionSchema = z.object({
+    sourceInstanceId: z.string().min(1).max(500),
+    sessionKeys: z.array(z.string().regex(/^[a-f0-9]{64}$/)).min(1).max(10_000),
+}).strict().refine(value => new Set(value.sessionKeys).size === value.sessionKeys.length, "Conversation selection contains duplicates.");
+/** Includes the source path so duplicate native IDs cannot select a sibling file. */
+export function nativeSessionSelectionKey(session) {
+    return contentHash(JSON.stringify([session.path, session.nativeSessionId]));
+}
+export async function nativeUploadInventory(source) {
+    if (!source.available || !source.capabilities.history)
+        throw new Error("The selected conversation source is unavailable.");
+    const items = [];
+    let cursor;
+    const seen = new Set();
+    do {
+        const page = await listSessions(source, { cursor, limit: 100 });
+        items.push(...page.items);
+        if (items.length > 10_000)
+            throw new Error("Choose a source with no more than 10,000 conversations.");
+        cursor = page.nextCursor ?? undefined;
+        if (cursor && seen.has(cursor))
+            throw new Error("Conversation listing did not advance. Refresh the source.");
+        if (cursor)
+            seen.add(cursor);
+    } while (cursor);
+    return items;
+}
+export async function validateNativeUploadSelection(source, raw) {
+    const selection = NativeUploadSelectionSchema.parse(raw);
+    if (selection.sourceInstanceId !== source.instanceId)
+        throw new Error("Conversation selection belongs to another source installation.");
+    const available = new Set((await nativeUploadInventory(source)).map(nativeSessionSelectionKey));
+    if (selection.sessionKeys.some(key => !available.has(key)))
+        throw new Error("A selected conversation is no longer available. Refresh and select it again.");
+    return selection;
+}
+export function nativeUploadConversationPage(connection, inventory, input) {
+    const query = (input.query ?? "").trim().toLocaleLowerCase(), range = input.range ?? "all", offset = input.offset ?? 0;
+    if (query.length > 500 || !["day", "week", "all"].includes(range) || !Number.isInteger(offset) || offset < 0 || offset > 10_000)
+        throw new Error("Invalid conversation search or page.");
+    const since = range === "all" ? 0 : Date.now() - (range === "day" ? 1 : 7) * 86400000;
+    const items = inventory.filter(item => Date.parse(item.updatedAt) >= since &&
+        (!query || [item.title, item.cwd, item.nativeSessionId].some(text => text?.toLocaleLowerCase().includes(query))))
+        .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
+    const approved = connection.selectedSessionKeys ? new Set(connection.selectedSessionKeys) : null;
+    return {
+        total: items.length, nextOffset: offset + 50 < items.length ? offset + 50 : null,
+        items: items.slice(offset, offset + 50).map(item => ({
+            key: nativeSessionSelectionKey(item), title: item.title, cwd: item.cwd,
+            nativeSessionId: item.nativeSessionId, updatedAt: item.updatedAt, issue: item.issue ?? null,
+            approved: approved === null || approved.has(nativeSessionSelectionKey(item)),
+        })),
+    };
+}

package/dist/native-conversations/supervisor.js

@@ -164,6 +164,8 @@ export async function setCollectorSchedule(input, connectionId, schedule) {
         const connection = store.connections().find(item => item.id === connectionId);
         if (!connection || (schedule && connection.state === "disconnected"))
             throw new Error("Select a connected source before scheduling imports.");
+        if (schedule && connection.selectedSessionKeys?.length === 0)
+            throw new Error("Choose conversations to upload before enabling continual imports.");
         const owner = store.setting("scheduleOwner");
         if (owner) {
             const { pid } = JSON.parse(owner);

package/dist/tasksets.js

@@ -7,6 +7,7 @@ export const RequiredOutputContractSchema = z.object({
     path: z.string().trim().min(1).max(2_000).refine(safeRelativePath),
     mediaType: z.string().trim().min(1).max(200),
     schemaRef: ImmutableAssetRefSchema.nullable().default(null),
+    submissionMode: z.literal("structured_json").optional(),
     maxBytes: z.number().int().positive().max(250_000_000).nullable().default(null),
     metadata: MetadataSchema,
 }).strict();

package/dist/types/artifact-verification.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"artifact-verification.d.ts","sourceRoot":"","sources":["../../../../src/artifact-verification.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE9D,OAAO,EAGL,KAAK,gBAAgB,EAGrB,KAAK,sBAAsB,EAC5B,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAE5D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACtC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,WAAW,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACpD,YAAY,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACrD,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,aAAa,GAAG,IAAI,CAAC;IAC7D,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAwEnB;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,QAAQ,EAAE,gBAAgB,CAAC;CAC5B,GAAG,sBAAsB,EAAE,CAqC3B"}
+{"version":3,"file":"artifact-verification.d.ts","sourceRoot":"","sources":["../../src/artifact-verification.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE9D,OAAO,EAGL,KAAK,gBAAgB,EAGrB,KAAK,sBAAsB,EAC5B,MAAM,0BAA0B,CAAC;AAElC,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,eAAe,CAAC;AAE5D,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACtC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,WAAW,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACpD,YAAY,CAAC,EAAE,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACrD,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,aAAa,GAAG,IAAI,CAAC;IAC7D,YAAY,CAAC,EAAE,oBAAoB,EAAE,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,CAAC;AAEF,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAChD,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,gBAAgB,CAwEnB;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE;IAC3C,eAAe,EAAE,sBAAsB,EAAE,CAAC;IAC1C,QAAQ,EAAE,gBAAgB,CAAC;CAC5B,GAAG,sBAAsB,EAAE,CAqC3B"}

package/dist/types/benchmarks.d.ts

@@ -2,10 +2,10 @@ import { z } from "zod";
 import { type AttemptReceipt, type EvaluationResult } from "./runs.js";
 export { harnessRefinerBenchmarkAssets, harnessRefinerBenchmarkRelease, } from "./builtin-benchmarks/harness-refiner.js";
 export declare const BenchmarkMetricSchema: z.ZodEnum<{
+    cost_usd: "cost_usd";
     foreground_tokens: "foreground_tokens";
-    success_rate: "success_rate";
     latency_ms: "latency_ms";
-    cost_usd: "cost_usd";
+    success_rate: "success_rate";
 }>;
 export declare const BenchmarkRunPhaseSchema: z.ZodEnum<{
     baseline: "baseline";
@@ -13,10 +13,10 @@ export declare const BenchmarkRunPhaseSchema: z.ZodEnum<{
 }>;
 export declare const BenchmarkProtocolSchema: z.ZodObject<{
     split: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     taskIds: z.ZodArray<z.ZodString>;
     seeds: z.ZodArray<z.ZodString>;
@@ -36,27 +36,27 @@ export declare const BenchmarkDefinitionSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>;
     adaptationSplit: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     evaluationSplit: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     primaryMetric: z.ZodEnum<{
+        cost_usd: "cost_usd";
         foreground_tokens: "foreground_tokens";
-        success_rate: "success_rate";
         latency_ms: "latency_ms";
-        cost_usd: "cost_usd";
+        success_rate: "success_rate";
     }>;
     qualityGate: z.ZodEnum<{
-        none: "none";
-        non_regression: "non_regression";
         all_pass: "all_pass";
+        non_regression: "non_regression";
+        none: "none";
     }>;
     caseCounts: z.ZodObject<{
         adaptation: z.ZodNumber;
@@ -80,10 +80,10 @@ export declare const BenchmarkRunRequestSchema: z.ZodObject<{
     }, z.core.$strict>;
     reasoningEffort: z.ZodNullable<z.ZodString>;
     split: z.ZodEnum<{
+        frozen_eval: "frozen_eval";
+        test: "test";
         train: "train";
         validation: "validation";
-        test: "test";
-        frozen_eval: "frozen_eval";
     }>;
     seeds: z.ZodArray<z.ZodString>;
     repetitions: z.ZodNumber;
@@ -119,10 +119,10 @@ export declare const BenchmarkRunSummaryContentSchema: z.ZodObject<{
     reasoningEffort: z.ZodNullable<z.ZodString>;
     protocol: z.ZodObject<{
         split: z.ZodEnum<{
+            frozen_eval: "frozen_eval";
+            test: "test";
             train: "train";
             validation: "validation";
-            test: "test";
-            frozen_eval: "frozen_eval";
         }>;
         taskIds: z.ZodArray<z.ZodString>;
         seeds: z.ZodArray<z.ZodString>;
@@ -175,10 +175,10 @@ export declare const BenchmarkRunSummarySchema: z.ZodObject<{
     reasoningEffort: z.ZodNullable<z.ZodString>;
     protocol: z.ZodObject<{
         split: z.ZodEnum<{
+            frozen_eval: "frozen_eval";
+            test: "test";
             train: "train";
             validation: "validation";
-            test: "test";
-            frozen_eval: "frozen_eval";
         }>;
         taskIds: z.ZodArray<z.ZodString>;
         seeds: z.ZodArray<z.ZodString>;
@@ -218,10 +218,10 @@ export declare const BenchmarkComparisonContentSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>;
     primaryMetric: z.ZodEnum<{
+        cost_usd: "cost_usd";
         foreground_tokens: "foreground_tokens";
-        success_rate: "success_rate";
         latency_ms: "latency_ms";
-        cost_usd: "cost_usd";
+        success_rate: "success_rate";
     }>;
     qualityPassed: z.ZodBoolean;
     baselinePassRate: z.ZodNumber;
@@ -248,10 +248,10 @@ export declare const BenchmarkComparisonSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>;
     primaryMetric: z.ZodEnum<{
+        cost_usd: "cost_usd";
         foreground_tokens: "foreground_tokens";
-        success_rate: "success_rate";
         latency_ms: "latency_ms";
-        cost_usd: "cost_usd";
+        success_rate: "success_rate";
     }>;
     qualityPassed: z.ZodBoolean;
     baselinePassRate: z.ZodNumber;

package/dist/types/benchmarks.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"benchmarks.d.ts","sourceRoot":"","sources":["../../../../src/benchmarks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAGvE,OAAO,EACL,6BAA6B,EAC7B,8BAA8B,GAC/B,MAAM,yCAAyC,CAAC;AAEjD,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AACH,eAAO,MAAM,uBAAuB;;;EAAoC,CAAC;AAEzE,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;kBASzB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAe3B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAQZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBlC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAelC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,GAC/C,mBAAmB,CAErB;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,iBAAiB,CAAC;IACzB,UAAU,EAAE,gBAAgB,CAAC;IAC7B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuCtB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,SAAS,EAAE,mBAAmB,CAAC;IAC/B,aAAa,EAAE,eAAe,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,gBAAgB,GAAG,UAAU,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuDtB;AAkDD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
+{"version":3,"file":"benchmarks.d.ts","sourceRoot":"","sources":["../../src/benchmarks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAGvE,OAAO,EACL,6BAA6B,EAC7B,8BAA8B,GAC/B,MAAM,yCAAyC,CAAC;AAEjD,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AACH,eAAO,MAAM,uBAAuB;;;EAAoC,CAAC;AAEzE,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;kBASzB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAe3B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAQZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBlC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAelC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,GAC/C,mBAAmB,CAErB;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,iBAAiB,CAAC;IACzB,UAAU,EAAE,gBAAgB,CAAC;IAC7B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuCtB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,SAAS,EAAE,mBAAmB,CAAC;IAC/B,aAAa,EAAE,eAAe,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,gBAAgB,GAAG,UAAU,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuDtB;AAkDD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}