@withgauge/cli 0.15.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -3
- package/dist/index.js +539 -394
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -1755,6 +1755,22 @@ var observedProviderSchema = z19.object({
|
|
|
1755
1755
|
slug: z19.string(),
|
|
1756
1756
|
source: observedProviderSourceSchema
|
|
1757
1757
|
});
|
|
1758
|
+
var RUN_TRIGGER_SOURCES = [
|
|
1759
|
+
"UNKNOWN",
|
|
1760
|
+
"UI",
|
|
1761
|
+
"API",
|
|
1762
|
+
"CADENCE",
|
|
1763
|
+
"STAFF",
|
|
1764
|
+
"OPERATOR",
|
|
1765
|
+
"GITHUB"
|
|
1766
|
+
];
|
|
1767
|
+
var runTriggerSourceSchema = z19.enum(RUN_TRIGGER_SOURCES);
|
|
1768
|
+
var runTriggeredBySchema = z19.object({
|
|
1769
|
+
source: runTriggerSourceSchema,
|
|
1770
|
+
userId: z19.string().nullable(),
|
|
1771
|
+
email: z19.string().nullable()
|
|
1772
|
+
});
|
|
1773
|
+
var runCreditsChargedSchema = z19.number().int().nullable();
|
|
1758
1774
|
var runStatusFilterSchema = z19.string().transform(
|
|
1759
1775
|
(raw) => raw.split(",").map((s) => s.trim().toUpperCase()).filter(Boolean)
|
|
1760
1776
|
).pipe(z19.array(runStatusSchema).min(1));
|
|
@@ -1791,6 +1807,8 @@ var runListItemSchema = z19.object({
|
|
|
1791
1807
|
evalSetId: z19.string().nullable(),
|
|
1792
1808
|
turns: z19.number().int().nullable(),
|
|
1793
1809
|
usdCost: z19.number().nullable(),
|
|
1810
|
+
creditsCharged: runCreditsChargedSchema.optional(),
|
|
1811
|
+
triggeredBy: runTriggeredBySchema.optional(),
|
|
1794
1812
|
durationMs: z19.number().int().nullable(),
|
|
1795
1813
|
exitReason: z19.string().nullable(),
|
|
1796
1814
|
createdAt: z19.string(),
|
|
@@ -2178,6 +2196,8 @@ var evalSetRunSchema = z22.object({
|
|
|
2178
2196
|
startedAt: z22.string().nullable(),
|
|
2179
2197
|
finishedAt: z22.string().nullable(),
|
|
2180
2198
|
failureReason: z22.string().nullable(),
|
|
2199
|
+
creditsCharged: runCreditsChargedSchema.optional(),
|
|
2200
|
+
triggeredBy: runTriggeredBySchema.optional(),
|
|
2181
2201
|
judge: z22.object({ status: z22.string(), error: z22.string().nullable() }).nullable(),
|
|
2182
2202
|
criterionResults: z22.array(
|
|
2183
2203
|
z22.object({
|
|
@@ -2188,9 +2208,7 @@ var evalSetRunSchema = z22.object({
|
|
|
2188
2208
|
})
|
|
2189
2209
|
)
|
|
2190
2210
|
});
|
|
2191
|
-
var listEvalSetRunsResponseSchema =
|
|
2192
|
-
items: z22.array(evalSetRunSchema)
|
|
2193
|
-
});
|
|
2211
|
+
var listEvalSetRunsResponseSchema = listResponseSchema(evalSetRunSchema);
|
|
2194
2212
|
|
|
2195
2213
|
// ../packages/api-schemas/src/executionTargets.ts
|
|
2196
2214
|
import { z as z23 } from "zod";
|
|
@@ -3954,61 +3972,115 @@ var runAddonsSchema = z39.object({
|
|
|
3954
3972
|
}).strict();
|
|
3955
3973
|
|
|
3956
3974
|
// ../packages/api-schemas/src/runDefinitions.ts
|
|
3957
|
-
import { z as
|
|
3975
|
+
import { z as z43 } from "zod";
|
|
3976
|
+
|
|
3977
|
+
// ../packages/api-schemas/src/preference.ts
|
|
3978
|
+
import { z as z40 } from "zod";
|
|
3979
|
+
var preferenceDefinitionSchema = z40.discriminatedUnion("kind", [
|
|
3980
|
+
z40.object({
|
|
3981
|
+
kind: z40.literal("open-ended"),
|
|
3982
|
+
branded: z40.boolean().default(false)
|
|
3983
|
+
}).strict(),
|
|
3984
|
+
z40.object({
|
|
3985
|
+
kind: z40.literal("head-to-head"),
|
|
3986
|
+
brandA: z40.string().trim().min(1),
|
|
3987
|
+
brandB: z40.string().trim().min(1)
|
|
3988
|
+
}).strict().refine(
|
|
3989
|
+
(p) => p.brandA.toLowerCase() !== p.brandB.toLowerCase(),
|
|
3990
|
+
"A head-to-head needs two different brands"
|
|
3991
|
+
)
|
|
3992
|
+
]);
|
|
3993
|
+
var preferenceSnapshotSchema = z40.object({ version: z40.literal(1), preference: preferenceDefinitionSchema }).strict();
|
|
3994
|
+
var headToHeadSchema = z40.object({
|
|
3995
|
+
kind: z40.literal("HEAD_TO_HEAD"),
|
|
3996
|
+
winner: z40.enum(["A", "B", "neither"]),
|
|
3997
|
+
brandA: z40.string(),
|
|
3998
|
+
brandB: z40.string(),
|
|
3999
|
+
rationale: z40.string(),
|
|
4000
|
+
brandAReasoning: z40.string(),
|
|
4001
|
+
brandBReasoning: z40.string(),
|
|
4002
|
+
brandAWhyRaised: z40.string().optional(),
|
|
4003
|
+
brandBWhyRaised: z40.string().optional()
|
|
4004
|
+
});
|
|
4005
|
+
var recommendPickSchema = z40.object({
|
|
4006
|
+
brand: z40.string(),
|
|
4007
|
+
rank: z40.number().int(),
|
|
4008
|
+
adopted: z40.boolean(),
|
|
4009
|
+
reasoning: z40.string().optional(),
|
|
4010
|
+
whyRaised: z40.string().optional()
|
|
4011
|
+
});
|
|
4012
|
+
var recommendSchema = z40.object({
|
|
4013
|
+
kind: z40.literal("RECOMMEND"),
|
|
4014
|
+
picks: z40.array(recommendPickSchema).default([]),
|
|
4015
|
+
singlePick: z40.boolean(),
|
|
4016
|
+
complianceNote: z40.string()
|
|
4017
|
+
});
|
|
4018
|
+
var visibilityVerdictSchema = z40.discriminatedUnion("kind", [
|
|
4019
|
+
headToHeadSchema,
|
|
4020
|
+
recommendSchema
|
|
4021
|
+
]);
|
|
4022
|
+
function preferenceSelectionLabel(verdict) {
|
|
4023
|
+
if (verdict.kind === "HEAD_TO_HEAD")
|
|
4024
|
+
return verdict.winner === "A" ? verdict.brandA : verdict.winner === "B" ? verdict.brandB : "Neither";
|
|
4025
|
+
if (!verdict.singlePick)
|
|
4026
|
+
return verdict.picks.length ? "No clear selection" : "No selection";
|
|
4027
|
+
return verdict.picks.find((pick) => pick.rank === 1)?.brand ?? "No selection";
|
|
4028
|
+
}
|
|
3958
4029
|
|
|
3959
4030
|
// ../packages/api-schemas/src/runDefinitionInputs.ts
|
|
3960
|
-
import { z as
|
|
4031
|
+
import { z as z42 } from "zod";
|
|
3961
4032
|
|
|
3962
4033
|
// ../packages/api-schemas/src/runPlan.ts
|
|
3963
|
-
import { z as
|
|
3964
|
-
var runPlanCriterionSchema =
|
|
3965
|
-
id:
|
|
3966
|
-
name:
|
|
3967
|
-
rubric:
|
|
4034
|
+
import { z as z41 } from "zod";
|
|
4035
|
+
var runPlanCriterionSchema = z41.object({
|
|
4036
|
+
id: z41.string().min(1),
|
|
4037
|
+
name: z41.string(),
|
|
4038
|
+
rubric: z41.string()
|
|
3968
4039
|
});
|
|
3969
|
-
var criteriaSnapshotV1Schema =
|
|
3970
|
-
version:
|
|
4040
|
+
var criteriaSnapshotV1Schema = z41.object({
|
|
4041
|
+
version: z41.literal(1),
|
|
3971
4042
|
// File-authored criteria have no saved EvalSet; their identity is frozen
|
|
3972
4043
|
// in the request and result rows retain the criterion text without an FK.
|
|
3973
|
-
evalSetId:
|
|
3974
|
-
criteria:
|
|
3975
|
-
});
|
|
3976
|
-
var runPlanCaseSchema =
|
|
3977
|
-
id:
|
|
3978
|
-
prompt:
|
|
3979
|
-
workspace:
|
|
3980
|
-
repoUrl:
|
|
3981
|
-
repoRef:
|
|
4044
|
+
evalSetId: z41.string().min(1).nullable(),
|
|
4045
|
+
criteria: z41.array(runPlanCriterionSchema)
|
|
4046
|
+
});
|
|
4047
|
+
var runPlanCaseSchema = z41.object({
|
|
4048
|
+
id: z41.string().min(1),
|
|
4049
|
+
prompt: z41.object({ id: z41.string().min(1), text: z41.string() }),
|
|
4050
|
+
workspace: z41.object({
|
|
4051
|
+
repoUrl: z41.string().nullable(),
|
|
4052
|
+
repoRef: z41.string().nullable()
|
|
3982
4053
|
}),
|
|
3983
|
-
agents:
|
|
4054
|
+
agents: z41.array(
|
|
3984
4055
|
agentConfigSchema.extend({
|
|
3985
4056
|
// Validated against the current catalog request schema when lowered.
|
|
3986
|
-
catalogSelections:
|
|
4057
|
+
catalogSelections: z41.record(z41.string(), z41.unknown()).optional()
|
|
3987
4058
|
})
|
|
3988
4059
|
),
|
|
3989
|
-
samples:
|
|
3990
|
-
evalSetId:
|
|
4060
|
+
samples: z41.number().int().min(1).max(50),
|
|
4061
|
+
evalSetId: z41.string().min(1).optional(),
|
|
3991
4062
|
criteriaSnapshot: criteriaSnapshotV1Schema.optional(),
|
|
3992
|
-
|
|
3993
|
-
|
|
3994
|
-
|
|
3995
|
-
|
|
3996
|
-
|
|
3997
|
-
|
|
3998
|
-
|
|
3999
|
-
|
|
4063
|
+
preferenceSnapshot: preferenceSnapshotSchema.optional(),
|
|
4064
|
+
sourceRepoCommit: z41.string().nullable().optional(),
|
|
4065
|
+
experiment: z41.string().nullable().optional(),
|
|
4066
|
+
membershipTagIds: z41.array(z41.string()).optional(),
|
|
4067
|
+
profile: z41.object({
|
|
4068
|
+
id: z41.string(),
|
|
4069
|
+
name: z41.string(),
|
|
4070
|
+
description: z41.string(),
|
|
4071
|
+
model: z41.string()
|
|
4000
4072
|
}).nullable().optional(),
|
|
4001
4073
|
skillSet: skillSetSchema.optional(),
|
|
4002
4074
|
mcpSet: mcpSetSchema.optional(),
|
|
4003
4075
|
connectionSet: connectionSetSchema.optional(),
|
|
4004
4076
|
// The server's agentContextSnapshotV1Schema validates this at lowering.
|
|
4005
|
-
agentContext:
|
|
4006
|
-
agentContextDigest:
|
|
4077
|
+
agentContext: z41.unknown().optional(),
|
|
4078
|
+
agentContextDigest: z41.string().nullable().optional(),
|
|
4007
4079
|
addons: runAddonsSchema.optional(),
|
|
4008
4080
|
/** Names of resources the coordinator must prepare before expansion. */
|
|
4009
|
-
inputs:
|
|
4081
|
+
inputs: z41.array(z41.string().min(1)).default([])
|
|
4010
4082
|
});
|
|
4011
|
-
var inputPathSchema =
|
|
4083
|
+
var inputPathSchema = z41.string().min(1).max(256).regex(
|
|
4012
4084
|
// biome-ignore lint/suspicious/noControlCharactersInRegex: Reject unsafe control characters in authored paths.
|
|
4013
4085
|
/^(?![!/])(?!.*[\\\x00-\x1f\x7f])(?!.*(?:^|\/)\.{1,2}(?:\/|$))(?!.*\/\/)(?!.*\/$).+$/,
|
|
4014
4086
|
"Expected a repository-relative path or glob without traversal"
|
|
@@ -4043,20 +4115,20 @@ var reservedBinaryCommands = [
|
|
|
4043
4115
|
"pi",
|
|
4044
4116
|
"wizard"
|
|
4045
4117
|
];
|
|
4046
|
-
var binaryCommandSchema =
|
|
4118
|
+
var binaryCommandSchema = z41.string().max(128).regex(
|
|
4047
4119
|
/^[A-Za-z0-9_][A-Za-z0-9_.+-]*$/,
|
|
4048
4120
|
"Expected a command name without paths or shell syntax"
|
|
4049
4121
|
).refine(
|
|
4050
4122
|
(name) => !reservedBinaryCommands.includes(name),
|
|
4051
4123
|
"Command is reserved for Gauge workspace setup or the agent harness; choose another command name"
|
|
4052
4124
|
);
|
|
4053
|
-
var binaryUseSchema =
|
|
4054
|
-
type:
|
|
4055
|
-
paths:
|
|
4056
|
-
platform:
|
|
4125
|
+
var binaryUseSchema = z41.object({
|
|
4126
|
+
type: z41.literal("binary"),
|
|
4127
|
+
paths: z41.array(inputPathSchema).min(1).max(32),
|
|
4128
|
+
platform: z41.literal("linux-x64").describe(
|
|
4057
4129
|
"Linux x86-64 ELF executables compatible with the consumer sandbox. No system packages are installed."
|
|
4058
4130
|
),
|
|
4059
|
-
commands:
|
|
4131
|
+
commands: z41.record(
|
|
4060
4132
|
binaryCommandSchema,
|
|
4061
4133
|
inputPathSchema.regex(
|
|
4062
4134
|
/^[^*?{}[\]]+$/,
|
|
@@ -4067,62 +4139,62 @@ var binaryUseSchema = z40.object({
|
|
|
4067
4139
|
"Declare between 1 and 32 commands"
|
|
4068
4140
|
).meta({ minProperties: 1, maxProperties: 32 })
|
|
4069
4141
|
}).strict();
|
|
4070
|
-
var inputUseSchema =
|
|
4142
|
+
var inputUseSchema = z41.discriminatedUnion("type", [
|
|
4071
4143
|
binaryUseSchema,
|
|
4072
4144
|
websiteUseSchema,
|
|
4073
|
-
|
|
4074
|
-
type:
|
|
4075
|
-
paths:
|
|
4145
|
+
z41.object({
|
|
4146
|
+
type: z41.literal("files"),
|
|
4147
|
+
paths: z41.array(inputPathSchema).min(1).max(32).optional()
|
|
4076
4148
|
}).strict(),
|
|
4077
|
-
|
|
4078
|
-
type:
|
|
4149
|
+
z41.object({
|
|
4150
|
+
type: z41.literal("npm-cli"),
|
|
4079
4151
|
package: inputPathSchema,
|
|
4080
|
-
install:
|
|
4152
|
+
install: z41.enum(["environment", "workspace"]).optional()
|
|
4081
4153
|
}).strict(),
|
|
4082
|
-
|
|
4083
|
-
type:
|
|
4154
|
+
z41.object({
|
|
4155
|
+
type: z41.literal("npm-package"),
|
|
4084
4156
|
package: inputPathSchema,
|
|
4085
|
-
install:
|
|
4157
|
+
install: z41.enum(["environment", "workspace"]).optional()
|
|
4086
4158
|
}).strict(),
|
|
4087
|
-
|
|
4088
|
-
type:
|
|
4089
|
-
root:
|
|
4159
|
+
z41.object({
|
|
4160
|
+
type: z41.literal("skill"),
|
|
4161
|
+
root: z41.union([z41.literal("."), inputPathSchema]).refine(
|
|
4090
4162
|
(path3) => !/[*?{}[\]]/.test(path3),
|
|
4091
4163
|
"Skill root must be an explicit directory"
|
|
4092
4164
|
)
|
|
4093
4165
|
}).strict()
|
|
4094
4166
|
]);
|
|
4095
|
-
var inputSourceSchema =
|
|
4096
|
-
|
|
4097
|
-
type:
|
|
4098
|
-
paths:
|
|
4167
|
+
var inputSourceSchema = z41.discriminatedUnion("type", [
|
|
4168
|
+
z41.object({
|
|
4169
|
+
type: z41.literal("git"),
|
|
4170
|
+
paths: z41.array(inputPathSchema).min(1).max(32)
|
|
4099
4171
|
}).strict(),
|
|
4100
|
-
|
|
4101
|
-
type:
|
|
4172
|
+
z41.object({
|
|
4173
|
+
type: z41.literal("github-actions"),
|
|
4102
4174
|
// Omit to discover the sole artifact from the selected workflow run.
|
|
4103
4175
|
// Names are supplied by CI and need not encode the PR head revision.
|
|
4104
|
-
artifact:
|
|
4176
|
+
artifact: z41.string().min(1).max(256).optional()
|
|
4105
4177
|
}).strict()
|
|
4106
4178
|
]);
|
|
4107
|
-
var runPlanResourceSchema =
|
|
4108
|
-
name:
|
|
4109
|
-
type:
|
|
4110
|
-
source:
|
|
4179
|
+
var runPlanResourceSchema = z41.object({
|
|
4180
|
+
name: z41.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
|
|
4181
|
+
type: z41.literal("input"),
|
|
4182
|
+
source: z41.discriminatedUnion("type", [
|
|
4111
4183
|
commentSourceSchema.extend({
|
|
4112
|
-
repoUrl:
|
|
4113
|
-
commit:
|
|
4184
|
+
repoUrl: z41.string(),
|
|
4185
|
+
commit: z41.string().regex(/^[a-f0-9]{40}$/i)
|
|
4114
4186
|
}).strict(),
|
|
4115
4187
|
deploymentSourceSchema.extend({
|
|
4116
|
-
repoUrl:
|
|
4117
|
-
commit:
|
|
4188
|
+
repoUrl: z41.string(),
|
|
4189
|
+
commit: z41.string().regex(/^[a-f0-9]{40}$/i)
|
|
4118
4190
|
}).strict(),
|
|
4119
4191
|
inputSourceSchema.options[0].extend({
|
|
4120
|
-
repoUrl:
|
|
4121
|
-
commit:
|
|
4192
|
+
repoUrl: z41.string(),
|
|
4193
|
+
commit: z41.string().regex(/^[a-f0-9]{40}$/i)
|
|
4122
4194
|
}).strict(),
|
|
4123
4195
|
inputSourceSchema.options[1].extend({
|
|
4124
|
-
repoUrl:
|
|
4125
|
-
commit:
|
|
4196
|
+
repoUrl: z41.string(),
|
|
4197
|
+
commit: z41.string().regex(/^[a-f0-9]{40}$/i),
|
|
4126
4198
|
workflow: inputPathSchema
|
|
4127
4199
|
}).strict()
|
|
4128
4200
|
]),
|
|
@@ -4131,42 +4203,47 @@ var runPlanResourceSchema = z40.object({
|
|
|
4131
4203
|
(r) => r.use.type === "website" === (r.source.type === "github-deployment" || r.source.type === "github-pr-comment"),
|
|
4132
4204
|
"Website inputs require a deployment or PR comment source; file inputs require git or Actions"
|
|
4133
4205
|
);
|
|
4134
|
-
var runPlanSchema =
|
|
4135
|
-
version:
|
|
4206
|
+
var runPlanSchema = z41.object({
|
|
4207
|
+
version: z41.literal(1),
|
|
4136
4208
|
/** Existing launches keep their current measurement semantics. */
|
|
4137
|
-
measurementMode:
|
|
4138
|
-
|
|
4139
|
-
|
|
4209
|
+
measurementMode: z41.enum([
|
|
4210
|
+
"EXISTING",
|
|
4211
|
+
"AGENT_EXPERIENCE",
|
|
4212
|
+
"AGENT_PREFERENCE",
|
|
4213
|
+
"MIXED"
|
|
4214
|
+
]),
|
|
4215
|
+
cases: z41.array(runPlanCaseSchema).min(1),
|
|
4216
|
+
resources: z41.array(runPlanResourceSchema).default([]),
|
|
4140
4217
|
/** Server-populated context retained across deferred preparation. */
|
|
4141
|
-
launch:
|
|
4142
|
-
origin:
|
|
4143
|
-
|
|
4144
|
-
initiator:
|
|
4145
|
-
userId:
|
|
4146
|
-
email:
|
|
4218
|
+
launch: z41.object({
|
|
4219
|
+
origin: z41.discriminatedUnion("initiator", [
|
|
4220
|
+
z41.object({
|
|
4221
|
+
initiator: z41.literal("API"),
|
|
4222
|
+
userId: z41.string().min(1),
|
|
4223
|
+
email: z41.string().nullable()
|
|
4147
4224
|
}).strict(),
|
|
4148
|
-
|
|
4149
|
-
initiator:
|
|
4150
|
-
userId:
|
|
4151
|
-
email:
|
|
4225
|
+
z41.object({
|
|
4226
|
+
initiator: z41.literal("GITHUB"),
|
|
4227
|
+
userId: z41.null(),
|
|
4228
|
+
email: z41.null()
|
|
4152
4229
|
}).strict()
|
|
4153
4230
|
]),
|
|
4154
|
-
funding:
|
|
4155
|
-
queueClass:
|
|
4231
|
+
funding: z41.enum(["ORG", "PLATFORM"]),
|
|
4232
|
+
queueClass: z41.literal("INTERACTIVE")
|
|
4156
4233
|
}).strict().optional(),
|
|
4157
|
-
provenance:
|
|
4158
|
-
definition:
|
|
4159
|
-
repoUrl:
|
|
4160
|
-
commit:
|
|
4161
|
-
digest:
|
|
4234
|
+
provenance: z41.object({
|
|
4235
|
+
definition: z41.object({
|
|
4236
|
+
repoUrl: z41.string().min(1),
|
|
4237
|
+
commit: z41.string().regex(/^[a-f0-9]{40}$/i),
|
|
4238
|
+
digest: z41.string().min(1),
|
|
4162
4239
|
/** File submissions are asserted by the caller until server-side
|
|
4163
4240
|
* committed-byte fetching is available. */
|
|
4164
|
-
verification:
|
|
4241
|
+
verification: z41.enum(["SUBMITTED_UNVERIFIED", "GITHUB_VERIFIED"]).optional()
|
|
4165
4242
|
}).optional(),
|
|
4166
|
-
liveDependencies:
|
|
4167
|
-
|
|
4168
|
-
name:
|
|
4169
|
-
url:
|
|
4243
|
+
liveDependencies: z41.array(
|
|
4244
|
+
z41.object({
|
|
4245
|
+
name: z41.string().min(1),
|
|
4246
|
+
url: z41.string().url().optional()
|
|
4170
4247
|
})
|
|
4171
4248
|
).default([])
|
|
4172
4249
|
})
|
|
@@ -4190,17 +4267,23 @@ var runPlanSchema = z40.object({
|
|
|
4190
4267
|
commandOwners.set(command, resource.name);
|
|
4191
4268
|
}
|
|
4192
4269
|
}
|
|
4193
|
-
if (plan.measurementMode
|
|
4270
|
+
if (plan.measurementMode !== "EXISTING" && !item.agents.length)
|
|
4194
4271
|
ctx.addIssue({
|
|
4195
4272
|
code: "custom",
|
|
4196
4273
|
path: ["cases", index, "agents"],
|
|
4197
|
-
message: "
|
|
4274
|
+
message: "File cases require a selected agent"
|
|
4198
4275
|
});
|
|
4199
|
-
if (plan.measurementMode
|
|
4276
|
+
if (plan.measurementMode !== "EXISTING" && !item.preferenceSnapshot && !item.criteriaSnapshot?.criteria.length)
|
|
4200
4277
|
ctx.addIssue({
|
|
4201
4278
|
code: "custom",
|
|
4202
4279
|
path: ["cases", index, "criteriaSnapshot"],
|
|
4203
|
-
message: "
|
|
4280
|
+
message: "File cases require criteria or preference"
|
|
4281
|
+
});
|
|
4282
|
+
if (item.preferenceSnapshot && (item.criteriaSnapshot || item.evalSetId))
|
|
4283
|
+
ctx.addIssue({
|
|
4284
|
+
code: "custom",
|
|
4285
|
+
path: ["cases", index],
|
|
4286
|
+
message: "Specify exactly one measurement per case"
|
|
4204
4287
|
});
|
|
4205
4288
|
if (item.criteriaSnapshot && item.criteriaSnapshot.evalSetId !== (item.evalSetId ?? null))
|
|
4206
4289
|
ctx.addIssue({
|
|
@@ -4232,43 +4315,43 @@ var runPlanSchema = z40.object({
|
|
|
4232
4315
|
});
|
|
4233
4316
|
|
|
4234
4317
|
// ../packages/api-schemas/src/runDefinitionInputs.ts
|
|
4235
|
-
var definitionInputSourceSchema =
|
|
4236
|
-
|
|
4237
|
-
|
|
4238
|
-
type:
|
|
4318
|
+
var definitionInputSourceSchema = z42.discriminatedUnion("type", [
|
|
4319
|
+
z42.object({ type: z42.literal("git") }).strict(),
|
|
4320
|
+
z42.object({
|
|
4321
|
+
type: z42.literal("github-actions"),
|
|
4239
4322
|
workflow: inputPathSchema.regex(/^\.github\/workflows\/[^/*?{}[\]]+\.ya?ml$/).describe(
|
|
4240
4323
|
"Workflow file in this repository, for example .github/workflows/build.yml."
|
|
4241
4324
|
),
|
|
4242
|
-
artifact:
|
|
4325
|
+
artifact: z42.string().min(1).max(256).optional().describe(
|
|
4243
4326
|
"Upload name or pattern. Omit only when the workflow produces exactly one artifact."
|
|
4244
4327
|
)
|
|
4245
4328
|
}).strict()
|
|
4246
4329
|
]);
|
|
4247
|
-
var skillInputPathSchema =
|
|
4330
|
+
var skillInputPathSchema = z42.union([z42.literal("."), inputPathSchema.regex(/^[^*?{}[\]]+$/)]).describe(
|
|
4248
4331
|
"Explicit Skill directory or ZIP/tar.gz file containing SKILL.md at its root."
|
|
4249
4332
|
);
|
|
4250
|
-
var runDefinitionInputSchema =
|
|
4333
|
+
var runDefinitionInputSchema = z42.discriminatedUnion("type", [
|
|
4251
4334
|
binaryUseSchema.extend({ source: definitionInputSourceSchema }).strict(),
|
|
4252
4335
|
websiteUseSchema.extend({ source: websiteSourceSchema }).strict(),
|
|
4253
|
-
|
|
4254
|
-
type:
|
|
4336
|
+
z42.object({
|
|
4337
|
+
type: z42.literal("npm-package"),
|
|
4255
4338
|
source: definitionInputSourceSchema,
|
|
4256
4339
|
path: inputPathSchema.describe(
|
|
4257
4340
|
"Path or glob selecting exactly one packed npm .tgz file."
|
|
4258
4341
|
),
|
|
4259
|
-
install:
|
|
4342
|
+
install: z42.enum(["environment", "workspace"]).default("environment").describe(
|
|
4260
4343
|
"environment exposes the package's commands on PATH; workspace adds a consumer application dependency."
|
|
4261
4344
|
)
|
|
4262
4345
|
}).strict(),
|
|
4263
|
-
|
|
4264
|
-
type:
|
|
4346
|
+
z42.object({
|
|
4347
|
+
type: z42.literal("skill"),
|
|
4265
4348
|
source: definitionInputSourceSchema,
|
|
4266
4349
|
path: skillInputPathSchema
|
|
4267
4350
|
}).strict(),
|
|
4268
|
-
|
|
4269
|
-
type:
|
|
4351
|
+
z42.object({
|
|
4352
|
+
type: z42.literal("files"),
|
|
4270
4353
|
source: definitionInputSourceSchema,
|
|
4271
|
-
paths:
|
|
4354
|
+
paths: z42.array(inputPathSchema).min(1).max(32).describe(
|
|
4272
4355
|
"Files or globs selected from the committed Git tree or the uploaded artifact."
|
|
4273
4356
|
)
|
|
4274
4357
|
}).strict()
|
|
@@ -4302,7 +4385,7 @@ function compileDefinitionInput(name, input, repoUrl, commit) {
|
|
|
4302
4385
|
}
|
|
4303
4386
|
|
|
4304
4387
|
// ../packages/api-schemas/src/runDefinitions.ts
|
|
4305
|
-
var uncredentialedRepoUrlSchema =
|
|
4388
|
+
var uncredentialedRepoUrlSchema = z43.string().min(1).refine((raw) => {
|
|
4306
4389
|
if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) return true;
|
|
4307
4390
|
try {
|
|
4308
4391
|
const url = new URL(raw);
|
|
@@ -4311,54 +4394,67 @@ var uncredentialedRepoUrlSchema = z42.string().min(1).refine((raw) => {
|
|
|
4311
4394
|
return false;
|
|
4312
4395
|
}
|
|
4313
4396
|
}, "Repository URL must not contain embedded credentials or query parameters");
|
|
4314
|
-
var
|
|
4315
|
-
|
|
4316
|
-
|
|
4397
|
+
var runDefinitionChecksSchema = z43.object({ paths: z43.array(inputPathSchema).min(1).max(32) }).strict();
|
|
4398
|
+
var runDefinitionConfigSchema = z43.object({
|
|
4399
|
+
agents: z43.array(agentConfigInputSchema).min(1),
|
|
4400
|
+
sampleCount: z43.number().int().min(1).max(50).optional(),
|
|
4317
4401
|
repoUrl: uncredentialedRepoUrlSchema.nullable().optional(),
|
|
4318
|
-
repoRef:
|
|
4319
|
-
profileId:
|
|
4402
|
+
repoRef: z43.string().min(1).nullable().optional(),
|
|
4403
|
+
profileId: z43.string().min(1).nullable().optional(),
|
|
4320
4404
|
skillRefs: assetRefListSchema.optional(),
|
|
4321
4405
|
mcpRefs: assetRefListSchema.optional(),
|
|
4322
|
-
connectionIds:
|
|
4406
|
+
connectionIds: z43.array(z43.string().min(1)).max(32).optional(),
|
|
4323
4407
|
addons: runAddonsSchema.optional(),
|
|
4324
|
-
inputs:
|
|
4408
|
+
inputs: z43.array(z43.string()).optional(),
|
|
4409
|
+
checks: runDefinitionChecksSchema.optional()
|
|
4325
4410
|
}).strict();
|
|
4326
|
-
var runDefinitionFileConfigSchema =
|
|
4327
|
-
version:
|
|
4328
|
-
org:
|
|
4329
|
-
checks:
|
|
4330
|
-
|
|
4331
|
-
|
|
4332
|
-
|
|
4333
|
-
inputs: z42.record(
|
|
4334
|
-
z42.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
|
|
4411
|
+
var runDefinitionFileConfigSchema = z43.object({
|
|
4412
|
+
version: z43.literal(2).default(2),
|
|
4413
|
+
org: z43.string().min(1).optional(),
|
|
4414
|
+
checks: runDefinitionChecksSchema.optional(),
|
|
4415
|
+
cases: z43.array(inputPathSchema).min(1).max(32).default(["gauge-evals/**/*.md"]),
|
|
4416
|
+
inputs: z43.record(
|
|
4417
|
+
z43.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
|
|
4335
4418
|
runDefinitionInputSchema
|
|
4336
4419
|
).refine(
|
|
4337
4420
|
(inputs) => Object.keys(inputs).length <= 32,
|
|
4338
4421
|
"At most 32 inputs are allowed"
|
|
4339
4422
|
).meta({ maxProperties: 32 }).default({})
|
|
4340
4423
|
}).strict();
|
|
4341
|
-
var runDefinitionCriterionSchema =
|
|
4342
|
-
var
|
|
4343
|
-
criteria:
|
|
4424
|
+
var runDefinitionCriterionSchema = z43.object({ name: z43.string().min(1), rubric: z43.string().min(1) }).strict();
|
|
4425
|
+
var caseMeasurement = {
|
|
4426
|
+
criteria: z43.array(runDefinitionCriterionSchema).min(1).optional(),
|
|
4427
|
+
preference: preferenceDefinitionSchema.optional()
|
|
4428
|
+
};
|
|
4429
|
+
function exactlyOneMeasurement(item) {
|
|
4430
|
+
return item.criteria !== void 0 !== (item.preference !== void 0);
|
|
4431
|
+
}
|
|
4432
|
+
var runDefinitionFrontmatterSchema = z43.object({
|
|
4433
|
+
...caseMeasurement,
|
|
4344
4434
|
config: runDefinitionConfigSchema
|
|
4345
|
-
}).strict()
|
|
4346
|
-
|
|
4347
|
-
|
|
4348
|
-
|
|
4349
|
-
|
|
4435
|
+
}).strict().refine(
|
|
4436
|
+
exactlyOneMeasurement,
|
|
4437
|
+
"Specify exactly one of criteria or preference"
|
|
4438
|
+
);
|
|
4439
|
+
var runDefinitionCaseSchema = z43.object({
|
|
4440
|
+
path: z43.string().min(1),
|
|
4441
|
+
prompt: z43.string().min(1),
|
|
4442
|
+
...caseMeasurement,
|
|
4350
4443
|
config: runDefinitionConfigSchema
|
|
4351
|
-
}).strict()
|
|
4352
|
-
|
|
4353
|
-
|
|
4354
|
-
|
|
4444
|
+
}).strict().refine(
|
|
4445
|
+
exactlyOneMeasurement,
|
|
4446
|
+
"Specify exactly one of criteria or preference"
|
|
4447
|
+
);
|
|
4448
|
+
var runDefinitionSubmissionSchema = z43.object({
|
|
4449
|
+
version: z43.literal(2),
|
|
4450
|
+
source: z43.object({
|
|
4355
4451
|
repoUrl: uncredentialedRepoUrlSchema,
|
|
4356
|
-
commit:
|
|
4357
|
-
digest:
|
|
4358
|
-
verification:
|
|
4452
|
+
commit: z43.string().regex(/^[a-f0-9]{40}$/i),
|
|
4453
|
+
digest: z43.string().regex(/^sha256:[a-f0-9]{64}$/),
|
|
4454
|
+
verification: z43.enum(["SUBMITTED_UNVERIFIED", "GITHUB_VERIFIED"])
|
|
4359
4455
|
}).strict(),
|
|
4360
|
-
resources:
|
|
4361
|
-
cases:
|
|
4456
|
+
resources: z43.array(runPlanResourceSchema).max(32).default([]),
|
|
4457
|
+
cases: z43.array(runDefinitionCaseSchema).min(1).max(20)
|
|
4362
4458
|
}).strict().superRefine((definition, ctx) => {
|
|
4363
4459
|
const paths = /* @__PURE__ */ new Set();
|
|
4364
4460
|
for (const [index, item] of definition.cases.entries()) {
|
|
@@ -4401,54 +4497,55 @@ var runDefinitionPublicSubmissionSchema = runDefinitionSubmissionSchema.refine(
|
|
|
4401
4497
|
message: "Only the GitHub App may verify committed bytes"
|
|
4402
4498
|
}
|
|
4403
4499
|
);
|
|
4404
|
-
var runDefinitionPlanResponseSchema =
|
|
4500
|
+
var runDefinitionPlanResponseSchema = z43.object({
|
|
4405
4501
|
source: runDefinitionSubmissionSchema.shape.source,
|
|
4406
|
-
measurementMode:
|
|
4407
|
-
cases:
|
|
4408
|
-
|
|
4409
|
-
path:
|
|
4410
|
-
agents:
|
|
4411
|
-
samples:
|
|
4412
|
-
expectedRuns:
|
|
4413
|
-
workspace:
|
|
4414
|
-
repoUrl:
|
|
4415
|
-
repoRef:
|
|
4502
|
+
measurementMode: z43.enum(["AGENT_EXPERIENCE", "AGENT_PREFERENCE", "MIXED"]),
|
|
4503
|
+
cases: z43.array(
|
|
4504
|
+
z43.object({
|
|
4505
|
+
path: z43.string(),
|
|
4506
|
+
agents: z43.array(agentConfigSchema),
|
|
4507
|
+
samples: z43.number().int().nonnegative(),
|
|
4508
|
+
expectedRuns: z43.number().int().nonnegative(),
|
|
4509
|
+
workspace: z43.object({
|
|
4510
|
+
repoUrl: z43.string().nullable(),
|
|
4511
|
+
repoRef: z43.string().nullable()
|
|
4416
4512
|
}),
|
|
4417
|
-
criteriaCount:
|
|
4418
|
-
|
|
4513
|
+
criteriaCount: z43.number().int().nonnegative(),
|
|
4514
|
+
preference: preferenceDefinitionSchema.optional(),
|
|
4515
|
+
inputs: z43.array(z43.string())
|
|
4419
4516
|
})
|
|
4420
4517
|
),
|
|
4421
|
-
resources:
|
|
4422
|
-
|
|
4423
|
-
name:
|
|
4424
|
-
type:
|
|
4425
|
-
source:
|
|
4518
|
+
resources: z43.array(
|
|
4519
|
+
z43.object({
|
|
4520
|
+
name: z43.string(),
|
|
4521
|
+
type: z43.literal("input"),
|
|
4522
|
+
source: z43.enum([
|
|
4426
4523
|
"git",
|
|
4427
4524
|
"github-actions",
|
|
4428
4525
|
"github-deployment",
|
|
4429
4526
|
"github-pr-comment"
|
|
4430
4527
|
]),
|
|
4431
4528
|
use: inputUseSchema,
|
|
4432
|
-
readiness:
|
|
4433
|
-
detail:
|
|
4434
|
-
commands:
|
|
4529
|
+
readiness: z43.enum(["READY", "WAITING", "FAILED"]),
|
|
4530
|
+
detail: z43.string().optional(),
|
|
4531
|
+
commands: z43.array(z43.string()).optional()
|
|
4435
4532
|
})
|
|
4436
4533
|
),
|
|
4437
|
-
expectedRuns:
|
|
4438
|
-
canRun:
|
|
4534
|
+
expectedRuns: z43.number().int().nonnegative(),
|
|
4535
|
+
canRun: z43.boolean()
|
|
4439
4536
|
});
|
|
4440
|
-
var runDefinitionRunResponseSchema =
|
|
4441
|
-
runRequestId:
|
|
4442
|
-
batchIds:
|
|
4443
|
-
runIds:
|
|
4444
|
-
requestedRuns:
|
|
4445
|
-
createdRuns:
|
|
4446
|
-
aggregateUrl:
|
|
4537
|
+
var runDefinitionRunResponseSchema = z43.object({
|
|
4538
|
+
runRequestId: z43.string(),
|
|
4539
|
+
batchIds: z43.array(z43.string()),
|
|
4540
|
+
runIds: z43.array(z43.string()),
|
|
4541
|
+
requestedRuns: z43.number().int().nonnegative(),
|
|
4542
|
+
createdRuns: z43.number().int().nonnegative(),
|
|
4543
|
+
aggregateUrl: z43.string()
|
|
4447
4544
|
});
|
|
4448
4545
|
|
|
4449
4546
|
// ../packages/api-schemas/src/runRequests.ts
|
|
4450
|
-
import { z as
|
|
4451
|
-
var runRequestStatusSchema =
|
|
4547
|
+
import { z as z44 } from "zod";
|
|
4548
|
+
var runRequestStatusSchema = z44.enum([
|
|
4452
4549
|
"PENDING",
|
|
4453
4550
|
"PREPARING",
|
|
4454
4551
|
"RUNNING",
|
|
@@ -4456,9 +4553,10 @@ var runRequestStatusSchema = z43.enum([
|
|
|
4456
4553
|
"FAILED",
|
|
4457
4554
|
"CANCELED"
|
|
4458
4555
|
]);
|
|
4459
|
-
var runRequestResultSchema =
|
|
4556
|
+
var runRequestResultSchema = z44.enum([
|
|
4460
4557
|
"PENDING",
|
|
4461
4558
|
"PASSED",
|
|
4559
|
+
"MEASURED",
|
|
4462
4560
|
"QUALITY_FAILED",
|
|
4463
4561
|
"EXECUTION_FAILED",
|
|
4464
4562
|
"INPUT_FAILED",
|
|
@@ -4466,224 +4564,229 @@ var runRequestResultSchema = z43.enum([
|
|
|
4466
4564
|
"INCOMPLETE",
|
|
4467
4565
|
"CANCELED"
|
|
4468
4566
|
]);
|
|
4469
|
-
var runRequestCriterionResultSchema =
|
|
4470
|
-
criterionId:
|
|
4471
|
-
name:
|
|
4472
|
-
passed:
|
|
4473
|
-
reasoning:
|
|
4474
|
-
});
|
|
4475
|
-
var runRequestRunSchema =
|
|
4476
|
-
id:
|
|
4477
|
-
batchId:
|
|
4478
|
-
status:
|
|
4567
|
+
var runRequestCriterionResultSchema = z44.object({
|
|
4568
|
+
criterionId: z44.string().nullable(),
|
|
4569
|
+
name: z44.string(),
|
|
4570
|
+
passed: z44.boolean(),
|
|
4571
|
+
reasoning: z44.string().nullable()
|
|
4572
|
+
});
|
|
4573
|
+
var runRequestRunSchema = z44.object({
|
|
4574
|
+
id: z44.string(),
|
|
4575
|
+
batchId: z44.string(),
|
|
4576
|
+
status: z44.string(),
|
|
4479
4577
|
agent: agentSchema,
|
|
4480
|
-
model:
|
|
4578
|
+
model: z44.string(),
|
|
4481
4579
|
result: runRequestResultSchema,
|
|
4482
|
-
judge:
|
|
4483
|
-
criterionResults:
|
|
4580
|
+
judge: z44.object({ status: z44.string() }).nullable(),
|
|
4581
|
+
criterionResults: z44.array(runRequestCriterionResultSchema),
|
|
4582
|
+
preference: visibilityVerdictSchema.nullable().optional(),
|
|
4583
|
+
measurement: z44.enum(["experience", "preference"]).optional()
|
|
4484
4584
|
});
|
|
4485
|
-
var runRequestResponseSchema =
|
|
4486
|
-
id:
|
|
4585
|
+
var runRequestResponseSchema = z44.object({
|
|
4586
|
+
id: z44.string(),
|
|
4487
4587
|
status: runRequestStatusSchema,
|
|
4488
4588
|
result: runRequestResultSchema,
|
|
4489
|
-
error:
|
|
4490
|
-
inputs:
|
|
4491
|
-
settled:
|
|
4492
|
-
counts:
|
|
4493
|
-
requestedRuns:
|
|
4494
|
-
createdRuns:
|
|
4495
|
-
terminalRuns:
|
|
4496
|
-
judgedRuns:
|
|
4497
|
-
passedRuns:
|
|
4498
|
-
|
|
4499
|
-
|
|
4500
|
-
|
|
4501
|
-
|
|
4589
|
+
error: z44.string().nullable().optional(),
|
|
4590
|
+
inputs: z44.unknown().optional(),
|
|
4591
|
+
settled: z44.boolean(),
|
|
4592
|
+
counts: z44.object({
|
|
4593
|
+
requestedRuns: z44.number().int().nonnegative(),
|
|
4594
|
+
createdRuns: z44.number().int().nonnegative(),
|
|
4595
|
+
terminalRuns: z44.number().int().nonnegative(),
|
|
4596
|
+
judgedRuns: z44.number().int().nonnegative(),
|
|
4597
|
+
passedRuns: z44.number().int().nonnegative(),
|
|
4598
|
+
measuredRuns: z44.number().int().nonnegative().optional(),
|
|
4599
|
+
qualityFailedRuns: z44.number().int().nonnegative(),
|
|
4600
|
+
executionFailedRuns: z44.number().int().nonnegative(),
|
|
4601
|
+
judgeFailedRuns: z44.number().int().nonnegative(),
|
|
4602
|
+
pendingRuns: z44.number().int().nonnegative()
|
|
4502
4603
|
}),
|
|
4503
|
-
batches:
|
|
4504
|
-
|
|
4604
|
+
batches: z44.array(
|
|
4605
|
+
z44.object({ id: z44.string(), runs: z44.array(runRequestRunSchema) })
|
|
4505
4606
|
),
|
|
4506
|
-
createdAt:
|
|
4507
|
-
updatedAt:
|
|
4508
|
-
finishedAt:
|
|
4607
|
+
createdAt: z44.string(),
|
|
4608
|
+
updatedAt: z44.string(),
|
|
4609
|
+
finishedAt: z44.string().nullable()
|
|
4509
4610
|
});
|
|
4510
4611
|
|
|
4511
4612
|
// ../packages/api-schemas/src/summary.ts
|
|
4512
|
-
import { z as
|
|
4613
|
+
import { z as z45 } from "zod";
|
|
4513
4614
|
function cardResultSchema(data) {
|
|
4514
|
-
return
|
|
4515
|
-
|
|
4516
|
-
|
|
4615
|
+
return z45.union([
|
|
4616
|
+
z45.object({ ok: z45.literal(true), data }),
|
|
4617
|
+
z45.object({ ok: z45.literal(false), error: z45.string() })
|
|
4517
4618
|
]);
|
|
4518
4619
|
}
|
|
4519
|
-
var summaryActionSchema =
|
|
4520
|
-
id:
|
|
4521
|
-
title:
|
|
4522
|
-
priority:
|
|
4523
|
-
status:
|
|
4524
|
-
evidenceCount:
|
|
4620
|
+
var summaryActionSchema = z45.object({
|
|
4621
|
+
id: z45.string(),
|
|
4622
|
+
title: z45.string(),
|
|
4623
|
+
priority: z45.string(),
|
|
4624
|
+
status: z45.string(),
|
|
4625
|
+
evidenceCount: z45.number().int()
|
|
4525
4626
|
});
|
|
4526
|
-
var summaryAeCardSchema =
|
|
4527
|
-
id:
|
|
4528
|
-
name:
|
|
4529
|
-
judgedRuns:
|
|
4530
|
-
passRate:
|
|
4627
|
+
var summaryAeCardSchema = z45.object({
|
|
4628
|
+
id: z45.string(),
|
|
4629
|
+
name: z45.string(),
|
|
4630
|
+
judgedRuns: z45.number().int(),
|
|
4631
|
+
passRate: z45.number().nullable(),
|
|
4531
4632
|
/** pp delta vs the prior 30d; null with no prior. */
|
|
4532
|
-
deltaPP:
|
|
4633
|
+
deltaPP: z45.number().nullable()
|
|
4533
4634
|
});
|
|
4534
|
-
var summaryVisibilityCardSchema =
|
|
4535
|
-
name:
|
|
4536
|
-
owned:
|
|
4537
|
-
name:
|
|
4538
|
-
rank:
|
|
4539
|
-
chosenRate:
|
|
4540
|
-
mentionRate:
|
|
4635
|
+
var summaryVisibilityCardSchema = z45.object({
|
|
4636
|
+
name: z45.string(),
|
|
4637
|
+
owned: z45.object({
|
|
4638
|
+
name: z45.string(),
|
|
4639
|
+
rank: z45.number(),
|
|
4640
|
+
chosenRate: z45.number(),
|
|
4641
|
+
mentionRate: z45.number()
|
|
4541
4642
|
}).nullable(),
|
|
4542
|
-
chosenDeltaPP:
|
|
4643
|
+
chosenDeltaPP: z45.number().nullable()
|
|
4543
4644
|
});
|
|
4544
|
-
var homeSummarySchema =
|
|
4545
|
-
hasAnyRuns:
|
|
4546
|
-
lastSynthesisAt:
|
|
4547
|
-
actions: cardResultSchema(
|
|
4645
|
+
var homeSummarySchema = z45.object({
|
|
4646
|
+
hasAnyRuns: z45.boolean(),
|
|
4647
|
+
lastSynthesisAt: z45.string().nullable(),
|
|
4648
|
+
actions: cardResultSchema(z45.array(summaryActionSchema)),
|
|
4548
4649
|
visibility: cardResultSchema(
|
|
4549
|
-
|
|
4550
|
-
cards:
|
|
4551
|
-
marketsTotal:
|
|
4552
|
-
promptCount:
|
|
4650
|
+
z45.object({
|
|
4651
|
+
cards: z45.array(summaryVisibilityCardSchema),
|
|
4652
|
+
marketsTotal: z45.number().int(),
|
|
4653
|
+
promptCount: z45.number().int()
|
|
4553
4654
|
})
|
|
4554
4655
|
),
|
|
4555
4656
|
agentExperience: cardResultSchema(
|
|
4556
|
-
|
|
4557
|
-
cards:
|
|
4558
|
-
total:
|
|
4559
|
-
setCount:
|
|
4657
|
+
z45.object({
|
|
4658
|
+
cards: z45.array(summaryAeCardSchema),
|
|
4659
|
+
total: z45.number().int(),
|
|
4660
|
+
setCount: z45.number().int()
|
|
4560
4661
|
})
|
|
4561
4662
|
),
|
|
4562
|
-
notableSessions: cardResultSchema(
|
|
4663
|
+
notableSessions: cardResultSchema(z45.array(z45.unknown()))
|
|
4563
4664
|
});
|
|
4564
4665
|
|
|
4565
4666
|
// ../packages/api-schemas/src/tokens.ts
|
|
4566
|
-
import { z as
|
|
4567
|
-
var apiTokenSchema =
|
|
4568
|
-
id:
|
|
4569
|
-
name:
|
|
4667
|
+
import { z as z46 } from "zod";
|
|
4668
|
+
var apiTokenSchema = z46.object({
|
|
4669
|
+
id: z46.string(),
|
|
4670
|
+
name: z46.string(),
|
|
4570
4671
|
/** Last 4 chars of the plaintext, for recognizability (non-secret). */
|
|
4571
|
-
last4:
|
|
4572
|
-
expiresAt:
|
|
4573
|
-
lastUsedAt:
|
|
4574
|
-
createdAt:
|
|
4575
|
-
revokedAt:
|
|
4576
|
-
});
|
|
4577
|
-
var createTokenRequestSchema =
|
|
4578
|
-
name:
|
|
4672
|
+
last4: z46.string(),
|
|
4673
|
+
expiresAt: z46.iso.datetime().nullable(),
|
|
4674
|
+
lastUsedAt: z46.iso.datetime().nullable(),
|
|
4675
|
+
createdAt: z46.iso.datetime(),
|
|
4676
|
+
revokedAt: z46.iso.datetime().nullable()
|
|
4677
|
+
});
|
|
4678
|
+
var createTokenRequestSchema = z46.object({
|
|
4679
|
+
name: z46.string().min(1).max(100),
|
|
4579
4680
|
/** ISO-8601 datetime; omit for no expiry. */
|
|
4580
|
-
expiresAt:
|
|
4681
|
+
expiresAt: z46.iso.datetime({ offset: true }).optional()
|
|
4581
4682
|
});
|
|
4582
4683
|
var createTokenResponseSchema = apiTokenSchema.extend({
|
|
4583
|
-
token:
|
|
4684
|
+
token: z46.string()
|
|
4584
4685
|
});
|
|
4585
4686
|
var listTokensResponseSchema = listResponseSchema(apiTokenSchema);
|
|
4586
4687
|
|
|
4587
4688
|
// ../packages/api-schemas/src/visibility.ts
|
|
4588
|
-
import { z as
|
|
4689
|
+
import { z as z47 } from "zod";
|
|
4589
4690
|
var VISIBILITY_PROMPT_KINDS = ["HEAD_TO_HEAD", "RECOMMEND"];
|
|
4590
|
-
var visibilityPromptKindSchema =
|
|
4691
|
+
var visibilityPromptKindSchema = z47.enum(VISIBILITY_PROMPT_KINDS);
|
|
4591
4692
|
var visibilityPromptSchema = visibilityPromptSettingsSchema.extend({
|
|
4592
|
-
id:
|
|
4593
|
-
name:
|
|
4693
|
+
id: z47.string(),
|
|
4694
|
+
name: z47.string(),
|
|
4594
4695
|
kind: visibilityPromptKindSchema,
|
|
4595
|
-
promptText:
|
|
4696
|
+
promptText: z47.string(),
|
|
4596
4697
|
/** HEAD_TO_HEAD is always branded; RECOMMEND may be either. */
|
|
4597
|
-
branded:
|
|
4598
|
-
brandA:
|
|
4599
|
-
brandB:
|
|
4600
|
-
market:
|
|
4601
|
-
position:
|
|
4602
|
-
createdAt:
|
|
4698
|
+
branded: z47.boolean(),
|
|
4699
|
+
brandA: z47.string().nullable(),
|
|
4700
|
+
brandB: z47.string().nullable(),
|
|
4701
|
+
market: z47.object({ id: z47.string(), slug: z47.string(), name: z47.string() }).nullable(),
|
|
4702
|
+
position: z47.number().int(),
|
|
4703
|
+
createdAt: z47.string(),
|
|
4603
4704
|
/** User tag names on the hidden backing prompt (Surface:* excluded). */
|
|
4604
|
-
tags:
|
|
4605
|
-
runCount:
|
|
4606
|
-
lastRunAt:
|
|
4607
|
-
scenarios:
|
|
4705
|
+
tags: z47.array(z47.string()),
|
|
4706
|
+
runCount: z47.number().int(),
|
|
4707
|
+
lastRunAt: z47.string().nullable(),
|
|
4708
|
+
scenarios: z47.array(visibilityScenarioSchema)
|
|
4608
4709
|
});
|
|
4609
|
-
var listVisibilityPromptsResponseSchema =
|
|
4610
|
-
items:
|
|
4710
|
+
var listVisibilityPromptsResponseSchema = z47.object({
|
|
4711
|
+
items: z47.array(visibilityPromptSchema)
|
|
4611
4712
|
});
|
|
4612
|
-
var createVisibilityPromptBodySchema =
|
|
4613
|
-
name:
|
|
4614
|
-
promptText:
|
|
4713
|
+
var createVisibilityPromptBodySchema = z47.object({
|
|
4714
|
+
name: z47.string().min(1),
|
|
4715
|
+
promptText: z47.string().min(1),
|
|
4615
4716
|
kind: visibilityPromptKindSchema,
|
|
4616
|
-
branded:
|
|
4717
|
+
branded: z47.boolean().optional(),
|
|
4617
4718
|
/** Required pair for HEAD_TO_HEAD (order-independent). */
|
|
4618
|
-
brandA:
|
|
4619
|
-
brandB:
|
|
4719
|
+
brandA: z47.string().nullish(),
|
|
4720
|
+
brandB: z47.string().nullish(),
|
|
4620
4721
|
/** Measured market id from the org's tracked set. */
|
|
4621
|
-
marketId:
|
|
4622
|
-
tags:
|
|
4722
|
+
marketId: z47.string().nullish(),
|
|
4723
|
+
tags: z47.array(z47.string()).optional(),
|
|
4623
4724
|
agents: visibilityPromptSettingsInputShape.agents.optional(),
|
|
4624
4725
|
skillRefs: visibilityPromptSettingsInputShape.skillRefs,
|
|
4625
4726
|
mcpRefs: visibilityPromptSettingsInputShape.mcpRefs,
|
|
4626
4727
|
connectionIds: visibilityPromptSettingsInputShape.connectionIds,
|
|
4627
4728
|
cadence: visibilityPromptSettingsInputShape.cadence,
|
|
4628
4729
|
sampleCount: visibilityPromptSettingsInputShape.sampleCount,
|
|
4629
|
-
scenarios:
|
|
4730
|
+
scenarios: z47.array(visibilityScenarioInputSchema).optional().default([])
|
|
4630
4731
|
}).strict();
|
|
4631
|
-
var synthesizeResponseSchema =
|
|
4732
|
+
var synthesizeResponseSchema = z47.object({
|
|
4632
4733
|
/** False when the pre-check found no new evidence and nothing was queued. */
|
|
4633
|
-
queued:
|
|
4634
|
-
});
|
|
4635
|
-
var updateVisibilityPromptBodySchema =
|
|
4636
|
-
name:
|
|
4637
|
-
promptText:
|
|
4638
|
-
branded:
|
|
4639
|
-
brandA:
|
|
4640
|
-
brandB:
|
|
4641
|
-
marketId:
|
|
4642
|
-
tags:
|
|
4734
|
+
queued: z47.boolean()
|
|
4735
|
+
});
|
|
4736
|
+
var updateVisibilityPromptBodySchema = z47.object({
|
|
4737
|
+
name: z47.string().min(1).optional(),
|
|
4738
|
+
promptText: z47.string().min(1).optional(),
|
|
4739
|
+
branded: z47.boolean().optional(),
|
|
4740
|
+
brandA: z47.string().nullish(),
|
|
4741
|
+
brandB: z47.string().nullish(),
|
|
4742
|
+
marketId: z47.string().nullish(),
|
|
4743
|
+
tags: z47.array(z47.string()).optional(),
|
|
4643
4744
|
...visibilityPromptSettingsPatchShape,
|
|
4644
4745
|
/** Replaces the whole scenario set when present. */
|
|
4645
|
-
scenarios:
|
|
4746
|
+
scenarios: z47.array(visibilityScenarioInputSchema).optional()
|
|
4646
4747
|
}).strict();
|
|
4647
|
-
var runVisibilityPromptResponseSchema =
|
|
4648
|
-
batchIds:
|
|
4649
|
-
runIds:
|
|
4748
|
+
var runVisibilityPromptResponseSchema = z47.object({
|
|
4749
|
+
batchIds: z47.array(z47.string()),
|
|
4750
|
+
runIds: z47.array(z47.string())
|
|
4650
4751
|
});
|
|
4651
|
-
var visibilityPromptRunSchema =
|
|
4652
|
-
id:
|
|
4653
|
-
status:
|
|
4752
|
+
var visibilityPromptRunSchema = z47.object({
|
|
4753
|
+
id: z47.string(),
|
|
4754
|
+
status: z47.string(),
|
|
4654
4755
|
agent: agentSchema,
|
|
4655
|
-
model:
|
|
4656
|
-
resolvedModel:
|
|
4657
|
-
createdAt:
|
|
4658
|
-
startedAt:
|
|
4659
|
-
finishedAt:
|
|
4660
|
-
failureReason:
|
|
4661
|
-
|
|
4662
|
-
|
|
4663
|
-
|
|
4664
|
-
|
|
4756
|
+
model: z47.string().nullable(),
|
|
4757
|
+
resolvedModel: z47.string().nullable(),
|
|
4758
|
+
createdAt: z47.string(),
|
|
4759
|
+
startedAt: z47.string().nullable(),
|
|
4760
|
+
finishedAt: z47.string().nullable(),
|
|
4761
|
+
failureReason: z47.string().nullable(),
|
|
4762
|
+
creditsCharged: runCreditsChargedSchema.optional(),
|
|
4763
|
+
triggeredBy: runTriggeredBySchema.optional(),
|
|
4764
|
+
verdict: z47.object({
|
|
4765
|
+
status: z47.string(),
|
|
4766
|
+
error: z47.string().nullable(),
|
|
4767
|
+
payload: z47.unknown().nullable()
|
|
4665
4768
|
}).nullable()
|
|
4666
4769
|
});
|
|
4667
|
-
var listVisibilityPromptRunsResponseSchema =
|
|
4668
|
-
|
|
4669
|
-
|
|
4670
|
-
var orgMarketSchema =
|
|
4671
|
-
marketId:
|
|
4672
|
-
slug:
|
|
4673
|
-
name:
|
|
4770
|
+
var listVisibilityPromptRunsResponseSchema = listResponseSchema(
|
|
4771
|
+
visibilityPromptRunSchema
|
|
4772
|
+
);
|
|
4773
|
+
var orgMarketSchema = z47.object({
|
|
4774
|
+
marketId: z47.string(),
|
|
4775
|
+
slug: z47.string(),
|
|
4776
|
+
name: z47.string(),
|
|
4674
4777
|
/** The one line saying what the market measures. */
|
|
4675
|
-
capability:
|
|
4778
|
+
capability: z47.string(),
|
|
4676
4779
|
/** SUGGESTED (the matcher) | USER (a human picked it). */
|
|
4677
|
-
source:
|
|
4780
|
+
source: z47.string()
|
|
4678
4781
|
});
|
|
4679
4782
|
var addableMarketSchema = orgMarketSchema.omit({ source: true });
|
|
4680
|
-
var listMarketsResponseSchema =
|
|
4681
|
-
tracked:
|
|
4783
|
+
var listMarketsResponseSchema = z47.object({
|
|
4784
|
+
tracked: z47.array(orgMarketSchema),
|
|
4682
4785
|
/** Published markets the org does not track yet (closed set — no free-text
|
|
4683
4786
|
* market creation). */
|
|
4684
|
-
addable:
|
|
4787
|
+
addable: z47.array(addableMarketSchema)
|
|
4685
4788
|
});
|
|
4686
|
-
var addMarketBodySchema =
|
|
4789
|
+
var addMarketBodySchema = z47.object({ marketId: z47.string().min(1) });
|
|
4687
4790
|
|
|
4688
4791
|
// src/version.ts
|
|
4689
4792
|
import { createRequire } from "module";
|
|
@@ -5068,7 +5171,7 @@ __export(apply_exports, {
|
|
|
5068
5171
|
register: () => register2
|
|
5069
5172
|
});
|
|
5070
5173
|
import { readFileSync as readFileSync3 } from "fs";
|
|
5071
|
-
import { z as
|
|
5174
|
+
import { z as z48 } from "zod";
|
|
5072
5175
|
function assertRequiredExperiment(spec, required) {
|
|
5073
5176
|
if (required.length === 0) {
|
|
5074
5177
|
throw new CliError(
|
|
@@ -5220,7 +5323,7 @@ function register2(program2) {
|
|
|
5220
5323
|
).option("--schema", "print the battery spec JSON Schema and exit").option("--example", "print a minimal example battery spec and exit").action(
|
|
5221
5324
|
async (opts, command) => {
|
|
5222
5325
|
if (opts.schema) {
|
|
5223
|
-
printJson(
|
|
5326
|
+
printJson(z48.toJSONSchema(batteries_exports.batterySpecSchema));
|
|
5224
5327
|
return;
|
|
5225
5328
|
}
|
|
5226
5329
|
if (opts.example) {
|
|
@@ -7392,7 +7495,7 @@ function register12(program2) {
|
|
|
7392
7495
|
console.log(`Expected sessions: ${result.expectedRuns}`);
|
|
7393
7496
|
for (const item of result.cases)
|
|
7394
7497
|
console.log(
|
|
7395
|
-
` ${item.path}: ${item.expectedRuns} sessions, ${item.criteriaCount} criteria`
|
|
7498
|
+
` ${item.path}: ${item.expectedRuns} sessions, ${item.preference ? `${item.preference.kind} preference (advisory)` : `${item.criteriaCount} criteria`}`
|
|
7396
7499
|
);
|
|
7397
7500
|
for (const resource of result.resources) {
|
|
7398
7501
|
const use = resource.use;
|
|
@@ -7594,29 +7697,38 @@ function register12(program2) {
|
|
|
7594
7697
|
);
|
|
7595
7698
|
}
|
|
7596
7699
|
);
|
|
7597
|
-
group.command("runs <id>").description("The eval set's run log with per-criterion verdicts").option("--limit <n>", "
|
|
7598
|
-
|
|
7599
|
-
|
|
7600
|
-
|
|
7601
|
-
|
|
7602
|
-
|
|
7603
|
-
|
|
7604
|
-
|
|
7605
|
-
|
|
7700
|
+
group.command("runs <id>").description("The eval set's run log with per-criterion verdicts").option("--limit <n>", "page size (1-500, default 100)").option("--cursor <cursor>", "resume from a previous page's nextCursor").action(
|
|
7701
|
+
async (id, opts, cmd) => {
|
|
7702
|
+
const org = resolveOrg(cmd);
|
|
7703
|
+
const res = await createClient().get(
|
|
7704
|
+
`${evalSetsPath(org)}/${encodeURIComponent(id)}/runs`,
|
|
7705
|
+
{ limit: opts.limit, cursor: opts.cursor }
|
|
7706
|
+
);
|
|
7707
|
+
if (resolveOutput(cmd) === "json") {
|
|
7708
|
+
printJson(res);
|
|
7709
|
+
return;
|
|
7710
|
+
}
|
|
7711
|
+
printItems(
|
|
7712
|
+
res.items.map((r) => ({
|
|
7713
|
+
id: r.id,
|
|
7714
|
+
status: r.status,
|
|
7715
|
+
agent: r.agent,
|
|
7716
|
+
model: r.resolvedModel ?? r.model ?? "",
|
|
7717
|
+
source: r.triggeredBy?.source ?? "",
|
|
7718
|
+
credits: r.creditsCharged ?? "",
|
|
7719
|
+
created: r.createdAt,
|
|
7720
|
+
judge: r.judge?.status ?? "-",
|
|
7721
|
+
verdicts: r.criterionResults.length === 0 ? "-" : `${r.criterionResults.filter((c) => c.passed).length}/${r.criterionResults.length} passed`
|
|
7722
|
+
})),
|
|
7723
|
+
"table"
|
|
7724
|
+
);
|
|
7725
|
+
if (res.nextCursor) {
|
|
7726
|
+
console.error(
|
|
7727
|
+
`(more results \u2014 rerun with --cursor ${res.nextCursor})`
|
|
7728
|
+
);
|
|
7729
|
+
}
|
|
7606
7730
|
}
|
|
7607
|
-
|
|
7608
|
-
res.items.map((r) => ({
|
|
7609
|
-
id: r.id,
|
|
7610
|
-
status: r.status,
|
|
7611
|
-
agent: r.agent,
|
|
7612
|
-
model: r.resolvedModel ?? r.model ?? "",
|
|
7613
|
-
created: r.createdAt,
|
|
7614
|
-
judge: r.judge?.status ?? "-",
|
|
7615
|
-
verdicts: r.criterionResults.length === 0 ? "-" : `${r.criterionResults.filter((c) => c.passed).length}/${r.criterionResults.length} passed`
|
|
7616
|
-
})),
|
|
7617
|
-
"table"
|
|
7618
|
-
);
|
|
7619
|
-
});
|
|
7731
|
+
);
|
|
7620
7732
|
group.command("tag <id>").description("Replace an eval set's user tags (Surface:* is preserved)").option(
|
|
7621
7733
|
"--tag <name>",
|
|
7622
7734
|
"tag name, create-on-type (repeatable)",
|
|
@@ -7665,6 +7777,7 @@ description: Create and review Gauge eval prompts, model selections, and judging
|
|
|
7665
7777
|
- Keep repository revisions, credentials, connections, runtime setup, and budgets in supported configuration or fixtures where possible. Inspect \`gauge evals create --help\` and \`gauge evals edit --help\` for current controls; do not assume proposed limits are enforced.
|
|
7666
7778
|
- Saved evals use \`gauge evals create --file <path> --criteria-file <path>\` (criteria JSON: [{"name":"...","rubric":"..."}]). For committed Markdown cases, preview with \`gauge evals plan\` before \`gauge evals run\`; committed \`gauge-evals/**/*.md\` cases are discovered automatically and \`--files\` overrides discovery with paths/globs. Root \`gauge.json\` can attach existing GitHub Actions artifacts and committed files; Gauge consumes them without building. Website inputs route a canonical HTTPS origin to an existing public preview; sources are GitHub Deployments or an existing provider PR comment with exact author, readiness pattern, and candidate SHA. Comments without commit identity and private or path-prefixed previews are unsupported.
|
|
7667
7779
|
- Native command inputs use \`type: "binary"\`, \`platform: "linux-x64"\`, selected \`paths\`, and a \`commands\` map from command names to explicit relative ELF executable files. CI must upload already unpacked files. Gauge restores executable permissions and exposes declared commands on PATH; compatible system libraries are required, nested archives and arbitrary installers are unsupported.
|
|
7780
|
+
- Automatic GitHub PR checks support root \`gauge.json\` \`checks.paths\` as a suite gate and per-case frontmatter \`config.checks.paths\` as a narrower filter. Both accept positive repository-relative globs. Cases without filters run when the suite gate passes; changes to \`gauge.json\` select all cases and edits to a case select that case. Explicit CLI launches and GitHub reruns ignore both filters. Failed/unfinished previous checks, changed target branches, and unavailable diffs run conservatively. \`config.inputs\` selects inputs, not triggers.
|
|
7668
7781
|
- Pilot before scaling; confirm before spending credits. Creating definitions does not authorize launching runs. Report sample counts and distinguish incorrect behavior, missing evidence, environment failures, and budget exhaustion.
|
|
7669
7782
|
- Before committing setup changes, run \`gauge evals verify -o json\` for an offline working-tree checklist. Only attempted static checks are scored yes, no, or unknown; unknown means a specific declaration could not be resolved. Informational findings (such as zero cases) are separate. JSON includes a version, static status (passed/failed/incomplete), summary counts, global/per-input checks with stable IDs, facts, info, and excluded capabilities in notChecked. Exit 1 means a definite problem; exit 0 can still contain unknowns. Live delivery, account settings, and sandbox execution are outside this command's scope. It requires no login or PR and launches no sessions.
|
|
7670
7783
|
|
|
@@ -7680,6 +7793,10 @@ Choose a roster for the question and budget. For exploratory testing, start with
|
|
|
7680
7793
|
|
|
7681
7794
|
Background: [Open models as preference proxies](https://www.withgauge.com/blog/open-models-useful-proxies-data-from-1000-sessions/).
|
|
7682
7795
|
|
|
7796
|
+
## Git-authored Agent Preference
|
|
7797
|
+
Replace \`criteria\` with \`preference: { kind: open-ended }\`, adding \`branded: true\` when the prompt names brands. For a comparison use \`preference: { kind: head-to-head, brandA: PostHog, brandB: Amplitude }\`. Specify exactly one of criteria/preference; the body remains the task and must name the comparison brands itself.
|
|
7798
|
+
Existing per-case settings, inputs, and path filters apply. The app's preference analyst reports advisory choices; competitor selections never fail criteria. Preference-only checks finish neutral; mixed suites retain experience gating. The wait command succeeds for MEASURED/PASSED and fails on operational errors. No base/head delta is inferred.
|
|
7799
|
+
|
|
7683
7800
|
## Migration prompt and criteria
|
|
7684
7801
|
|
|
7685
7802
|
This example tests migration of an existing feature-flag application across browser and server workflows. Its starting repository contains the application, while attached connections supply destination credentials. The source account's targeting configuration is unavailable.
|
|
@@ -7727,7 +7844,7 @@ Explain product choices in their task context, then identify changes worth testi
|
|
|
7727
7844
|
|
|
7728
7845
|
## Establish the measurement
|
|
7729
7846
|
|
|
7730
|
-
Define the question, product role, cohort, and time window. Record prompts, repository revisions, personas, model/harness targets, attached resources, and interventions so comparisons remain interpretable. Open-ended choices and forced head-to-head comparisons answer different questions.
|
|
7847
|
+
Define the question, product role, cohort, and time window. Record prompts, repository revisions, personas, model/harness targets, attached resources, and interventions so comparisons remain interpretable. Open-ended choices and forced head-to-head comparisons answer different questions. Git-authored Markdown cases replace criteria with a preference block: kind open-ended (optionally branded: true), or kind head-to-head with brandA and brandB. The body is the prompt. Use gauge evals verify/plan/run; PR selections are advisory and use the app analyst.
|
|
7731
7848
|
|
|
7732
7849
|
Use \`gauge preference list -o json\` and \`gauge preference get <id> -o json\` to orient, \`gauge runs list --prompt <id> -o json\` for sessions, and \`gauge runs insight <run-id> -o json\` plus \`gauge runs logs <run-id> -o json\` for judgments and trace evidence. Follow \`nextCursor\` across list pages; one page is not the whole cohort. Inspect command help for available filters and configuration.
|
|
7733
7850
|
|
|
@@ -9885,12 +10002,19 @@ function register25(program2) {
|
|
|
9885
10002
|
if (result.settled) {
|
|
9886
10003
|
if (resolveOutput(cmd) === "json") printJson(result);
|
|
9887
10004
|
else {
|
|
10005
|
+
const measured = result.counts.measuredRuns ?? 0;
|
|
9888
10006
|
console.log(
|
|
9889
|
-
`${result.result}: ${result.counts.passedRuns}/${result.counts.createdRuns} passed sessions (${result.counts.judgedRuns} judged)`
|
|
10007
|
+
measured > 0 ? `${result.result}: ${measured} preference sessions measured${result.counts.passedRuns ? `; ${result.counts.passedRuns} experience sessions passed` : ""}` : `${result.result}: ${result.counts.passedRuns}/${result.counts.createdRuns} passed sessions (${result.counts.judgedRuns} judged)`
|
|
9890
10008
|
);
|
|
10009
|
+
for (const batch of result.batches)
|
|
10010
|
+
for (const run of batch.runs)
|
|
10011
|
+
if (run.preference)
|
|
10012
|
+
console.log(
|
|
10013
|
+
` ${run.id}: ${preferenceSelectionLabel(run.preference)} (advisory)`
|
|
10014
|
+
);
|
|
9891
10015
|
console.log(`Request: ${result.id}`);
|
|
9892
10016
|
}
|
|
9893
|
-
if (result.result !== "PASSED")
|
|
10017
|
+
if (result.result !== "PASSED" && result.result !== "MEASURED")
|
|
9894
10018
|
throw new CliError(
|
|
9895
10019
|
"internal",
|
|
9896
10020
|
`Run request ${id}: ${result.result}`
|
|
@@ -9921,6 +10045,10 @@ function fmtDuration(ms) {
|
|
|
9921
10045
|
function fmtUsd(v) {
|
|
9922
10046
|
return v == null ? "" : `$${v.toFixed(2)}`;
|
|
9923
10047
|
}
|
|
10048
|
+
function fmtTriggeredBy(t) {
|
|
10049
|
+
if (!t) return "";
|
|
10050
|
+
return t.email ? `${t.source} (${t.email})` : t.source;
|
|
10051
|
+
}
|
|
9924
10052
|
function compact(value, max = 140) {
|
|
9925
10053
|
const s = typeof value === "string" ? value : JSON.stringify(value) ?? "null";
|
|
9926
10054
|
const one = s.replace(/\s+/g, " ").trim();
|
|
@@ -9936,8 +10064,10 @@ function toRow5(run) {
|
|
|
9936
10064
|
model: run.modelDisplayName ?? run.resolvedModel ?? run.model,
|
|
9937
10065
|
provider: observed && observed !== selected ? `${selected || "?"} \u2192 ${observed}` : selected || observed || "",
|
|
9938
10066
|
experiment: run.experiment ?? "",
|
|
10067
|
+
source: run.triggeredBy?.source ?? "",
|
|
9939
10068
|
turns: run.turns ?? "",
|
|
9940
10069
|
cost: fmtUsd(run.usdCost),
|
|
10070
|
+
credits: run.creditsCharged ?? "",
|
|
9941
10071
|
duration: fmtDuration(run.durationMs),
|
|
9942
10072
|
created: run.createdAt,
|
|
9943
10073
|
prompt: compact(run.promptSnippet, 48)
|
|
@@ -9974,7 +10104,9 @@ function detailLines(run) {
|
|
|
9974
10104
|
],
|
|
9975
10105
|
["Attempt", String(run.attempt)],
|
|
9976
10106
|
["Turns", run.turns == null ? "\u2014" : String(run.turns)],
|
|
10107
|
+
["Triggered by", fmtTriggeredBy(run.triggeredBy) || "\u2014"],
|
|
9977
10108
|
["Cost", fmtUsd(run.usdCost) || "\u2014"],
|
|
10109
|
+
["Credits", run.creditsCharged == null ? "\u2014" : String(run.creditsCharged)],
|
|
9978
10110
|
["Duration", fmtDuration(run.durationMs) || "\u2014"],
|
|
9979
10111
|
["Exit reason", run.exitReason ?? "\u2014"],
|
|
9980
10112
|
["Failure reason", run.failureReason ?? "\u2014"],
|
|
@@ -10196,7 +10328,9 @@ function register26(program2) {
|
|
|
10196
10328
|
}
|
|
10197
10329
|
}
|
|
10198
10330
|
);
|
|
10199
|
-
runs.command("get <id>").description(
|
|
10331
|
+
runs.command("get <id>").description(
|
|
10332
|
+
"Show a run (status, trigger source, cost, credits, turns, exit reason)"
|
|
10333
|
+
).action(async (id, _opts, command) => {
|
|
10200
10334
|
const run = await createClient().get(runPath(id));
|
|
10201
10335
|
if (resolveOutput(command) === "json") {
|
|
10202
10336
|
printJson(run);
|
|
@@ -10851,9 +10985,9 @@ function scenarioRow(scenario) {
|
|
|
10851
10985
|
persona: scenario.profile?.name ?? "(default)"
|
|
10852
10986
|
};
|
|
10853
10987
|
}
|
|
10854
|
-
var KIND_CHOICES = ["head-to-head", "
|
|
10988
|
+
var KIND_CHOICES = ["head-to-head", "open-ended"];
|
|
10855
10989
|
function parseKind(raw) {
|
|
10856
|
-
const upper = raw.trim().toUpperCase().replace(/-/g, "_");
|
|
10990
|
+
const upper = raw === "open-ended" ? "RECOMMEND" : raw.trim().toUpperCase().replace(/-/g, "_");
|
|
10857
10991
|
if (upper !== "HEAD_TO_HEAD" && upper !== "RECOMMEND")
|
|
10858
10992
|
throw new CliError(
|
|
10859
10993
|
"usage",
|
|
@@ -10869,7 +11003,7 @@ function toRow6(v) {
|
|
|
10869
11003
|
return {
|
|
10870
11004
|
id: v.id,
|
|
10871
11005
|
name: v.name,
|
|
10872
|
-
kind: v.kind,
|
|
11006
|
+
kind: v.kind === "RECOMMEND" ? "open-ended" : "head-to-head",
|
|
10873
11007
|
brands: brandsLabel(v),
|
|
10874
11008
|
market: v.market?.slug ?? "",
|
|
10875
11009
|
tags: v.tags.join(", "),
|
|
@@ -10880,7 +11014,9 @@ function toRow6(v) {
|
|
|
10880
11014
|
function printDetail4(v) {
|
|
10881
11015
|
console.log(`id: ${v.id}`);
|
|
10882
11016
|
console.log(`name: ${v.name}`);
|
|
10883
|
-
console.log(
|
|
11017
|
+
console.log(
|
|
11018
|
+
`kind: ${v.kind === "RECOMMEND" ? "open-ended" : "head-to-head"}`
|
|
11019
|
+
);
|
|
10884
11020
|
console.log(`brands: ${brandsLabel(v)}`);
|
|
10885
11021
|
console.log(
|
|
10886
11022
|
`market: ${v.market ? `${v.market.name} (${v.market.slug})` : "(none)"}`
|
|
@@ -10925,7 +11061,7 @@ function registerPreferenceGroup(group) {
|
|
|
10925
11061
|
addRunSettingsCreateOptions(
|
|
10926
11062
|
group.command("create").description(
|
|
10927
11063
|
"Create a preference prompt. Run-settings flags set its roster, resources, cadence, and samples; --repo/--ref/--persona add its first scenario. Omit them all for a bare definition on the org's default roster."
|
|
10928
|
-
).requiredOption("--name <name>", "preference prompt name").requiredOption("--kind <kind>", KIND_CHOICES.join(" | ")).option("--text <text>", "prompt text inline").option("--file <path>", 'prompt text from a file ("-" for stdin)').option("--brand-a <name>", "first brand (required for head-to-head)").option("--brand-b <name>", "second brand (required for head-to-head)").option("--branded", "
|
|
11064
|
+
).requiredOption("--name <name>", "preference prompt name").requiredOption("--kind <kind>", KIND_CHOICES.join(" | ")).option("--text <text>", "prompt text inline").option("--file <path>", 'prompt text from a file ("-" for stdin)').option("--brand-a <name>", "first brand (required for head-to-head)").option("--brand-b <name>", "second brand (required for head-to-head)").option("--branded", "an open-ended prompt that names a brand").option(
|
|
10929
11065
|
"--repo <url>",
|
|
10930
11066
|
"first scenario's repository (default: empty project)"
|
|
10931
11067
|
).option("--ref <ref>", "branch, tag, or commit (with --repo)").option("--persona <profileId>", "first scenario's persona profile id")
|
|
@@ -11106,28 +11242,37 @@ function registerPreferenceGroup(group) {
|
|
|
11106
11242
|
result.queued ? "Synthesis queued \u2014 insights land on the Agent Preference page shortly" : "Nothing new since the last pass \u2014 no synthesis queued"
|
|
11107
11243
|
);
|
|
11108
11244
|
});
|
|
11109
|
-
group.command("runs <id>").description("The prompt's run log with Agent Preference verdicts").option("--limit <n>", "
|
|
11110
|
-
|
|
11111
|
-
|
|
11112
|
-
|
|
11113
|
-
|
|
11114
|
-
|
|
11115
|
-
|
|
11116
|
-
|
|
11117
|
-
|
|
11245
|
+
group.command("runs <id>").description("The prompt's run log with Agent Preference verdicts").option("--limit <n>", "page size (1-500, default 100)").option("--cursor <cursor>", "resume from a previous page's nextCursor").action(
|
|
11246
|
+
async (id, opts, cmd) => {
|
|
11247
|
+
const org = resolveOrg(cmd);
|
|
11248
|
+
const res = await createClient().get(
|
|
11249
|
+
`${visibilityPath(org)}/${encodeURIComponent(id)}/runs`,
|
|
11250
|
+
{ limit: opts.limit, cursor: opts.cursor }
|
|
11251
|
+
);
|
|
11252
|
+
if (resolveOutput(cmd) === "json") {
|
|
11253
|
+
printJson(res);
|
|
11254
|
+
return;
|
|
11255
|
+
}
|
|
11256
|
+
printItems(
|
|
11257
|
+
res.items.map((r) => ({
|
|
11258
|
+
id: r.id,
|
|
11259
|
+
status: r.status,
|
|
11260
|
+
agent: r.agent,
|
|
11261
|
+
model: r.resolvedModel ?? r.model ?? "",
|
|
11262
|
+
source: r.triggeredBy?.source ?? "",
|
|
11263
|
+
credits: r.creditsCharged ?? "",
|
|
11264
|
+
created: r.createdAt,
|
|
11265
|
+
verdict: r.verdict?.status ?? "-"
|
|
11266
|
+
})),
|
|
11267
|
+
"table"
|
|
11268
|
+
);
|
|
11269
|
+
if (res.nextCursor) {
|
|
11270
|
+
console.error(
|
|
11271
|
+
`(more results \u2014 rerun with --cursor ${res.nextCursor})`
|
|
11272
|
+
);
|
|
11273
|
+
}
|
|
11118
11274
|
}
|
|
11119
|
-
|
|
11120
|
-
res.items.map((r) => ({
|
|
11121
|
-
id: r.id,
|
|
11122
|
-
status: r.status,
|
|
11123
|
-
agent: r.agent,
|
|
11124
|
-
model: r.resolvedModel ?? r.model ?? "",
|
|
11125
|
-
created: r.createdAt,
|
|
11126
|
-
verdict: r.verdict?.status ?? "-"
|
|
11127
|
-
})),
|
|
11128
|
-
"table"
|
|
11129
|
-
);
|
|
11130
|
-
});
|
|
11275
|
+
);
|
|
11131
11276
|
group.command("tag <id>").description("Replace a preference prompt's user tags").option(
|
|
11132
11277
|
"--tag <name>",
|
|
11133
11278
|
"tag name, create-on-type (repeatable)",
|