@withgauge/cli 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/README.md +47 -2
  2. package/dist/index.js +536 -392
  3. package/package.json +1 -1
package/dist/index.js CHANGED
@@ -1755,6 +1755,22 @@ var observedProviderSchema = z19.object({
1755
1755
  slug: z19.string(),
1756
1756
  source: observedProviderSourceSchema
1757
1757
  });
1758
+ var RUN_TRIGGER_SOURCES = [
1759
+ "UNKNOWN",
1760
+ "UI",
1761
+ "API",
1762
+ "CADENCE",
1763
+ "STAFF",
1764
+ "OPERATOR",
1765
+ "GITHUB"
1766
+ ];
1767
+ var runTriggerSourceSchema = z19.enum(RUN_TRIGGER_SOURCES);
1768
+ var runTriggeredBySchema = z19.object({
1769
+ source: runTriggerSourceSchema,
1770
+ userId: z19.string().nullable(),
1771
+ email: z19.string().nullable()
1772
+ });
1773
+ var runCreditsChargedSchema = z19.number().int().nullable();
1758
1774
  var runStatusFilterSchema = z19.string().transform(
1759
1775
  (raw) => raw.split(",").map((s) => s.trim().toUpperCase()).filter(Boolean)
1760
1776
  ).pipe(z19.array(runStatusSchema).min(1));
@@ -1791,6 +1807,8 @@ var runListItemSchema = z19.object({
1791
1807
  evalSetId: z19.string().nullable(),
1792
1808
  turns: z19.number().int().nullable(),
1793
1809
  usdCost: z19.number().nullable(),
1810
+ creditsCharged: runCreditsChargedSchema.optional(),
1811
+ triggeredBy: runTriggeredBySchema.optional(),
1794
1812
  durationMs: z19.number().int().nullable(),
1795
1813
  exitReason: z19.string().nullable(),
1796
1814
  createdAt: z19.string(),
@@ -2178,6 +2196,8 @@ var evalSetRunSchema = z22.object({
2178
2196
  startedAt: z22.string().nullable(),
2179
2197
  finishedAt: z22.string().nullable(),
2180
2198
  failureReason: z22.string().nullable(),
2199
+ creditsCharged: runCreditsChargedSchema.optional(),
2200
+ triggeredBy: runTriggeredBySchema.optional(),
2181
2201
  judge: z22.object({ status: z22.string(), error: z22.string().nullable() }).nullable(),
2182
2202
  criterionResults: z22.array(
2183
2203
  z22.object({
@@ -2188,9 +2208,7 @@ var evalSetRunSchema = z22.object({
2188
2208
  })
2189
2209
  )
2190
2210
  });
2191
- var listEvalSetRunsResponseSchema = z22.object({
2192
- items: z22.array(evalSetRunSchema)
2193
- });
2211
+ var listEvalSetRunsResponseSchema = listResponseSchema(evalSetRunSchema);
2194
2212
 
2195
2213
  // ../packages/api-schemas/src/executionTargets.ts
2196
2214
  import { z as z23 } from "zod";
@@ -3954,61 +3972,115 @@ var runAddonsSchema = z39.object({
3954
3972
  }).strict();
3955
3973
 
3956
3974
  // ../packages/api-schemas/src/runDefinitions.ts
3957
- import { z as z42 } from "zod";
3975
+ import { z as z43 } from "zod";
3976
+
3977
+ // ../packages/api-schemas/src/preference.ts
3978
+ import { z as z40 } from "zod";
3979
+ var preferenceDefinitionSchema = z40.discriminatedUnion("kind", [
3980
+ z40.object({
3981
+ kind: z40.literal("open-ended"),
3982
+ branded: z40.boolean().default(false)
3983
+ }).strict(),
3984
+ z40.object({
3985
+ kind: z40.literal("head-to-head"),
3986
+ brandA: z40.string().trim().min(1),
3987
+ brandB: z40.string().trim().min(1)
3988
+ }).strict().refine(
3989
+ (p) => p.brandA.toLowerCase() !== p.brandB.toLowerCase(),
3990
+ "A head-to-head needs two different brands"
3991
+ )
3992
+ ]);
3993
+ var preferenceSnapshotSchema = z40.object({ version: z40.literal(1), preference: preferenceDefinitionSchema }).strict();
3994
+ var headToHeadSchema = z40.object({
3995
+ kind: z40.literal("HEAD_TO_HEAD"),
3996
+ winner: z40.enum(["A", "B", "neither"]),
3997
+ brandA: z40.string(),
3998
+ brandB: z40.string(),
3999
+ rationale: z40.string(),
4000
+ brandAReasoning: z40.string(),
4001
+ brandBReasoning: z40.string(),
4002
+ brandAWhyRaised: z40.string().optional(),
4003
+ brandBWhyRaised: z40.string().optional()
4004
+ });
4005
+ var recommendPickSchema = z40.object({
4006
+ brand: z40.string(),
4007
+ rank: z40.number().int(),
4008
+ adopted: z40.boolean(),
4009
+ reasoning: z40.string().optional(),
4010
+ whyRaised: z40.string().optional()
4011
+ });
4012
+ var recommendSchema = z40.object({
4013
+ kind: z40.literal("RECOMMEND"),
4014
+ picks: z40.array(recommendPickSchema).default([]),
4015
+ singlePick: z40.boolean(),
4016
+ complianceNote: z40.string()
4017
+ });
4018
+ var visibilityVerdictSchema = z40.discriminatedUnion("kind", [
4019
+ headToHeadSchema,
4020
+ recommendSchema
4021
+ ]);
4022
+ function preferenceSelectionLabel(verdict) {
4023
+ if (verdict.kind === "HEAD_TO_HEAD")
4024
+ return verdict.winner === "A" ? verdict.brandA : verdict.winner === "B" ? verdict.brandB : "Neither";
4025
+ if (!verdict.singlePick)
4026
+ return verdict.picks.length ? "No clear selection" : "No selection";
4027
+ return verdict.picks.find((pick) => pick.rank === 1)?.brand ?? "No selection";
4028
+ }
3958
4029
 
3959
4030
  // ../packages/api-schemas/src/runDefinitionInputs.ts
3960
- import { z as z41 } from "zod";
4031
+ import { z as z42 } from "zod";
3961
4032
 
3962
4033
  // ../packages/api-schemas/src/runPlan.ts
3963
- import { z as z40 } from "zod";
3964
- var runPlanCriterionSchema = z40.object({
3965
- id: z40.string().min(1),
3966
- name: z40.string(),
3967
- rubric: z40.string()
4034
+ import { z as z41 } from "zod";
4035
+ var runPlanCriterionSchema = z41.object({
4036
+ id: z41.string().min(1),
4037
+ name: z41.string(),
4038
+ rubric: z41.string()
3968
4039
  });
3969
- var criteriaSnapshotV1Schema = z40.object({
3970
- version: z40.literal(1),
4040
+ var criteriaSnapshotV1Schema = z41.object({
4041
+ version: z41.literal(1),
3971
4042
  // File-authored criteria have no saved EvalSet; their identity is frozen
3972
4043
  // in the request and result rows retain the criterion text without an FK.
3973
- evalSetId: z40.string().min(1).nullable(),
3974
- criteria: z40.array(runPlanCriterionSchema)
3975
- });
3976
- var runPlanCaseSchema = z40.object({
3977
- id: z40.string().min(1),
3978
- prompt: z40.object({ id: z40.string().min(1), text: z40.string() }),
3979
- workspace: z40.object({
3980
- repoUrl: z40.string().nullable(),
3981
- repoRef: z40.string().nullable()
4044
+ evalSetId: z41.string().min(1).nullable(),
4045
+ criteria: z41.array(runPlanCriterionSchema)
4046
+ });
4047
+ var runPlanCaseSchema = z41.object({
4048
+ id: z41.string().min(1),
4049
+ prompt: z41.object({ id: z41.string().min(1), text: z41.string() }),
4050
+ workspace: z41.object({
4051
+ repoUrl: z41.string().nullable(),
4052
+ repoRef: z41.string().nullable()
3982
4053
  }),
3983
- agents: z40.array(
4054
+ agents: z41.array(
3984
4055
  agentConfigSchema.extend({
3985
4056
  // Validated against the current catalog request schema when lowered.
3986
- catalogSelections: z40.record(z40.string(), z40.unknown()).optional()
4057
+ catalogSelections: z41.record(z41.string(), z41.unknown()).optional()
3987
4058
  })
3988
4059
  ),
3989
- samples: z40.number().int().min(1).max(50),
3990
- evalSetId: z40.string().min(1).optional(),
4060
+ samples: z41.number().int().min(1).max(50),
4061
+ evalSetId: z41.string().min(1).optional(),
3991
4062
  criteriaSnapshot: criteriaSnapshotV1Schema.optional(),
3992
- sourceRepoCommit: z40.string().nullable().optional(),
3993
- experiment: z40.string().nullable().optional(),
3994
- membershipTagIds: z40.array(z40.string()).optional(),
3995
- profile: z40.object({
3996
- id: z40.string(),
3997
- name: z40.string(),
3998
- description: z40.string(),
3999
- model: z40.string()
4063
+ preferenceSnapshot: preferenceSnapshotSchema.optional(),
4064
+ sourceRepoCommit: z41.string().nullable().optional(),
4065
+ experiment: z41.string().nullable().optional(),
4066
+ membershipTagIds: z41.array(z41.string()).optional(),
4067
+ profile: z41.object({
4068
+ id: z41.string(),
4069
+ name: z41.string(),
4070
+ description: z41.string(),
4071
+ model: z41.string()
4000
4072
  }).nullable().optional(),
4001
4073
  skillSet: skillSetSchema.optional(),
4002
4074
  mcpSet: mcpSetSchema.optional(),
4003
4075
  connectionSet: connectionSetSchema.optional(),
4004
4076
  // The server's agentContextSnapshotV1Schema validates this at lowering.
4005
- agentContext: z40.unknown().optional(),
4006
- agentContextDigest: z40.string().nullable().optional(),
4077
+ agentContext: z41.unknown().optional(),
4078
+ agentContextDigest: z41.string().nullable().optional(),
4007
4079
  addons: runAddonsSchema.optional(),
4008
4080
  /** Names of resources the coordinator must prepare before expansion. */
4009
- inputs: z40.array(z40.string().min(1)).default([])
4081
+ inputs: z41.array(z41.string().min(1)).default([])
4010
4082
  });
4011
- var inputPathSchema = z40.string().min(1).max(256).regex(
4083
+ var inputPathSchema = z41.string().min(1).max(256).regex(
4012
4084
  // biome-ignore lint/suspicious/noControlCharactersInRegex: Reject unsafe control characters in authored paths.
4013
4085
  /^(?![!/])(?!.*[\\\x00-\x1f\x7f])(?!.*(?:^|\/)\.{1,2}(?:\/|$))(?!.*\/\/)(?!.*\/$).+$/,
4014
4086
  "Expected a repository-relative path or glob without traversal"
@@ -4043,20 +4115,20 @@ var reservedBinaryCommands = [
4043
4115
  "pi",
4044
4116
  "wizard"
4045
4117
  ];
4046
- var binaryCommandSchema = z40.string().max(128).regex(
4118
+ var binaryCommandSchema = z41.string().max(128).regex(
4047
4119
  /^[A-Za-z0-9_][A-Za-z0-9_.+-]*$/,
4048
4120
  "Expected a command name without paths or shell syntax"
4049
4121
  ).refine(
4050
4122
  (name) => !reservedBinaryCommands.includes(name),
4051
4123
  "Command is reserved for Gauge workspace setup or the agent harness; choose another command name"
4052
4124
  );
4053
- var binaryUseSchema = z40.object({
4054
- type: z40.literal("binary"),
4055
- paths: z40.array(inputPathSchema).min(1).max(32),
4056
- platform: z40.literal("linux-x64").describe(
4125
+ var binaryUseSchema = z41.object({
4126
+ type: z41.literal("binary"),
4127
+ paths: z41.array(inputPathSchema).min(1).max(32),
4128
+ platform: z41.literal("linux-x64").describe(
4057
4129
  "Linux x86-64 ELF executables compatible with the consumer sandbox. No system packages are installed."
4058
4130
  ),
4059
- commands: z40.record(
4131
+ commands: z41.record(
4060
4132
  binaryCommandSchema,
4061
4133
  inputPathSchema.regex(
4062
4134
  /^[^*?{}[\]]+$/,
@@ -4067,62 +4139,62 @@ var binaryUseSchema = z40.object({
4067
4139
  "Declare between 1 and 32 commands"
4068
4140
  ).meta({ minProperties: 1, maxProperties: 32 })
4069
4141
  }).strict();
4070
- var inputUseSchema = z40.discriminatedUnion("type", [
4142
+ var inputUseSchema = z41.discriminatedUnion("type", [
4071
4143
  binaryUseSchema,
4072
4144
  websiteUseSchema,
4073
- z40.object({
4074
- type: z40.literal("files"),
4075
- paths: z40.array(inputPathSchema).min(1).max(32).optional()
4145
+ z41.object({
4146
+ type: z41.literal("files"),
4147
+ paths: z41.array(inputPathSchema).min(1).max(32).optional()
4076
4148
  }).strict(),
4077
- z40.object({
4078
- type: z40.literal("npm-cli"),
4149
+ z41.object({
4150
+ type: z41.literal("npm-cli"),
4079
4151
  package: inputPathSchema,
4080
- install: z40.enum(["environment", "workspace"]).optional()
4152
+ install: z41.enum(["environment", "workspace"]).optional()
4081
4153
  }).strict(),
4082
- z40.object({
4083
- type: z40.literal("npm-package"),
4154
+ z41.object({
4155
+ type: z41.literal("npm-package"),
4084
4156
  package: inputPathSchema,
4085
- install: z40.enum(["environment", "workspace"]).optional()
4157
+ install: z41.enum(["environment", "workspace"]).optional()
4086
4158
  }).strict(),
4087
- z40.object({
4088
- type: z40.literal("skill"),
4089
- root: z40.union([z40.literal("."), inputPathSchema]).refine(
4159
+ z41.object({
4160
+ type: z41.literal("skill"),
4161
+ root: z41.union([z41.literal("."), inputPathSchema]).refine(
4090
4162
  (path3) => !/[*?{}[\]]/.test(path3),
4091
4163
  "Skill root must be an explicit directory"
4092
4164
  )
4093
4165
  }).strict()
4094
4166
  ]);
4095
- var inputSourceSchema = z40.discriminatedUnion("type", [
4096
- z40.object({
4097
- type: z40.literal("git"),
4098
- paths: z40.array(inputPathSchema).min(1).max(32)
4167
+ var inputSourceSchema = z41.discriminatedUnion("type", [
4168
+ z41.object({
4169
+ type: z41.literal("git"),
4170
+ paths: z41.array(inputPathSchema).min(1).max(32)
4099
4171
  }).strict(),
4100
- z40.object({
4101
- type: z40.literal("github-actions"),
4172
+ z41.object({
4173
+ type: z41.literal("github-actions"),
4102
4174
  // Omit to discover the sole artifact from the selected workflow run.
4103
4175
  // Names are supplied by CI and need not encode the PR head revision.
4104
- artifact: z40.string().min(1).max(256).optional()
4176
+ artifact: z41.string().min(1).max(256).optional()
4105
4177
  }).strict()
4106
4178
  ]);
4107
- var runPlanResourceSchema = z40.object({
4108
- name: z40.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
4109
- type: z40.literal("input"),
4110
- source: z40.discriminatedUnion("type", [
4179
+ var runPlanResourceSchema = z41.object({
4180
+ name: z41.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
4181
+ type: z41.literal("input"),
4182
+ source: z41.discriminatedUnion("type", [
4111
4183
  commentSourceSchema.extend({
4112
- repoUrl: z40.string(),
4113
- commit: z40.string().regex(/^[a-f0-9]{40}$/i)
4184
+ repoUrl: z41.string(),
4185
+ commit: z41.string().regex(/^[a-f0-9]{40}$/i)
4114
4186
  }).strict(),
4115
4187
  deploymentSourceSchema.extend({
4116
- repoUrl: z40.string(),
4117
- commit: z40.string().regex(/^[a-f0-9]{40}$/i)
4188
+ repoUrl: z41.string(),
4189
+ commit: z41.string().regex(/^[a-f0-9]{40}$/i)
4118
4190
  }).strict(),
4119
4191
  inputSourceSchema.options[0].extend({
4120
- repoUrl: z40.string(),
4121
- commit: z40.string().regex(/^[a-f0-9]{40}$/i)
4192
+ repoUrl: z41.string(),
4193
+ commit: z41.string().regex(/^[a-f0-9]{40}$/i)
4122
4194
  }).strict(),
4123
4195
  inputSourceSchema.options[1].extend({
4124
- repoUrl: z40.string(),
4125
- commit: z40.string().regex(/^[a-f0-9]{40}$/i),
4196
+ repoUrl: z41.string(),
4197
+ commit: z41.string().regex(/^[a-f0-9]{40}$/i),
4126
4198
  workflow: inputPathSchema
4127
4199
  }).strict()
4128
4200
  ]),
@@ -4131,42 +4203,47 @@ var runPlanResourceSchema = z40.object({
4131
4203
  (r) => r.use.type === "website" === (r.source.type === "github-deployment" || r.source.type === "github-pr-comment"),
4132
4204
  "Website inputs require a deployment or PR comment source; file inputs require git or Actions"
4133
4205
  );
4134
- var runPlanSchema = z40.object({
4135
- version: z40.literal(1),
4206
+ var runPlanSchema = z41.object({
4207
+ version: z41.literal(1),
4136
4208
  /** Existing launches keep their current measurement semantics. */
4137
- measurementMode: z40.enum(["EXISTING", "AGENT_EXPERIENCE"]),
4138
- cases: z40.array(runPlanCaseSchema).min(1),
4139
- resources: z40.array(runPlanResourceSchema).default([]),
4209
+ measurementMode: z41.enum([
4210
+ "EXISTING",
4211
+ "AGENT_EXPERIENCE",
4212
+ "AGENT_PREFERENCE",
4213
+ "MIXED"
4214
+ ]),
4215
+ cases: z41.array(runPlanCaseSchema).min(1),
4216
+ resources: z41.array(runPlanResourceSchema).default([]),
4140
4217
  /** Server-populated context retained across deferred preparation. */
4141
- launch: z40.object({
4142
- origin: z40.discriminatedUnion("initiator", [
4143
- z40.object({
4144
- initiator: z40.literal("API"),
4145
- userId: z40.string().min(1),
4146
- email: z40.string().nullable()
4218
+ launch: z41.object({
4219
+ origin: z41.discriminatedUnion("initiator", [
4220
+ z41.object({
4221
+ initiator: z41.literal("API"),
4222
+ userId: z41.string().min(1),
4223
+ email: z41.string().nullable()
4147
4224
  }).strict(),
4148
- z40.object({
4149
- initiator: z40.literal("GITHUB"),
4150
- userId: z40.null(),
4151
- email: z40.null()
4225
+ z41.object({
4226
+ initiator: z41.literal("GITHUB"),
4227
+ userId: z41.null(),
4228
+ email: z41.null()
4152
4229
  }).strict()
4153
4230
  ]),
4154
- funding: z40.enum(["ORG", "PLATFORM"]),
4155
- queueClass: z40.literal("INTERACTIVE")
4231
+ funding: z41.enum(["ORG", "PLATFORM"]),
4232
+ queueClass: z41.literal("INTERACTIVE")
4156
4233
  }).strict().optional(),
4157
- provenance: z40.object({
4158
- definition: z40.object({
4159
- repoUrl: z40.string().min(1),
4160
- commit: z40.string().regex(/^[a-f0-9]{40}$/i),
4161
- digest: z40.string().min(1),
4234
+ provenance: z41.object({
4235
+ definition: z41.object({
4236
+ repoUrl: z41.string().min(1),
4237
+ commit: z41.string().regex(/^[a-f0-9]{40}$/i),
4238
+ digest: z41.string().min(1),
4162
4239
  /** File submissions are asserted by the caller until server-side
4163
4240
  * committed-byte fetching is available. */
4164
- verification: z40.enum(["SUBMITTED_UNVERIFIED", "GITHUB_VERIFIED"]).optional()
4241
+ verification: z41.enum(["SUBMITTED_UNVERIFIED", "GITHUB_VERIFIED"]).optional()
4165
4242
  }).optional(),
4166
- liveDependencies: z40.array(
4167
- z40.object({
4168
- name: z40.string().min(1),
4169
- url: z40.string().url().optional()
4243
+ liveDependencies: z41.array(
4244
+ z41.object({
4245
+ name: z41.string().min(1),
4246
+ url: z41.string().url().optional()
4170
4247
  })
4171
4248
  ).default([])
4172
4249
  })
@@ -4190,17 +4267,23 @@ var runPlanSchema = z40.object({
4190
4267
  commandOwners.set(command, resource.name);
4191
4268
  }
4192
4269
  }
4193
- if (plan.measurementMode === "AGENT_EXPERIENCE" && !item.agents.length)
4270
+ if (plan.measurementMode !== "EXISTING" && !item.agents.length)
4194
4271
  ctx.addIssue({
4195
4272
  code: "custom",
4196
4273
  path: ["cases", index, "agents"],
4197
- message: "Agent Experience cases require a selected agent"
4274
+ message: "File cases require a selected agent"
4198
4275
  });
4199
- if (plan.measurementMode === "AGENT_EXPERIENCE" && !item.criteriaSnapshot?.criteria.length)
4276
+ if (plan.measurementMode !== "EXISTING" && !item.preferenceSnapshot && !item.criteriaSnapshot?.criteria.length)
4200
4277
  ctx.addIssue({
4201
4278
  code: "custom",
4202
4279
  path: ["cases", index, "criteriaSnapshot"],
4203
- message: "Agent Experience cases require criteria"
4280
+ message: "File cases require criteria or preference"
4281
+ });
4282
+ if (item.preferenceSnapshot && (item.criteriaSnapshot || item.evalSetId))
4283
+ ctx.addIssue({
4284
+ code: "custom",
4285
+ path: ["cases", index],
4286
+ message: "Specify exactly one measurement per case"
4204
4287
  });
4205
4288
  if (item.criteriaSnapshot && item.criteriaSnapshot.evalSetId !== (item.evalSetId ?? null))
4206
4289
  ctx.addIssue({
@@ -4232,43 +4315,43 @@ var runPlanSchema = z40.object({
4232
4315
  });
4233
4316
 
4234
4317
  // ../packages/api-schemas/src/runDefinitionInputs.ts
4235
- var definitionInputSourceSchema = z41.discriminatedUnion("type", [
4236
- z41.object({ type: z41.literal("git") }).strict(),
4237
- z41.object({
4238
- type: z41.literal("github-actions"),
4318
+ var definitionInputSourceSchema = z42.discriminatedUnion("type", [
4319
+ z42.object({ type: z42.literal("git") }).strict(),
4320
+ z42.object({
4321
+ type: z42.literal("github-actions"),
4239
4322
  workflow: inputPathSchema.regex(/^\.github\/workflows\/[^/*?{}[\]]+\.ya?ml$/).describe(
4240
4323
  "Workflow file in this repository, for example .github/workflows/build.yml."
4241
4324
  ),
4242
- artifact: z41.string().min(1).max(256).optional().describe(
4325
+ artifact: z42.string().min(1).max(256).optional().describe(
4243
4326
  "Upload name or pattern. Omit only when the workflow produces exactly one artifact."
4244
4327
  )
4245
4328
  }).strict()
4246
4329
  ]);
4247
- var skillInputPathSchema = z41.union([z41.literal("."), inputPathSchema.regex(/^[^*?{}[\]]+$/)]).describe(
4330
+ var skillInputPathSchema = z42.union([z42.literal("."), inputPathSchema.regex(/^[^*?{}[\]]+$/)]).describe(
4248
4331
  "Explicit Skill directory or ZIP/tar.gz file containing SKILL.md at its root."
4249
4332
  );
4250
- var runDefinitionInputSchema = z41.discriminatedUnion("type", [
4333
+ var runDefinitionInputSchema = z42.discriminatedUnion("type", [
4251
4334
  binaryUseSchema.extend({ source: definitionInputSourceSchema }).strict(),
4252
4335
  websiteUseSchema.extend({ source: websiteSourceSchema }).strict(),
4253
- z41.object({
4254
- type: z41.literal("npm-package"),
4336
+ z42.object({
4337
+ type: z42.literal("npm-package"),
4255
4338
  source: definitionInputSourceSchema,
4256
4339
  path: inputPathSchema.describe(
4257
4340
  "Path or glob selecting exactly one packed npm .tgz file."
4258
4341
  ),
4259
- install: z41.enum(["environment", "workspace"]).default("environment").describe(
4342
+ install: z42.enum(["environment", "workspace"]).default("environment").describe(
4260
4343
  "environment exposes the package's commands on PATH; workspace adds a consumer application dependency."
4261
4344
  )
4262
4345
  }).strict(),
4263
- z41.object({
4264
- type: z41.literal("skill"),
4346
+ z42.object({
4347
+ type: z42.literal("skill"),
4265
4348
  source: definitionInputSourceSchema,
4266
4349
  path: skillInputPathSchema
4267
4350
  }).strict(),
4268
- z41.object({
4269
- type: z41.literal("files"),
4351
+ z42.object({
4352
+ type: z42.literal("files"),
4270
4353
  source: definitionInputSourceSchema,
4271
- paths: z41.array(inputPathSchema).min(1).max(32).describe(
4354
+ paths: z42.array(inputPathSchema).min(1).max(32).describe(
4272
4355
  "Files or globs selected from the committed Git tree or the uploaded artifact."
4273
4356
  )
4274
4357
  }).strict()
@@ -4302,7 +4385,7 @@ function compileDefinitionInput(name, input, repoUrl, commit) {
4302
4385
  }
4303
4386
 
4304
4387
  // ../packages/api-schemas/src/runDefinitions.ts
4305
- var uncredentialedRepoUrlSchema = z42.string().min(1).refine((raw) => {
4388
+ var uncredentialedRepoUrlSchema = z43.string().min(1).refine((raw) => {
4306
4389
  if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) return true;
4307
4390
  try {
4308
4391
  const url = new URL(raw);
@@ -4311,54 +4394,67 @@ var uncredentialedRepoUrlSchema = z42.string().min(1).refine((raw) => {
4311
4394
  return false;
4312
4395
  }
4313
4396
  }, "Repository URL must not contain embedded credentials or query parameters");
4314
- var runDefinitionChecksSchema = z42.object({ paths: z42.array(inputPathSchema).min(1).max(32) }).strict();
4315
- var runDefinitionConfigSchema = z42.object({
4316
- agents: z42.array(agentConfigInputSchema).min(1),
4317
- sampleCount: z42.number().int().min(1).max(50).optional(),
4397
+ var runDefinitionChecksSchema = z43.object({ paths: z43.array(inputPathSchema).min(1).max(32) }).strict();
4398
+ var runDefinitionConfigSchema = z43.object({
4399
+ agents: z43.array(agentConfigInputSchema).min(1),
4400
+ sampleCount: z43.number().int().min(1).max(50).optional(),
4318
4401
  repoUrl: uncredentialedRepoUrlSchema.nullable().optional(),
4319
- repoRef: z42.string().min(1).nullable().optional(),
4320
- profileId: z42.string().min(1).nullable().optional(),
4402
+ repoRef: z43.string().min(1).nullable().optional(),
4403
+ profileId: z43.string().min(1).nullable().optional(),
4321
4404
  skillRefs: assetRefListSchema.optional(),
4322
4405
  mcpRefs: assetRefListSchema.optional(),
4323
- connectionIds: z42.array(z42.string().min(1)).max(32).optional(),
4406
+ connectionIds: z43.array(z43.string().min(1)).max(32).optional(),
4324
4407
  addons: runAddonsSchema.optional(),
4325
- inputs: z42.array(z42.string()).optional(),
4408
+ inputs: z43.array(z43.string()).optional(),
4326
4409
  checks: runDefinitionChecksSchema.optional()
4327
4410
  }).strict();
4328
- var runDefinitionFileConfigSchema = z42.object({
4329
- version: z42.literal(2).default(2),
4330
- org: z42.string().min(1).optional(),
4411
+ var runDefinitionFileConfigSchema = z43.object({
4412
+ version: z43.literal(2).default(2),
4413
+ org: z43.string().min(1).optional(),
4331
4414
  checks: runDefinitionChecksSchema.optional(),
4332
- cases: z42.array(inputPathSchema).min(1).max(32).default(["gauge-evals/**/*.md"]),
4333
- inputs: z42.record(
4334
- z42.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
4415
+ cases: z43.array(inputPathSchema).min(1).max(32).default(["gauge-evals/**/*.md"]),
4416
+ inputs: z43.record(
4417
+ z43.string().regex(/^[a-z0-9]+(?:-[a-z0-9]+)*$/).max(64),
4335
4418
  runDefinitionInputSchema
4336
4419
  ).refine(
4337
4420
  (inputs) => Object.keys(inputs).length <= 32,
4338
4421
  "At most 32 inputs are allowed"
4339
4422
  ).meta({ maxProperties: 32 }).default({})
4340
4423
  }).strict();
4341
- var runDefinitionCriterionSchema = z42.object({ name: z42.string().min(1), rubric: z42.string().min(1) }).strict();
4342
- var runDefinitionFrontmatterSchema = z42.object({
4343
- criteria: z42.array(runDefinitionCriterionSchema).min(1),
4424
+ var runDefinitionCriterionSchema = z43.object({ name: z43.string().min(1), rubric: z43.string().min(1) }).strict();
4425
+ var caseMeasurement = {
4426
+ criteria: z43.array(runDefinitionCriterionSchema).min(1).optional(),
4427
+ preference: preferenceDefinitionSchema.optional()
4428
+ };
4429
+ function exactlyOneMeasurement(item) {
4430
+ return item.criteria !== void 0 !== (item.preference !== void 0);
4431
+ }
4432
+ var runDefinitionFrontmatterSchema = z43.object({
4433
+ ...caseMeasurement,
4344
4434
  config: runDefinitionConfigSchema
4345
- }).strict();
4346
- var runDefinitionCaseSchema = z42.object({
4347
- path: z42.string().min(1),
4348
- prompt: z42.string().min(1),
4349
- criteria: z42.array(runDefinitionCriterionSchema).min(1),
4435
+ }).strict().refine(
4436
+ exactlyOneMeasurement,
4437
+ "Specify exactly one of criteria or preference"
4438
+ );
4439
+ var runDefinitionCaseSchema = z43.object({
4440
+ path: z43.string().min(1),
4441
+ prompt: z43.string().min(1),
4442
+ ...caseMeasurement,
4350
4443
  config: runDefinitionConfigSchema
4351
- }).strict();
4352
- var runDefinitionSubmissionSchema = z42.object({
4353
- version: z42.literal(2),
4354
- source: z42.object({
4444
+ }).strict().refine(
4445
+ exactlyOneMeasurement,
4446
+ "Specify exactly one of criteria or preference"
4447
+ );
4448
+ var runDefinitionSubmissionSchema = z43.object({
4449
+ version: z43.literal(2),
4450
+ source: z43.object({
4355
4451
  repoUrl: uncredentialedRepoUrlSchema,
4356
- commit: z42.string().regex(/^[a-f0-9]{40}$/i),
4357
- digest: z42.string().regex(/^sha256:[a-f0-9]{64}$/),
4358
- verification: z42.enum(["SUBMITTED_UNVERIFIED", "GITHUB_VERIFIED"])
4452
+ commit: z43.string().regex(/^[a-f0-9]{40}$/i),
4453
+ digest: z43.string().regex(/^sha256:[a-f0-9]{64}$/),
4454
+ verification: z43.enum(["SUBMITTED_UNVERIFIED", "GITHUB_VERIFIED"])
4359
4455
  }).strict(),
4360
- resources: z42.array(runPlanResourceSchema).max(32).default([]),
4361
- cases: z42.array(runDefinitionCaseSchema).min(1).max(20)
4456
+ resources: z43.array(runPlanResourceSchema).max(32).default([]),
4457
+ cases: z43.array(runDefinitionCaseSchema).min(1).max(20)
4362
4458
  }).strict().superRefine((definition, ctx) => {
4363
4459
  const paths = /* @__PURE__ */ new Set();
4364
4460
  for (const [index, item] of definition.cases.entries()) {
@@ -4401,54 +4497,55 @@ var runDefinitionPublicSubmissionSchema = runDefinitionSubmissionSchema.refine(
4401
4497
  message: "Only the GitHub App may verify committed bytes"
4402
4498
  }
4403
4499
  );
4404
- var runDefinitionPlanResponseSchema = z42.object({
4500
+ var runDefinitionPlanResponseSchema = z43.object({
4405
4501
  source: runDefinitionSubmissionSchema.shape.source,
4406
- measurementMode: z42.literal("AGENT_EXPERIENCE"),
4407
- cases: z42.array(
4408
- z42.object({
4409
- path: z42.string(),
4410
- agents: z42.array(agentConfigSchema),
4411
- samples: z42.number().int().nonnegative(),
4412
- expectedRuns: z42.number().int().nonnegative(),
4413
- workspace: z42.object({
4414
- repoUrl: z42.string().nullable(),
4415
- repoRef: z42.string().nullable()
4502
+ measurementMode: z43.enum(["AGENT_EXPERIENCE", "AGENT_PREFERENCE", "MIXED"]),
4503
+ cases: z43.array(
4504
+ z43.object({
4505
+ path: z43.string(),
4506
+ agents: z43.array(agentConfigSchema),
4507
+ samples: z43.number().int().nonnegative(),
4508
+ expectedRuns: z43.number().int().nonnegative(),
4509
+ workspace: z43.object({
4510
+ repoUrl: z43.string().nullable(),
4511
+ repoRef: z43.string().nullable()
4416
4512
  }),
4417
- criteriaCount: z42.number().int().nonnegative(),
4418
- inputs: z42.array(z42.string())
4513
+ criteriaCount: z43.number().int().nonnegative(),
4514
+ preference: preferenceDefinitionSchema.optional(),
4515
+ inputs: z43.array(z43.string())
4419
4516
  })
4420
4517
  ),
4421
- resources: z42.array(
4422
- z42.object({
4423
- name: z42.string(),
4424
- type: z42.literal("input"),
4425
- source: z42.enum([
4518
+ resources: z43.array(
4519
+ z43.object({
4520
+ name: z43.string(),
4521
+ type: z43.literal("input"),
4522
+ source: z43.enum([
4426
4523
  "git",
4427
4524
  "github-actions",
4428
4525
  "github-deployment",
4429
4526
  "github-pr-comment"
4430
4527
  ]),
4431
4528
  use: inputUseSchema,
4432
- readiness: z42.enum(["READY", "WAITING", "FAILED"]),
4433
- detail: z42.string().optional(),
4434
- commands: z42.array(z42.string()).optional()
4529
+ readiness: z43.enum(["READY", "WAITING", "FAILED"]),
4530
+ detail: z43.string().optional(),
4531
+ commands: z43.array(z43.string()).optional()
4435
4532
  })
4436
4533
  ),
4437
- expectedRuns: z42.number().int().nonnegative(),
4438
- canRun: z42.boolean()
4534
+ expectedRuns: z43.number().int().nonnegative(),
4535
+ canRun: z43.boolean()
4439
4536
  });
4440
- var runDefinitionRunResponseSchema = z42.object({
4441
- runRequestId: z42.string(),
4442
- batchIds: z42.array(z42.string()),
4443
- runIds: z42.array(z42.string()),
4444
- requestedRuns: z42.number().int().nonnegative(),
4445
- createdRuns: z42.number().int().nonnegative(),
4446
- aggregateUrl: z42.string()
4537
+ var runDefinitionRunResponseSchema = z43.object({
4538
+ runRequestId: z43.string(),
4539
+ batchIds: z43.array(z43.string()),
4540
+ runIds: z43.array(z43.string()),
4541
+ requestedRuns: z43.number().int().nonnegative(),
4542
+ createdRuns: z43.number().int().nonnegative(),
4543
+ aggregateUrl: z43.string()
4447
4544
  });
4448
4545
 
4449
4546
  // ../packages/api-schemas/src/runRequests.ts
4450
- import { z as z43 } from "zod";
4451
- var runRequestStatusSchema = z43.enum([
4547
+ import { z as z44 } from "zod";
4548
+ var runRequestStatusSchema = z44.enum([
4452
4549
  "PENDING",
4453
4550
  "PREPARING",
4454
4551
  "RUNNING",
@@ -4456,9 +4553,10 @@ var runRequestStatusSchema = z43.enum([
4456
4553
  "FAILED",
4457
4554
  "CANCELED"
4458
4555
  ]);
4459
- var runRequestResultSchema = z43.enum([
4556
+ var runRequestResultSchema = z44.enum([
4460
4557
  "PENDING",
4461
4558
  "PASSED",
4559
+ "MEASURED",
4462
4560
  "QUALITY_FAILED",
4463
4561
  "EXECUTION_FAILED",
4464
4562
  "INPUT_FAILED",
@@ -4466,224 +4564,229 @@ var runRequestResultSchema = z43.enum([
4466
4564
  "INCOMPLETE",
4467
4565
  "CANCELED"
4468
4566
  ]);
4469
- var runRequestCriterionResultSchema = z43.object({
4470
- criterionId: z43.string().nullable(),
4471
- name: z43.string(),
4472
- passed: z43.boolean(),
4473
- reasoning: z43.string().nullable()
4474
- });
4475
- var runRequestRunSchema = z43.object({
4476
- id: z43.string(),
4477
- batchId: z43.string(),
4478
- status: z43.string(),
4567
+ var runRequestCriterionResultSchema = z44.object({
4568
+ criterionId: z44.string().nullable(),
4569
+ name: z44.string(),
4570
+ passed: z44.boolean(),
4571
+ reasoning: z44.string().nullable()
4572
+ });
4573
+ var runRequestRunSchema = z44.object({
4574
+ id: z44.string(),
4575
+ batchId: z44.string(),
4576
+ status: z44.string(),
4479
4577
  agent: agentSchema,
4480
- model: z43.string(),
4578
+ model: z44.string(),
4481
4579
  result: runRequestResultSchema,
4482
- judge: z43.object({ status: z43.string() }).nullable(),
4483
- criterionResults: z43.array(runRequestCriterionResultSchema)
4580
+ judge: z44.object({ status: z44.string() }).nullable(),
4581
+ criterionResults: z44.array(runRequestCriterionResultSchema),
4582
+ preference: visibilityVerdictSchema.nullable().optional(),
4583
+ measurement: z44.enum(["experience", "preference"]).optional()
4484
4584
  });
4485
- var runRequestResponseSchema = z43.object({
4486
- id: z43.string(),
4585
+ var runRequestResponseSchema = z44.object({
4586
+ id: z44.string(),
4487
4587
  status: runRequestStatusSchema,
4488
4588
  result: runRequestResultSchema,
4489
- error: z43.string().nullable().optional(),
4490
- inputs: z43.unknown().optional(),
4491
- settled: z43.boolean(),
4492
- counts: z43.object({
4493
- requestedRuns: z43.number().int().nonnegative(),
4494
- createdRuns: z43.number().int().nonnegative(),
4495
- terminalRuns: z43.number().int().nonnegative(),
4496
- judgedRuns: z43.number().int().nonnegative(),
4497
- passedRuns: z43.number().int().nonnegative(),
4498
- qualityFailedRuns: z43.number().int().nonnegative(),
4499
- executionFailedRuns: z43.number().int().nonnegative(),
4500
- judgeFailedRuns: z43.number().int().nonnegative(),
4501
- pendingRuns: z43.number().int().nonnegative()
4589
+ error: z44.string().nullable().optional(),
4590
+ inputs: z44.unknown().optional(),
4591
+ settled: z44.boolean(),
4592
+ counts: z44.object({
4593
+ requestedRuns: z44.number().int().nonnegative(),
4594
+ createdRuns: z44.number().int().nonnegative(),
4595
+ terminalRuns: z44.number().int().nonnegative(),
4596
+ judgedRuns: z44.number().int().nonnegative(),
4597
+ passedRuns: z44.number().int().nonnegative(),
4598
+ measuredRuns: z44.number().int().nonnegative().optional(),
4599
+ qualityFailedRuns: z44.number().int().nonnegative(),
4600
+ executionFailedRuns: z44.number().int().nonnegative(),
4601
+ judgeFailedRuns: z44.number().int().nonnegative(),
4602
+ pendingRuns: z44.number().int().nonnegative()
4502
4603
  }),
4503
- batches: z43.array(
4504
- z43.object({ id: z43.string(), runs: z43.array(runRequestRunSchema) })
4604
+ batches: z44.array(
4605
+ z44.object({ id: z44.string(), runs: z44.array(runRequestRunSchema) })
4505
4606
  ),
4506
- createdAt: z43.string(),
4507
- updatedAt: z43.string(),
4508
- finishedAt: z43.string().nullable()
4607
+ createdAt: z44.string(),
4608
+ updatedAt: z44.string(),
4609
+ finishedAt: z44.string().nullable()
4509
4610
  });
4510
4611
 
4511
4612
  // ../packages/api-schemas/src/summary.ts
4512
- import { z as z44 } from "zod";
4613
+ import { z as z45 } from "zod";
4513
4614
  function cardResultSchema(data) {
4514
- return z44.union([
4515
- z44.object({ ok: z44.literal(true), data }),
4516
- z44.object({ ok: z44.literal(false), error: z44.string() })
4615
+ return z45.union([
4616
+ z45.object({ ok: z45.literal(true), data }),
4617
+ z45.object({ ok: z45.literal(false), error: z45.string() })
4517
4618
  ]);
4518
4619
  }
4519
- var summaryActionSchema = z44.object({
4520
- id: z44.string(),
4521
- title: z44.string(),
4522
- priority: z44.string(),
4523
- status: z44.string(),
4524
- evidenceCount: z44.number().int()
4620
+ var summaryActionSchema = z45.object({
4621
+ id: z45.string(),
4622
+ title: z45.string(),
4623
+ priority: z45.string(),
4624
+ status: z45.string(),
4625
+ evidenceCount: z45.number().int()
4525
4626
  });
4526
- var summaryAeCardSchema = z44.object({
4527
- id: z44.string(),
4528
- name: z44.string(),
4529
- judgedRuns: z44.number().int(),
4530
- passRate: z44.number().nullable(),
4627
+ var summaryAeCardSchema = z45.object({
4628
+ id: z45.string(),
4629
+ name: z45.string(),
4630
+ judgedRuns: z45.number().int(),
4631
+ passRate: z45.number().nullable(),
4531
4632
  /** pp delta vs the prior 30d; null with no prior. */
4532
- deltaPP: z44.number().nullable()
4633
+ deltaPP: z45.number().nullable()
4533
4634
  });
4534
- var summaryVisibilityCardSchema = z44.object({
4535
- name: z44.string(),
4536
- owned: z44.object({
4537
- name: z44.string(),
4538
- rank: z44.number(),
4539
- chosenRate: z44.number(),
4540
- mentionRate: z44.number()
4635
+ var summaryVisibilityCardSchema = z45.object({
4636
+ name: z45.string(),
4637
+ owned: z45.object({
4638
+ name: z45.string(),
4639
+ rank: z45.number(),
4640
+ chosenRate: z45.number(),
4641
+ mentionRate: z45.number()
4541
4642
  }).nullable(),
4542
- chosenDeltaPP: z44.number().nullable()
4643
+ chosenDeltaPP: z45.number().nullable()
4543
4644
  });
4544
- var homeSummarySchema = z44.object({
4545
- hasAnyRuns: z44.boolean(),
4546
- lastSynthesisAt: z44.string().nullable(),
4547
- actions: cardResultSchema(z44.array(summaryActionSchema)),
4645
+ var homeSummarySchema = z45.object({
4646
+ hasAnyRuns: z45.boolean(),
4647
+ lastSynthesisAt: z45.string().nullable(),
4648
+ actions: cardResultSchema(z45.array(summaryActionSchema)),
4548
4649
  visibility: cardResultSchema(
4549
- z44.object({
4550
- cards: z44.array(summaryVisibilityCardSchema),
4551
- marketsTotal: z44.number().int(),
4552
- promptCount: z44.number().int()
4650
+ z45.object({
4651
+ cards: z45.array(summaryVisibilityCardSchema),
4652
+ marketsTotal: z45.number().int(),
4653
+ promptCount: z45.number().int()
4553
4654
  })
4554
4655
  ),
4555
4656
  agentExperience: cardResultSchema(
4556
- z44.object({
4557
- cards: z44.array(summaryAeCardSchema),
4558
- total: z44.number().int(),
4559
- setCount: z44.number().int()
4657
+ z45.object({
4658
+ cards: z45.array(summaryAeCardSchema),
4659
+ total: z45.number().int(),
4660
+ setCount: z45.number().int()
4560
4661
  })
4561
4662
  ),
4562
- notableSessions: cardResultSchema(z44.array(z44.unknown()))
4663
+ notableSessions: cardResultSchema(z45.array(z45.unknown()))
4563
4664
  });
4564
4665
 
4565
4666
  // ../packages/api-schemas/src/tokens.ts
4566
- import { z as z45 } from "zod";
4567
- var apiTokenSchema = z45.object({
4568
- id: z45.string(),
4569
- name: z45.string(),
4667
+ import { z as z46 } from "zod";
4668
+ var apiTokenSchema = z46.object({
4669
+ id: z46.string(),
4670
+ name: z46.string(),
4570
4671
  /** Last 4 chars of the plaintext, for recognizability (non-secret). */
4571
- last4: z45.string(),
4572
- expiresAt: z45.iso.datetime().nullable(),
4573
- lastUsedAt: z45.iso.datetime().nullable(),
4574
- createdAt: z45.iso.datetime(),
4575
- revokedAt: z45.iso.datetime().nullable()
4576
- });
4577
- var createTokenRequestSchema = z45.object({
4578
- name: z45.string().min(1).max(100),
4672
+ last4: z46.string(),
4673
+ expiresAt: z46.iso.datetime().nullable(),
4674
+ lastUsedAt: z46.iso.datetime().nullable(),
4675
+ createdAt: z46.iso.datetime(),
4676
+ revokedAt: z46.iso.datetime().nullable()
4677
+ });
4678
+ var createTokenRequestSchema = z46.object({
4679
+ name: z46.string().min(1).max(100),
4579
4680
  /** ISO-8601 datetime; omit for no expiry. */
4580
- expiresAt: z45.iso.datetime({ offset: true }).optional()
4681
+ expiresAt: z46.iso.datetime({ offset: true }).optional()
4581
4682
  });
4582
4683
  var createTokenResponseSchema = apiTokenSchema.extend({
4583
- token: z45.string()
4684
+ token: z46.string()
4584
4685
  });
4585
4686
  var listTokensResponseSchema = listResponseSchema(apiTokenSchema);
4586
4687
 
4587
4688
  // ../packages/api-schemas/src/visibility.ts
4588
- import { z as z46 } from "zod";
4689
+ import { z as z47 } from "zod";
4589
4690
  var VISIBILITY_PROMPT_KINDS = ["HEAD_TO_HEAD", "RECOMMEND"];
4590
- var visibilityPromptKindSchema = z46.enum(VISIBILITY_PROMPT_KINDS);
4691
+ var visibilityPromptKindSchema = z47.enum(VISIBILITY_PROMPT_KINDS);
4591
4692
  var visibilityPromptSchema = visibilityPromptSettingsSchema.extend({
4592
- id: z46.string(),
4593
- name: z46.string(),
4693
+ id: z47.string(),
4694
+ name: z47.string(),
4594
4695
  kind: visibilityPromptKindSchema,
4595
- promptText: z46.string(),
4696
+ promptText: z47.string(),
4596
4697
  /** HEAD_TO_HEAD is always branded; RECOMMEND may be either. */
4597
- branded: z46.boolean(),
4598
- brandA: z46.string().nullable(),
4599
- brandB: z46.string().nullable(),
4600
- market: z46.object({ id: z46.string(), slug: z46.string(), name: z46.string() }).nullable(),
4601
- position: z46.number().int(),
4602
- createdAt: z46.string(),
4698
+ branded: z47.boolean(),
4699
+ brandA: z47.string().nullable(),
4700
+ brandB: z47.string().nullable(),
4701
+ market: z47.object({ id: z47.string(), slug: z47.string(), name: z47.string() }).nullable(),
4702
+ position: z47.number().int(),
4703
+ createdAt: z47.string(),
4603
4704
  /** User tag names on the hidden backing prompt (Surface:* excluded). */
4604
- tags: z46.array(z46.string()),
4605
- runCount: z46.number().int(),
4606
- lastRunAt: z46.string().nullable(),
4607
- scenarios: z46.array(visibilityScenarioSchema)
4705
+ tags: z47.array(z47.string()),
4706
+ runCount: z47.number().int(),
4707
+ lastRunAt: z47.string().nullable(),
4708
+ scenarios: z47.array(visibilityScenarioSchema)
4608
4709
  });
4609
- var listVisibilityPromptsResponseSchema = z46.object({
4610
- items: z46.array(visibilityPromptSchema)
4710
+ var listVisibilityPromptsResponseSchema = z47.object({
4711
+ items: z47.array(visibilityPromptSchema)
4611
4712
  });
4612
- var createVisibilityPromptBodySchema = z46.object({
4613
- name: z46.string().min(1),
4614
- promptText: z46.string().min(1),
4713
+ var createVisibilityPromptBodySchema = z47.object({
4714
+ name: z47.string().min(1),
4715
+ promptText: z47.string().min(1),
4615
4716
  kind: visibilityPromptKindSchema,
4616
- branded: z46.boolean().optional(),
4717
+ branded: z47.boolean().optional(),
4617
4718
  /** Required pair for HEAD_TO_HEAD (order-independent). */
4618
- brandA: z46.string().nullish(),
4619
- brandB: z46.string().nullish(),
4719
+ brandA: z47.string().nullish(),
4720
+ brandB: z47.string().nullish(),
4620
4721
  /** Measured market id from the org's tracked set. */
4621
- marketId: z46.string().nullish(),
4622
- tags: z46.array(z46.string()).optional(),
4722
+ marketId: z47.string().nullish(),
4723
+ tags: z47.array(z47.string()).optional(),
4623
4724
  agents: visibilityPromptSettingsInputShape.agents.optional(),
4624
4725
  skillRefs: visibilityPromptSettingsInputShape.skillRefs,
4625
4726
  mcpRefs: visibilityPromptSettingsInputShape.mcpRefs,
4626
4727
  connectionIds: visibilityPromptSettingsInputShape.connectionIds,
4627
4728
  cadence: visibilityPromptSettingsInputShape.cadence,
4628
4729
  sampleCount: visibilityPromptSettingsInputShape.sampleCount,
4629
- scenarios: z46.array(visibilityScenarioInputSchema).optional().default([])
4730
+ scenarios: z47.array(visibilityScenarioInputSchema).optional().default([])
4630
4731
  }).strict();
4631
- var synthesizeResponseSchema = z46.object({
4732
+ var synthesizeResponseSchema = z47.object({
4632
4733
  /** False when the pre-check found no new evidence and nothing was queued. */
4633
- queued: z46.boolean()
4634
- });
4635
- var updateVisibilityPromptBodySchema = z46.object({
4636
- name: z46.string().min(1).optional(),
4637
- promptText: z46.string().min(1).optional(),
4638
- branded: z46.boolean().optional(),
4639
- brandA: z46.string().nullish(),
4640
- brandB: z46.string().nullish(),
4641
- marketId: z46.string().nullish(),
4642
- tags: z46.array(z46.string()).optional(),
4734
+ queued: z47.boolean()
4735
+ });
4736
+ var updateVisibilityPromptBodySchema = z47.object({
4737
+ name: z47.string().min(1).optional(),
4738
+ promptText: z47.string().min(1).optional(),
4739
+ branded: z47.boolean().optional(),
4740
+ brandA: z47.string().nullish(),
4741
+ brandB: z47.string().nullish(),
4742
+ marketId: z47.string().nullish(),
4743
+ tags: z47.array(z47.string()).optional(),
4643
4744
  ...visibilityPromptSettingsPatchShape,
4644
4745
  /** Replaces the whole scenario set when present. */
4645
- scenarios: z46.array(visibilityScenarioInputSchema).optional()
4746
+ scenarios: z47.array(visibilityScenarioInputSchema).optional()
4646
4747
  }).strict();
4647
- var runVisibilityPromptResponseSchema = z46.object({
4648
- batchIds: z46.array(z46.string()),
4649
- runIds: z46.array(z46.string())
4748
+ var runVisibilityPromptResponseSchema = z47.object({
4749
+ batchIds: z47.array(z47.string()),
4750
+ runIds: z47.array(z47.string())
4650
4751
  });
4651
- var visibilityPromptRunSchema = z46.object({
4652
- id: z46.string(),
4653
- status: z46.string(),
4752
+ var visibilityPromptRunSchema = z47.object({
4753
+ id: z47.string(),
4754
+ status: z47.string(),
4654
4755
  agent: agentSchema,
4655
- model: z46.string().nullable(),
4656
- resolvedModel: z46.string().nullable(),
4657
- createdAt: z46.string(),
4658
- startedAt: z46.string().nullable(),
4659
- finishedAt: z46.string().nullable(),
4660
- failureReason: z46.string().nullable(),
4661
- verdict: z46.object({
4662
- status: z46.string(),
4663
- error: z46.string().nullable(),
4664
- payload: z46.unknown().nullable()
4756
+ model: z47.string().nullable(),
4757
+ resolvedModel: z47.string().nullable(),
4758
+ createdAt: z47.string(),
4759
+ startedAt: z47.string().nullable(),
4760
+ finishedAt: z47.string().nullable(),
4761
+ failureReason: z47.string().nullable(),
4762
+ creditsCharged: runCreditsChargedSchema.optional(),
4763
+ triggeredBy: runTriggeredBySchema.optional(),
4764
+ verdict: z47.object({
4765
+ status: z47.string(),
4766
+ error: z47.string().nullable(),
4767
+ payload: z47.unknown().nullable()
4665
4768
  }).nullable()
4666
4769
  });
4667
- var listVisibilityPromptRunsResponseSchema = z46.object({
4668
- items: z46.array(visibilityPromptRunSchema)
4669
- });
4670
- var orgMarketSchema = z46.object({
4671
- marketId: z46.string(),
4672
- slug: z46.string(),
4673
- name: z46.string(),
4770
+ var listVisibilityPromptRunsResponseSchema = listResponseSchema(
4771
+ visibilityPromptRunSchema
4772
+ );
4773
+ var orgMarketSchema = z47.object({
4774
+ marketId: z47.string(),
4775
+ slug: z47.string(),
4776
+ name: z47.string(),
4674
4777
  /** The one line saying what the market measures. */
4675
- capability: z46.string(),
4778
+ capability: z47.string(),
4676
4779
  /** SUGGESTED (the matcher) | USER (a human picked it). */
4677
- source: z46.string()
4780
+ source: z47.string()
4678
4781
  });
4679
4782
  var addableMarketSchema = orgMarketSchema.omit({ source: true });
4680
- var listMarketsResponseSchema = z46.object({
4681
- tracked: z46.array(orgMarketSchema),
4783
+ var listMarketsResponseSchema = z47.object({
4784
+ tracked: z47.array(orgMarketSchema),
4682
4785
  /** Published markets the org does not track yet (closed set — no free-text
4683
4786
  * market creation). */
4684
- addable: z46.array(addableMarketSchema)
4787
+ addable: z47.array(addableMarketSchema)
4685
4788
  });
4686
- var addMarketBodySchema = z46.object({ marketId: z46.string().min(1) });
4789
+ var addMarketBodySchema = z47.object({ marketId: z47.string().min(1) });
4687
4790
 
4688
4791
  // src/version.ts
4689
4792
  import { createRequire } from "module";
@@ -5068,7 +5171,7 @@ __export(apply_exports, {
5068
5171
  register: () => register2
5069
5172
  });
5070
5173
  import { readFileSync as readFileSync3 } from "fs";
5071
- import { z as z47 } from "zod";
5174
+ import { z as z48 } from "zod";
5072
5175
  function assertRequiredExperiment(spec, required) {
5073
5176
  if (required.length === 0) {
5074
5177
  throw new CliError(
@@ -5220,7 +5323,7 @@ function register2(program2) {
5220
5323
  ).option("--schema", "print the battery spec JSON Schema and exit").option("--example", "print a minimal example battery spec and exit").action(
5221
5324
  async (opts, command) => {
5222
5325
  if (opts.schema) {
5223
- printJson(z47.toJSONSchema(batteries_exports.batterySpecSchema));
5326
+ printJson(z48.toJSONSchema(batteries_exports.batterySpecSchema));
5224
5327
  return;
5225
5328
  }
5226
5329
  if (opts.example) {
@@ -7392,7 +7495,7 @@ function register12(program2) {
7392
7495
  console.log(`Expected sessions: ${result.expectedRuns}`);
7393
7496
  for (const item of result.cases)
7394
7497
  console.log(
7395
- ` ${item.path}: ${item.expectedRuns} sessions, ${item.criteriaCount} criteria`
7498
+ ` ${item.path}: ${item.expectedRuns} sessions, ${item.preference ? `${item.preference.kind} preference (advisory)` : `${item.criteriaCount} criteria`}`
7396
7499
  );
7397
7500
  for (const resource of result.resources) {
7398
7501
  const use = resource.use;
@@ -7594,29 +7697,38 @@ function register12(program2) {
7594
7697
  );
7595
7698
  }
7596
7699
  );
7597
- group.command("runs <id>").description("The eval set's run log with per-criterion verdicts").option("--limit <n>", "max runs (default 100)").action(async (id, opts, cmd) => {
7598
- const org = resolveOrg(cmd);
7599
- const res = await createClient().get(
7600
- `${evalSetsPath(org)}/${encodeURIComponent(id)}/runs`,
7601
- { limit: opts.limit }
7602
- );
7603
- if (resolveOutput(cmd) === "json") {
7604
- printJson(res.items);
7605
- return;
7700
+ group.command("runs <id>").description("The eval set's run log with per-criterion verdicts").option("--limit <n>", "page size (1-500, default 100)").option("--cursor <cursor>", "resume from a previous page's nextCursor").action(
7701
+ async (id, opts, cmd) => {
7702
+ const org = resolveOrg(cmd);
7703
+ const res = await createClient().get(
7704
+ `${evalSetsPath(org)}/${encodeURIComponent(id)}/runs`,
7705
+ { limit: opts.limit, cursor: opts.cursor }
7706
+ );
7707
+ if (resolveOutput(cmd) === "json") {
7708
+ printJson(res);
7709
+ return;
7710
+ }
7711
+ printItems(
7712
+ res.items.map((r) => ({
7713
+ id: r.id,
7714
+ status: r.status,
7715
+ agent: r.agent,
7716
+ model: r.resolvedModel ?? r.model ?? "",
7717
+ source: r.triggeredBy?.source ?? "",
7718
+ credits: r.creditsCharged ?? "",
7719
+ created: r.createdAt,
7720
+ judge: r.judge?.status ?? "-",
7721
+ verdicts: r.criterionResults.length === 0 ? "-" : `${r.criterionResults.filter((c) => c.passed).length}/${r.criterionResults.length} passed`
7722
+ })),
7723
+ "table"
7724
+ );
7725
+ if (res.nextCursor) {
7726
+ console.error(
7727
+ `(more results \u2014 rerun with --cursor ${res.nextCursor})`
7728
+ );
7729
+ }
7606
7730
  }
7607
- printItems(
7608
- res.items.map((r) => ({
7609
- id: r.id,
7610
- status: r.status,
7611
- agent: r.agent,
7612
- model: r.resolvedModel ?? r.model ?? "",
7613
- created: r.createdAt,
7614
- judge: r.judge?.status ?? "-",
7615
- verdicts: r.criterionResults.length === 0 ? "-" : `${r.criterionResults.filter((c) => c.passed).length}/${r.criterionResults.length} passed`
7616
- })),
7617
- "table"
7618
- );
7619
- });
7731
+ );
7620
7732
  group.command("tag <id>").description("Replace an eval set's user tags (Surface:* is preserved)").option(
7621
7733
  "--tag <name>",
7622
7734
  "tag name, create-on-type (repeatable)",
@@ -7681,6 +7793,10 @@ Choose a roster for the question and budget. For exploratory testing, start with
7681
7793
 
7682
7794
  Background: [Open models as preference proxies](https://www.withgauge.com/blog/open-models-useful-proxies-data-from-1000-sessions/).
7683
7795
 
7796
+ ## Git-authored Agent Preference
7797
+ Replace \`criteria\` with \`preference: { kind: open-ended }\`, adding \`branded: true\` when the prompt names brands. For a comparison use \`preference: { kind: head-to-head, brandA: PostHog, brandB: Amplitude }\`. Specify exactly one of criteria/preference; the body remains the task and must name the comparison brands itself.
7798
+ Existing per-case settings, inputs, and path filters apply. The app's preference analyst reports advisory choices; competitor selections never fail criteria. Preference-only checks finish neutral; mixed suites retain experience gating. The wait command succeeds for MEASURED/PASSED and fails on operational errors. No base/head delta is inferred.
7799
+
7684
7800
  ## Migration prompt and criteria
7685
7801
 
7686
7802
  This example tests migration of an existing feature-flag application across browser and server workflows. Its starting repository contains the application, while attached connections supply destination credentials. The source account's targeting configuration is unavailable.
@@ -7728,7 +7844,7 @@ Explain product choices in their task context, then identify changes worth testi
7728
7844
 
7729
7845
  ## Establish the measurement
7730
7846
 
7731
- Define the question, product role, cohort, and time window. Record prompts, repository revisions, personas, model/harness targets, attached resources, and interventions so comparisons remain interpretable. Open-ended choices and forced head-to-head comparisons answer different questions.
7847
+ Define the question, product role, cohort, and time window. Record prompts, repository revisions, personas, model/harness targets, attached resources, and interventions so comparisons remain interpretable. Open-ended choices and forced head-to-head comparisons answer different questions. Git-authored Markdown cases replace criteria with a preference block: kind open-ended (optionally branded: true), or kind head-to-head with brandA and brandB. The body is the prompt. Use gauge evals verify/plan/run; PR selections are advisory and use the app analyst.
7732
7848
 
7733
7849
  Use \`gauge preference list -o json\` and \`gauge preference get <id> -o json\` to orient, \`gauge runs list --prompt <id> -o json\` for sessions, and \`gauge runs insight <run-id> -o json\` plus \`gauge runs logs <run-id> -o json\` for judgments and trace evidence. Follow \`nextCursor\` across list pages; one page is not the whole cohort. Inspect command help for available filters and configuration.
7734
7850
 
@@ -9886,12 +10002,19 @@ function register25(program2) {
9886
10002
  if (result.settled) {
9887
10003
  if (resolveOutput(cmd) === "json") printJson(result);
9888
10004
  else {
10005
+ const measured = result.counts.measuredRuns ?? 0;
9889
10006
  console.log(
9890
- `${result.result}: ${result.counts.passedRuns}/${result.counts.createdRuns} passed sessions (${result.counts.judgedRuns} judged)`
10007
+ measured > 0 ? `${result.result}: ${measured} preference sessions measured${result.counts.passedRuns ? `; ${result.counts.passedRuns} experience sessions passed` : ""}` : `${result.result}: ${result.counts.passedRuns}/${result.counts.createdRuns} passed sessions (${result.counts.judgedRuns} judged)`
9891
10008
  );
10009
+ for (const batch of result.batches)
10010
+ for (const run of batch.runs)
10011
+ if (run.preference)
10012
+ console.log(
10013
+ ` ${run.id}: ${preferenceSelectionLabel(run.preference)} (advisory)`
10014
+ );
9892
10015
  console.log(`Request: ${result.id}`);
9893
10016
  }
9894
- if (result.result !== "PASSED")
10017
+ if (result.result !== "PASSED" && result.result !== "MEASURED")
9895
10018
  throw new CliError(
9896
10019
  "internal",
9897
10020
  `Run request ${id}: ${result.result}`
@@ -9922,6 +10045,10 @@ function fmtDuration(ms) {
9922
10045
  function fmtUsd(v) {
9923
10046
  return v == null ? "" : `$${v.toFixed(2)}`;
9924
10047
  }
10048
+ function fmtTriggeredBy(t) {
10049
+ if (!t) return "";
10050
+ return t.email ? `${t.source} (${t.email})` : t.source;
10051
+ }
9925
10052
  function compact(value, max = 140) {
9926
10053
  const s = typeof value === "string" ? value : JSON.stringify(value) ?? "null";
9927
10054
  const one = s.replace(/\s+/g, " ").trim();
@@ -9937,8 +10064,10 @@ function toRow5(run) {
9937
10064
  model: run.modelDisplayName ?? run.resolvedModel ?? run.model,
9938
10065
  provider: observed && observed !== selected ? `${selected || "?"} \u2192 ${observed}` : selected || observed || "",
9939
10066
  experiment: run.experiment ?? "",
10067
+ source: run.triggeredBy?.source ?? "",
9940
10068
  turns: run.turns ?? "",
9941
10069
  cost: fmtUsd(run.usdCost),
10070
+ credits: run.creditsCharged ?? "",
9942
10071
  duration: fmtDuration(run.durationMs),
9943
10072
  created: run.createdAt,
9944
10073
  prompt: compact(run.promptSnippet, 48)
@@ -9975,7 +10104,9 @@ function detailLines(run) {
9975
10104
  ],
9976
10105
  ["Attempt", String(run.attempt)],
9977
10106
  ["Turns", run.turns == null ? "\u2014" : String(run.turns)],
10107
+ ["Triggered by", fmtTriggeredBy(run.triggeredBy) || "\u2014"],
9978
10108
  ["Cost", fmtUsd(run.usdCost) || "\u2014"],
10109
+ ["Credits", run.creditsCharged == null ? "\u2014" : String(run.creditsCharged)],
9979
10110
  ["Duration", fmtDuration(run.durationMs) || "\u2014"],
9980
10111
  ["Exit reason", run.exitReason ?? "\u2014"],
9981
10112
  ["Failure reason", run.failureReason ?? "\u2014"],
@@ -10197,7 +10328,9 @@ function register26(program2) {
10197
10328
  }
10198
10329
  }
10199
10330
  );
10200
- runs.command("get <id>").description("Show a run (status, cost, turns, exit reason)").action(async (id, _opts, command) => {
10331
+ runs.command("get <id>").description(
10332
+ "Show a run (status, trigger source, cost, credits, turns, exit reason)"
10333
+ ).action(async (id, _opts, command) => {
10201
10334
  const run = await createClient().get(runPath(id));
10202
10335
  if (resolveOutput(command) === "json") {
10203
10336
  printJson(run);
@@ -10852,9 +10985,9 @@ function scenarioRow(scenario) {
10852
10985
  persona: scenario.profile?.name ?? "(default)"
10853
10986
  };
10854
10987
  }
10855
- var KIND_CHOICES = ["head-to-head", "recommend"];
10988
+ var KIND_CHOICES = ["head-to-head", "open-ended"];
10856
10989
  function parseKind(raw) {
10857
- const upper = raw.trim().toUpperCase().replace(/-/g, "_");
10990
+ const upper = raw === "open-ended" ? "RECOMMEND" : raw.trim().toUpperCase().replace(/-/g, "_");
10858
10991
  if (upper !== "HEAD_TO_HEAD" && upper !== "RECOMMEND")
10859
10992
  throw new CliError(
10860
10993
  "usage",
@@ -10870,7 +11003,7 @@ function toRow6(v) {
10870
11003
  return {
10871
11004
  id: v.id,
10872
11005
  name: v.name,
10873
- kind: v.kind,
11006
+ kind: v.kind === "RECOMMEND" ? "open-ended" : "head-to-head",
10874
11007
  brands: brandsLabel(v),
10875
11008
  market: v.market?.slug ?? "",
10876
11009
  tags: v.tags.join(", "),
@@ -10881,7 +11014,9 @@ function toRow6(v) {
10881
11014
  function printDetail4(v) {
10882
11015
  console.log(`id: ${v.id}`);
10883
11016
  console.log(`name: ${v.name}`);
10884
- console.log(`kind: ${v.kind}`);
11017
+ console.log(
11018
+ `kind: ${v.kind === "RECOMMEND" ? "open-ended" : "head-to-head"}`
11019
+ );
10885
11020
  console.log(`brands: ${brandsLabel(v)}`);
10886
11021
  console.log(
10887
11022
  `market: ${v.market ? `${v.market.name} (${v.market.slug})` : "(none)"}`
@@ -10926,7 +11061,7 @@ function registerPreferenceGroup(group) {
10926
11061
  addRunSettingsCreateOptions(
10927
11062
  group.command("create").description(
10928
11063
  "Create a preference prompt. Run-settings flags set its roster, resources, cadence, and samples; --repo/--ref/--persona add its first scenario. Omit them all for a bare definition on the org's default roster."
10929
- ).requiredOption("--name <name>", "preference prompt name").requiredOption("--kind <kind>", KIND_CHOICES.join(" | ")).option("--text <text>", "prompt text inline").option("--file <path>", 'prompt text from a file ("-" for stdin)').option("--brand-a <name>", "first brand (required for head-to-head)").option("--brand-b <name>", "second brand (required for head-to-head)").option("--branded", "a branded RECOMMEND measurement").option(
11064
+ ).requiredOption("--name <name>", "preference prompt name").requiredOption("--kind <kind>", KIND_CHOICES.join(" | ")).option("--text <text>", "prompt text inline").option("--file <path>", 'prompt text from a file ("-" for stdin)').option("--brand-a <name>", "first brand (required for head-to-head)").option("--brand-b <name>", "second brand (required for head-to-head)").option("--branded", "an open-ended prompt that names a brand").option(
10930
11065
  "--repo <url>",
10931
11066
  "first scenario's repository (default: empty project)"
10932
11067
  ).option("--ref <ref>", "branch, tag, or commit (with --repo)").option("--persona <profileId>", "first scenario's persona profile id")
@@ -11107,28 +11242,37 @@ function registerPreferenceGroup(group) {
11107
11242
  result.queued ? "Synthesis queued \u2014 insights land on the Agent Preference page shortly" : "Nothing new since the last pass \u2014 no synthesis queued"
11108
11243
  );
11109
11244
  });
11110
- group.command("runs <id>").description("The prompt's run log with Agent Preference verdicts").option("--limit <n>", "max runs (default 100)").action(async (id, opts, cmd) => {
11111
- const org = resolveOrg(cmd);
11112
- const res = await createClient().get(
11113
- `${visibilityPath(org)}/${encodeURIComponent(id)}/runs`,
11114
- { limit: opts.limit }
11115
- );
11116
- if (resolveOutput(cmd) === "json") {
11117
- printJson(res.items);
11118
- return;
11245
+ group.command("runs <id>").description("The prompt's run log with Agent Preference verdicts").option("--limit <n>", "page size (1-500, default 100)").option("--cursor <cursor>", "resume from a previous page's nextCursor").action(
11246
+ async (id, opts, cmd) => {
11247
+ const org = resolveOrg(cmd);
11248
+ const res = await createClient().get(
11249
+ `${visibilityPath(org)}/${encodeURIComponent(id)}/runs`,
11250
+ { limit: opts.limit, cursor: opts.cursor }
11251
+ );
11252
+ if (resolveOutput(cmd) === "json") {
11253
+ printJson(res);
11254
+ return;
11255
+ }
11256
+ printItems(
11257
+ res.items.map((r) => ({
11258
+ id: r.id,
11259
+ status: r.status,
11260
+ agent: r.agent,
11261
+ model: r.resolvedModel ?? r.model ?? "",
11262
+ source: r.triggeredBy?.source ?? "",
11263
+ credits: r.creditsCharged ?? "",
11264
+ created: r.createdAt,
11265
+ verdict: r.verdict?.status ?? "-"
11266
+ })),
11267
+ "table"
11268
+ );
11269
+ if (res.nextCursor) {
11270
+ console.error(
11271
+ `(more results \u2014 rerun with --cursor ${res.nextCursor})`
11272
+ );
11273
+ }
11119
11274
  }
11120
- printItems(
11121
- res.items.map((r) => ({
11122
- id: r.id,
11123
- status: r.status,
11124
- agent: r.agent,
11125
- model: r.resolvedModel ?? r.model ?? "",
11126
- created: r.createdAt,
11127
- verdict: r.verdict?.status ?? "-"
11128
- })),
11129
- "table"
11130
- );
11131
- });
11275
+ );
11132
11276
  group.command("tag <id>").description("Replace a preference prompt's user tags").option(
11133
11277
  "--tag <name>",
11134
11278
  "tag name, create-on-type (repeatable)",