@withgauge/cli 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -1
- package/dist/index.js +1906 -708
- package/package.json +5 -2
package/dist/index.js
CHANGED
|
@@ -6,7 +6,7 @@ var __export = (target, all) => {
|
|
|
6
6
|
};
|
|
7
7
|
|
|
8
8
|
// src/index.ts
|
|
9
|
-
import { Command, CommanderError, Option as
|
|
9
|
+
import { Command, CommanderError, Option as Option4 } from "commander";
|
|
10
10
|
|
|
11
11
|
// src/commands/actions.ts
|
|
12
12
|
var actions_exports2 = {};
|
|
@@ -1576,6 +1576,11 @@ var cancelRunResponseSchema = z17.object({
|
|
|
1576
1576
|
cancelRequested: z17.boolean(),
|
|
1577
1577
|
note: z17.string().optional()
|
|
1578
1578
|
});
|
|
1579
|
+
var regradeRunResponseSchema = z17.object({
|
|
1580
|
+
runId: z17.string(),
|
|
1581
|
+
evalSetId: z17.string(),
|
|
1582
|
+
criteria: z17.number().int()
|
|
1583
|
+
});
|
|
1579
1584
|
var forkRunBodySchema = z17.object({
|
|
1580
1585
|
prompt: z17.string().trim().min(1).max(2e4),
|
|
1581
1586
|
turn: z17.number().int().min(0).optional(),
|
|
@@ -1718,6 +1723,7 @@ function totalRunsPerCycle(owners, samplesPerCycle) {
|
|
|
1718
1723
|
|
|
1719
1724
|
// ../packages/api-schemas/src/ownedConfigurations.ts
|
|
1720
1725
|
var sampleCountSchema = z19.number().int().min(1).max(50);
|
|
1726
|
+
var nextRunAtInputSchema = z19.string().regex(/^\d{4}-\d{2}-\d{2}/, "Next run must be a calendar date").nullable();
|
|
1721
1727
|
var ownedCadenceSchema = z19.enum([
|
|
1722
1728
|
"NONE",
|
|
1723
1729
|
"DAILY",
|
|
@@ -1733,6 +1739,7 @@ var ownedConfigurationInputShape = {
|
|
|
1733
1739
|
mcpRefs: assetRefListSchema.optional().default([]),
|
|
1734
1740
|
connectionIds: z19.array(z19.string().min(1)).max(32).optional().default([]),
|
|
1735
1741
|
cadence: ownedCadenceSchema.optional().default("NONE"),
|
|
1742
|
+
nextRunAt: nextRunAtInputSchema.optional(),
|
|
1736
1743
|
sampleCount: sampleCountSchema.optional().default(1)
|
|
1737
1744
|
};
|
|
1738
1745
|
var ownedConfigurationInputSchema = z19.object(ownedConfigurationInputShape).strict();
|
|
@@ -1745,6 +1752,7 @@ var ownedConfigurationPatchShape = {
|
|
|
1745
1752
|
mcpRefs: assetRefListSchema.optional(),
|
|
1746
1753
|
connectionIds: z19.array(z19.string().min(1)).max(32).optional(),
|
|
1747
1754
|
cadence: ownedCadenceSchema.optional(),
|
|
1755
|
+
nextRunAt: nextRunAtInputSchema.optional(),
|
|
1748
1756
|
sampleCount: sampleCountSchema.optional()
|
|
1749
1757
|
};
|
|
1750
1758
|
var ownedConfigurationSchema = z19.object({
|
|
@@ -1757,7 +1765,8 @@ var ownedConfigurationSchema = z19.object({
|
|
|
1757
1765
|
connectionIds: z19.array(z19.string()),
|
|
1758
1766
|
cadence: ownedCadenceSchema,
|
|
1759
1767
|
sampleCount: sampleCountSchema,
|
|
1760
|
-
lastScheduledAt: z19.string().nullable()
|
|
1768
|
+
lastScheduledAt: z19.string().nullable(),
|
|
1769
|
+
nextRunAt: z19.string().nullable()
|
|
1761
1770
|
});
|
|
1762
1771
|
var visibilityScenarioSchema = z19.object({
|
|
1763
1772
|
id: z19.string(),
|
|
@@ -1776,6 +1785,7 @@ var visibilityPromptSettingsInputShape = {
|
|
|
1776
1785
|
mcpRefs: ownedConfigurationInputShape.mcpRefs,
|
|
1777
1786
|
connectionIds: ownedConfigurationInputShape.connectionIds,
|
|
1778
1787
|
cadence: ownedConfigurationInputShape.cadence,
|
|
1788
|
+
nextRunAt: ownedConfigurationInputShape.nextRunAt,
|
|
1779
1789
|
sampleCount: ownedConfigurationInputShape.sampleCount
|
|
1780
1790
|
};
|
|
1781
1791
|
var visibilityPromptSettingsPatchShape = {
|
|
@@ -1784,6 +1794,7 @@ var visibilityPromptSettingsPatchShape = {
|
|
|
1784
1794
|
mcpRefs: ownedConfigurationPatchShape.mcpRefs,
|
|
1785
1795
|
connectionIds: ownedConfigurationPatchShape.connectionIds,
|
|
1786
1796
|
cadence: ownedConfigurationPatchShape.cadence,
|
|
1797
|
+
nextRunAt: ownedConfigurationPatchShape.nextRunAt,
|
|
1787
1798
|
sampleCount: ownedConfigurationPatchShape.sampleCount
|
|
1788
1799
|
};
|
|
1789
1800
|
var visibilityPromptSettingsSchema = z19.object({
|
|
@@ -1793,11 +1804,13 @@ var visibilityPromptSettingsSchema = z19.object({
|
|
|
1793
1804
|
connectionIds: z19.array(z19.string()),
|
|
1794
1805
|
cadence: ownedCadenceSchema,
|
|
1795
1806
|
sampleCount: sampleCountSchema,
|
|
1796
|
-
lastScheduledAt: z19.string().nullable()
|
|
1807
|
+
lastScheduledAt: z19.string().nullable(),
|
|
1808
|
+
nextRunAt: z19.string().nullable()
|
|
1797
1809
|
});
|
|
1798
1810
|
var runAgentsOverrideSchema = z19.array(agentConfigInputSchema).min(1, "Select at least one agent");
|
|
1799
|
-
var runEvalBodySchema = z19.object({}).strict();
|
|
1811
|
+
var runEvalBodySchema = z19.object({ billToOrg: z19.boolean().optional() }).strict();
|
|
1800
1812
|
var runVisibilityBodySchema = z19.object({
|
|
1813
|
+
billToOrg: z19.boolean().optional(),
|
|
1801
1814
|
visibilityScenarioIds: z19.array(z19.string().min(1)).min(1).optional()
|
|
1802
1815
|
}).strict();
|
|
1803
1816
|
|
|
@@ -1870,7 +1883,9 @@ var updateEvalSetBodySchema = z20.object({
|
|
|
1870
1883
|
}).extend(ownedConfigurationPatchShape).strict();
|
|
1871
1884
|
var runEvalSetResponseSchema = z20.object({
|
|
1872
1885
|
batchIds: z20.array(z20.string()),
|
|
1873
|
-
runIds: z20.array(z20.string())
|
|
1886
|
+
runIds: z20.array(z20.string()),
|
|
1887
|
+
/** Durable launch request; absent for older and climb-trial launch paths. */
|
|
1888
|
+
runRequestId: z20.string().optional()
|
|
1874
1889
|
});
|
|
1875
1890
|
var evalSetRunSchema = z20.object({
|
|
1876
1891
|
id: z20.string(),
|
|
@@ -2551,10 +2566,534 @@ var meResponseSchema = z27.object({
|
|
|
2551
2566
|
});
|
|
2552
2567
|
|
|
2553
2568
|
// ../packages/api-schemas/src/optimizations.ts
|
|
2554
|
-
import { z as
|
|
2569
|
+
import { z as z31 } from "zod";
|
|
2555
2570
|
|
|
2556
2571
|
// ../packages/api-schemas/src/climb.ts
|
|
2557
|
-
import { z as
|
|
2572
|
+
import { z as z30 } from "zod";
|
|
2573
|
+
|
|
2574
|
+
// ../packages/api-schemas/src/cli-interventions.ts
|
|
2575
|
+
import { z as z28 } from "zod";
|
|
2576
|
+
var digest = z28.string().regex(/^[a-f0-9]{64}$/);
|
|
2577
|
+
var token = z28.string().min(1).max(4096);
|
|
2578
|
+
var optionFlag = z28.string().regex(/^--?[A-Za-z0-9][A-Za-z0-9-]*$/);
|
|
2579
|
+
var envKey = z28.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/);
|
|
2580
|
+
var relativePath = z28.string().min(1).max(1024).refine(
|
|
2581
|
+
(path2) => !path2.startsWith("/") && !path2.includes("\\") && !path2.includes("\0") && path2.split("/").every(
|
|
2582
|
+
(part) => part && part !== "." && part !== ".." && ![".git", "node_modules", ".pnpm", ".yarn", ".venv"].includes(part)
|
|
2583
|
+
),
|
|
2584
|
+
"Use a workspace-relative path without traversal"
|
|
2585
|
+
);
|
|
2586
|
+
var sourcedValue = z28.discriminatedUnion("source", [
|
|
2587
|
+
z28.object({ source: z28.literal("literal"), value: z28.string().max(8192) }).strict(),
|
|
2588
|
+
z28.object({ source: z28.literal("localFact"), key: envKey }).strict()
|
|
2589
|
+
]);
|
|
2590
|
+
var cliLaunchFormSchema = z28.enum([
|
|
2591
|
+
"native",
|
|
2592
|
+
"path",
|
|
2593
|
+
"symlink",
|
|
2594
|
+
"shebang",
|
|
2595
|
+
"node-script",
|
|
2596
|
+
"python-script",
|
|
2597
|
+
"python-module",
|
|
2598
|
+
"execveat-path",
|
|
2599
|
+
"execveat-fd"
|
|
2600
|
+
]);
|
|
2601
|
+
var cliIoProfileSchema = z28.enum([
|
|
2602
|
+
"pipes",
|
|
2603
|
+
"redirected",
|
|
2604
|
+
"shared-descriptor",
|
|
2605
|
+
"pty"
|
|
2606
|
+
]);
|
|
2607
|
+
var cliOperationKindSchema = z28.enum([
|
|
2608
|
+
"ensureOption",
|
|
2609
|
+
"replaceOption",
|
|
2610
|
+
"setEnv",
|
|
2611
|
+
"streamOutput",
|
|
2612
|
+
"completeOutput",
|
|
2613
|
+
"structuredStdin",
|
|
2614
|
+
"promptRecipe",
|
|
2615
|
+
"generatedFile"
|
|
2616
|
+
]);
|
|
2617
|
+
var commandCapabilitiesSchema = z28.object({
|
|
2618
|
+
operations: z28.array(cliOperationKindSchema).max(8),
|
|
2619
|
+
options: z28.array(z28.object({ flag: optionFlag, takesValue: z28.boolean() }).strict()).max(100).default([]),
|
|
2620
|
+
settings: z28.array(envKey).max(100).default([]),
|
|
2621
|
+
localFacts: z28.array(envKey).max(30).default([]),
|
|
2622
|
+
generatedPaths: z28.array(relativePath).max(50).default([]),
|
|
2623
|
+
outputModes: z28.array(z28.enum(["streaming", "completed"])).max(2).default([]),
|
|
2624
|
+
stdinFormats: z28.array(z28.enum(["text", "json", "jsonl"])).max(3).default([]),
|
|
2625
|
+
recipeIds: z28.array(token).max(50).default([])
|
|
2626
|
+
}).strict();
|
|
2627
|
+
var cliTargetSchema = z28.object({
|
|
2628
|
+
id: token,
|
|
2629
|
+
vendor: token,
|
|
2630
|
+
package: token,
|
|
2631
|
+
version: token.nullable(),
|
|
2632
|
+
/** Published artifact identity and exact executable/entrypoint identity. */
|
|
2633
|
+
artifactSha256: digest.nullable(),
|
|
2634
|
+
entrypointSha256: digest.nullable(),
|
|
2635
|
+
/** Trace-derived recognition hints. Actual bytes are captured by record mode. */
|
|
2636
|
+
discovery: z28.object({
|
|
2637
|
+
ecosystem: token.nullable(),
|
|
2638
|
+
program: token,
|
|
2639
|
+
executables: z28.array(token).max(20)
|
|
2640
|
+
}).strict().optional(),
|
|
2641
|
+
ownershipEvidence: z28.string().min(1).max(2048),
|
|
2642
|
+
entrypoints: z28.array(z28.string().min(1).max(1024)).max(20),
|
|
2643
|
+
/** Flags accepted before the subcommand, with explicit value arity. */
|
|
2644
|
+
globalOptions: z28.array(z28.object({ flag: optionFlag, takesValue: z28.boolean() }).strict()).max(100),
|
|
2645
|
+
launchForms: z28.array(cliLaunchFormSchema).min(1),
|
|
2646
|
+
ioProfiles: z28.array(cliIoProfileSchema).min(1),
|
|
2647
|
+
commands: z28.array(
|
|
2648
|
+
z28.object({
|
|
2649
|
+
id: token,
|
|
2650
|
+
/** Tokens following global options, before a literal `--`. */
|
|
2651
|
+
subcommand: z28.array(token).max(8),
|
|
2652
|
+
capabilities: commandCapabilitiesSchema
|
|
2653
|
+
}).strict()
|
|
2654
|
+
).min(1)
|
|
2655
|
+
}).strict().superRefine((target, ctx) => {
|
|
2656
|
+
if (!target.discovery && (!target.version || !target.artifactSha256 || !target.entrypointSha256 || !target.entrypoints.length))
|
|
2657
|
+
ctx.addIssue({
|
|
2658
|
+
code: "custom",
|
|
2659
|
+
message: "Registered targets need complete artifact identities"
|
|
2660
|
+
});
|
|
2661
|
+
if (target.discovery && (target.artifactSha256 !== null || target.entrypointSha256 !== null))
|
|
2662
|
+
ctx.addIssue({
|
|
2663
|
+
code: "custom",
|
|
2664
|
+
message: "Discovered targets learn bytes from baseline invocations"
|
|
2665
|
+
});
|
|
2666
|
+
for (const [index, command] of target.commands.entries()) {
|
|
2667
|
+
if (target.commands.findIndex((other) => other.id === command.id) !== index)
|
|
2668
|
+
ctx.addIssue({
|
|
2669
|
+
code: "custom",
|
|
2670
|
+
path: ["commands", index, "id"],
|
|
2671
|
+
message: "Duplicate command ID"
|
|
2672
|
+
});
|
|
2673
|
+
const { capabilities } = command;
|
|
2674
|
+
if (new Set(capabilities.operations).size !== capabilities.operations.length)
|
|
2675
|
+
ctx.addIssue({
|
|
2676
|
+
code: "custom",
|
|
2677
|
+
path: ["commands", index, "capabilities", "operations"],
|
|
2678
|
+
message: "Duplicate operation"
|
|
2679
|
+
});
|
|
2680
|
+
const requires = (kind, valid, message) => {
|
|
2681
|
+
if (capabilities.operations.includes(kind) && !valid)
|
|
2682
|
+
ctx.addIssue({
|
|
2683
|
+
code: "custom",
|
|
2684
|
+
path: ["commands", index, "capabilities", "operations"],
|
|
2685
|
+
message
|
|
2686
|
+
});
|
|
2687
|
+
};
|
|
2688
|
+
requires(
|
|
2689
|
+
"streamOutput",
|
|
2690
|
+
capabilities.outputModes.includes("streaming"),
|
|
2691
|
+
"Streaming output needs a qualified streaming mode"
|
|
2692
|
+
);
|
|
2693
|
+
requires(
|
|
2694
|
+
"completeOutput",
|
|
2695
|
+
capabilities.outputModes.includes("completed"),
|
|
2696
|
+
"Completed output needs a qualified completed mode"
|
|
2697
|
+
);
|
|
2698
|
+
requires(
|
|
2699
|
+
"structuredStdin",
|
|
2700
|
+
capabilities.stdinFormats.length > 0,
|
|
2701
|
+
"Structured stdin needs a qualified format"
|
|
2702
|
+
);
|
|
2703
|
+
requires(
|
|
2704
|
+
"promptRecipe",
|
|
2705
|
+
capabilities.recipeIds.length > 0,
|
|
2706
|
+
"Prompt recipes need qualified prompt IDs"
|
|
2707
|
+
);
|
|
2708
|
+
requires(
|
|
2709
|
+
"generatedFile",
|
|
2710
|
+
capabilities.generatedPaths.length > 0,
|
|
2711
|
+
"Generated files need attributed output paths"
|
|
2712
|
+
);
|
|
2713
|
+
requires(
|
|
2714
|
+
"setEnv",
|
|
2715
|
+
capabilities.settings.length > 0,
|
|
2716
|
+
"Environment changes need supported settings"
|
|
2717
|
+
);
|
|
2718
|
+
requires(
|
|
2719
|
+
"ensureOption",
|
|
2720
|
+
capabilities.options.length > 0,
|
|
2721
|
+
"Arguments need supported options"
|
|
2722
|
+
);
|
|
2723
|
+
requires(
|
|
2724
|
+
"replaceOption",
|
|
2725
|
+
capabilities.options.length > 0,
|
|
2726
|
+
"Arguments need supported options"
|
|
2727
|
+
);
|
|
2728
|
+
}
|
|
2729
|
+
});
|
|
2730
|
+
var CLI_CATALOG_MAX_TARGETS = 100;
|
|
2731
|
+
var cliCatalogSchema = z28.object({
|
|
2732
|
+
version: z28.union([z28.literal(1), z28.literal(2)]),
|
|
2733
|
+
engineVersion: token,
|
|
2734
|
+
digest,
|
|
2735
|
+
targets: z28.array(cliTargetSchema).max(CLI_CATALOG_MAX_TARGETS)
|
|
2736
|
+
}).strict().superRefine((catalog, ctx) => {
|
|
2737
|
+
if (catalog.version === 1 && (!catalog.targets.length || catalog.targets.some((target) => target.discovery)))
|
|
2738
|
+
ctx.addIssue({
|
|
2739
|
+
code: "custom",
|
|
2740
|
+
message: "Discovery requires catalog version 2"
|
|
2741
|
+
});
|
|
2742
|
+
catalog.targets.forEach((target, index) => {
|
|
2743
|
+
if (catalog.targets.findIndex((other) => other.id === target.id) !== index)
|
|
2744
|
+
ctx.addIssue({
|
|
2745
|
+
code: "custom",
|
|
2746
|
+
path: ["targets", index, "id"],
|
|
2747
|
+
message: "Duplicate target ID"
|
|
2748
|
+
});
|
|
2749
|
+
});
|
|
2750
|
+
});
|
|
2751
|
+
var exitPredicate = z28.discriminatedUnion("kind", [
|
|
2752
|
+
z28.object({ kind: z28.literal("any") }).strict(),
|
|
2753
|
+
z28.object({
|
|
2754
|
+
kind: z28.literal("code"),
|
|
2755
|
+
value: z28.number().int().min(0).max(255)
|
|
2756
|
+
}).strict()
|
|
2757
|
+
]);
|
|
2758
|
+
var outputStream = z28.enum(["stdout", "stderr", "combined"]);
|
|
2759
|
+
var literalEdit = z28.object({ find: z28.string().min(1).max(8192), replace: z28.string().max(8192) }).strict();
|
|
2760
|
+
function boundedOutputPattern(pattern) {
|
|
2761
|
+
let at = pattern.startsWith("^") ? 1 : 0;
|
|
2762
|
+
const end = pattern.endsWith("$") && pattern.length > at ? pattern.length - 1 : pattern.length;
|
|
2763
|
+
let stars = 0;
|
|
2764
|
+
let literalBytes = 0;
|
|
2765
|
+
while (at < end) {
|
|
2766
|
+
const char = pattern[at];
|
|
2767
|
+
if (char === "\\") {
|
|
2768
|
+
if (++at >= end || !"\\.^$*+?[]{}()|".includes(pattern[at])) return false;
|
|
2769
|
+
literalBytes++;
|
|
2770
|
+
} else if (char === "." && pattern[at + 1] === "*") {
|
|
2771
|
+
if (++stars > 1) return false;
|
|
2772
|
+
at++;
|
|
2773
|
+
} else if (".^$*+?[]{}()|\n\r".includes(char)) {
|
|
2774
|
+
return false;
|
|
2775
|
+
} else {
|
|
2776
|
+
literalBytes++;
|
|
2777
|
+
}
|
|
2778
|
+
at++;
|
|
2779
|
+
}
|
|
2780
|
+
return literalBytes > 0;
|
|
2781
|
+
}
|
|
2782
|
+
var operationSchema = z28.discriminatedUnion("kind", [
|
|
2783
|
+
z28.object({
|
|
2784
|
+
kind: z28.literal("ensureOption"),
|
|
2785
|
+
option: optionFlag,
|
|
2786
|
+
value: sourcedValue.optional()
|
|
2787
|
+
}).strict(),
|
|
2788
|
+
z28.object({
|
|
2789
|
+
kind: z28.literal("replaceOption"),
|
|
2790
|
+
option: optionFlag,
|
|
2791
|
+
value: sourcedValue
|
|
2792
|
+
}).strict(),
|
|
2793
|
+
z28.object({
|
|
2794
|
+
kind: z28.literal("setEnv"),
|
|
2795
|
+
key: envKey,
|
|
2796
|
+
value: sourcedValue,
|
|
2797
|
+
whenAbsent: z28.boolean()
|
|
2798
|
+
}).strict(),
|
|
2799
|
+
z28.object({
|
|
2800
|
+
kind: z28.literal("streamOutput"),
|
|
2801
|
+
stream: outputStream,
|
|
2802
|
+
edit: literalEdit
|
|
2803
|
+
}).strict(),
|
|
2804
|
+
z28.object({
|
|
2805
|
+
kind: z28.literal("completeOutput"),
|
|
2806
|
+
stream: outputStream,
|
|
2807
|
+
matcher: z28.discriminatedUnion("kind", [
|
|
2808
|
+
z28.object({
|
|
2809
|
+
kind: z28.literal("literal"),
|
|
2810
|
+
text: z28.string().min(1).max(8192)
|
|
2811
|
+
}).strict(),
|
|
2812
|
+
z28.object({
|
|
2813
|
+
kind: z28.literal("re2"),
|
|
2814
|
+
pattern: z28.string().min(1).max(2048).refine(
|
|
2815
|
+
boundedOutputPattern,
|
|
2816
|
+
"Use literals, anchors, and at most one greedy .*; escape literal regex punctuation"
|
|
2817
|
+
)
|
|
2818
|
+
}).strict()
|
|
2819
|
+
]),
|
|
2820
|
+
replace: z28.string().max(8192),
|
|
2821
|
+
exit: exitPredicate
|
|
2822
|
+
}).strict(),
|
|
2823
|
+
z28.object({
|
|
2824
|
+
kind: z28.literal("structuredStdin"),
|
|
2825
|
+
format: z28.enum(["text", "json", "jsonl"]),
|
|
2826
|
+
payload: sourcedValue,
|
|
2827
|
+
timing: z28.literal("onLaunch"),
|
|
2828
|
+
callerInput: z28.literal("replace"),
|
|
2829
|
+
eof: z28.literal("close")
|
|
2830
|
+
}).strict(),
|
|
2831
|
+
z28.object({
|
|
2832
|
+
kind: z28.literal("promptRecipe"),
|
|
2833
|
+
recipeId: token,
|
|
2834
|
+
steps: z28.array(
|
|
2835
|
+
z28.object({
|
|
2836
|
+
prompt: z28.string().min(1).max(4096),
|
|
2837
|
+
response: sourcedValue,
|
|
2838
|
+
timeoutMs: z28.number().int().min(1).max(12e4)
|
|
2839
|
+
}).strict()
|
|
2840
|
+
).min(1).max(20),
|
|
2841
|
+
overallTimeoutMs: z28.number().int().min(1).max(6e5),
|
|
2842
|
+
completion: z28.string().min(1).max(4096)
|
|
2843
|
+
}).strict(),
|
|
2844
|
+
z28.object({
|
|
2845
|
+
kind: z28.literal("generatedFile"),
|
|
2846
|
+
path: relativePath,
|
|
2847
|
+
beforeSha256: digest.nullable(),
|
|
2848
|
+
generatedSha256: digest,
|
|
2849
|
+
outcome: exitPredicate,
|
|
2850
|
+
edit: literalEdit,
|
|
2851
|
+
required: z28.boolean()
|
|
2852
|
+
}).strict()
|
|
2853
|
+
]);
|
|
2854
|
+
var cliBaselineRecordFileSchema = z28.object({
|
|
2855
|
+
pathIndex: z28.number().int().nonnegative().max(49),
|
|
2856
|
+
path: relativePath,
|
|
2857
|
+
status: z28.enum(["generated", "not-exercised"]),
|
|
2858
|
+
beforeSha256: digest.nullable(),
|
|
2859
|
+
generatedSha256: digest.nullable()
|
|
2860
|
+
}).strict();
|
|
2861
|
+
var baselineInvocationSchema = z28.object({
|
|
2862
|
+
runId: token,
|
|
2863
|
+
invocationId: token,
|
|
2864
|
+
targetId: token,
|
|
2865
|
+
commandId: token,
|
|
2866
|
+
artifactSha256: digest.nullable(),
|
|
2867
|
+
entrypointSha256: digest,
|
|
2868
|
+
engineSha256: digest.optional(),
|
|
2869
|
+
launchForm: cliLaunchFormSchema,
|
|
2870
|
+
ioProfile: cliIoProfileSchema,
|
|
2871
|
+
recordFiles: z28.array(cliBaselineRecordFileSchema).max(50).default([])
|
|
2872
|
+
}).strict();
|
|
2873
|
+
var cliInterventionSchema = z28.object({
|
|
2874
|
+
schemaVersion: z28.union([z28.literal(1), z28.literal(2)]),
|
|
2875
|
+
engineVersion: token,
|
|
2876
|
+
engineSha256: digest.nullable(),
|
|
2877
|
+
bundleSha256: digest,
|
|
2878
|
+
catalog: cliCatalogSchema,
|
|
2879
|
+
mode: z28.enum(["record", "trial"]),
|
|
2880
|
+
captureProfiles: z28.array(cliIoProfileSchema).min(1),
|
|
2881
|
+
activeTargetId: token.nullable(),
|
|
2882
|
+
activeCommandId: token.nullable(),
|
|
2883
|
+
selector: z28.object({
|
|
2884
|
+
optionPresent: z28.array(optionFlag).max(20),
|
|
2885
|
+
optionAbsent: z28.array(optionFlag).max(20)
|
|
2886
|
+
}).strict().nullable(),
|
|
2887
|
+
baselineInvocations: z28.array(baselineInvocationSchema).max(500),
|
|
2888
|
+
operations: z28.array(operationSchema).max(50),
|
|
2889
|
+
limits: z28.object({
|
|
2890
|
+
completedStreamBytes: z28.number().int().min(1).max(2 * 1024 * 1024),
|
|
2891
|
+
replacementBytes: z28.number().int().min(1).max(8192),
|
|
2892
|
+
generatedFileBytes: z28.number().int().min(1).max(2 * 1024 * 1024),
|
|
2893
|
+
generatedAggregateBytes: z28.number().int().min(1).max(8 * 1024 * 1024),
|
|
2894
|
+
generatedFileCount: z28.number().int().min(1).max(50)
|
|
2895
|
+
}).strict(),
|
|
2896
|
+
shippingIntent: z28.enum([
|
|
2897
|
+
"default",
|
|
2898
|
+
"message",
|
|
2899
|
+
"interaction",
|
|
2900
|
+
"invocation-guidance",
|
|
2901
|
+
"template"
|
|
2902
|
+
]).nullable()
|
|
2903
|
+
}).strict().superRefine((bundle, ctx) => {
|
|
2904
|
+
const fail = (path2, message) => ctx.addIssue({ code: "custom", path: path2, message });
|
|
2905
|
+
if (bundle.schemaVersion !== bundle.catalog.version)
|
|
2906
|
+
fail(["schemaVersion"], "Bundle and catalog schema versions differ");
|
|
2907
|
+
if (!bundle.engineSha256 && (bundle.schemaVersion === 1 || bundle.mode !== "record"))
|
|
2908
|
+
fail(["engineSha256"], "A trial must pin the recorded engine bytes");
|
|
2909
|
+
if (!bundle.catalog.targets.length)
|
|
2910
|
+
fail(["catalog"], "A run needs at least one CLI target");
|
|
2911
|
+
if (bundle.schemaVersion === 2 && bundle.mode === "trial" && bundle.baselineInvocations.some(
|
|
2912
|
+
(item) => item.engineSha256 !== bundle.engineSha256
|
|
2913
|
+
))
|
|
2914
|
+
fail(
|
|
2915
|
+
["baselineInvocations"],
|
|
2916
|
+
"Baseline invocations must share the pinned engine"
|
|
2917
|
+
);
|
|
2918
|
+
bundle.baselineInvocations.forEach((invocation, index) => {
|
|
2919
|
+
const command2 = bundle.catalog.targets.find((target2) => target2.id === invocation.targetId)?.commands.find((item) => item.id === invocation.commandId);
|
|
2920
|
+
for (const [fileIndex, file] of invocation.recordFiles.entries()) {
|
|
2921
|
+
if (command2?.capabilities.generatedPaths[file.pathIndex] !== file.path || file.status === "generated" && !file.generatedSha256 || file.status === "not-exercised" && file.generatedSha256 !== file.beforeSha256)
|
|
2922
|
+
fail(
|
|
2923
|
+
["baselineInvocations", index, "recordFiles", fileIndex],
|
|
2924
|
+
"Recorded file does not match the qualified command"
|
|
2925
|
+
);
|
|
2926
|
+
}
|
|
2927
|
+
});
|
|
2928
|
+
if (bundle.engineVersion !== bundle.catalog.engineVersion)
|
|
2929
|
+
fail(["engineVersion"], "Engine and catalog versions differ");
|
|
2930
|
+
if (bundle.mode === "record") {
|
|
2931
|
+
if (bundle.activeTargetId || bundle.activeCommandId || bundle.selector || bundle.operations.length || bundle.shippingIntent)
|
|
2932
|
+
fail(["mode"], "Record mode cannot select or modify a command");
|
|
2933
|
+
return;
|
|
2934
|
+
}
|
|
2935
|
+
const target = bundle.catalog.targets.find(
|
|
2936
|
+
(item) => item.id === bundle.activeTargetId
|
|
2937
|
+
);
|
|
2938
|
+
const command = target?.commands.find(
|
|
2939
|
+
(item) => item.id === bundle.activeCommandId
|
|
2940
|
+
);
|
|
2941
|
+
if (!target || !command) {
|
|
2942
|
+
fail(
|
|
2943
|
+
["activeCommandId"],
|
|
2944
|
+
"Select a qualified command in the frozen catalog"
|
|
2945
|
+
);
|
|
2946
|
+
return;
|
|
2947
|
+
}
|
|
2948
|
+
if (!bundle.selector) fail(["selector"], "A trial needs token predicates");
|
|
2949
|
+
else {
|
|
2950
|
+
for (const flag of [
|
|
2951
|
+
...bundle.selector.optionPresent,
|
|
2952
|
+
...bundle.selector.optionAbsent
|
|
2953
|
+
])
|
|
2954
|
+
if (!command.capabilities.options.some((option) => option.flag === flag))
|
|
2955
|
+
fail(["selector"], `Unsupported selector option: ${flag}`);
|
|
2956
|
+
if (bundle.selector.optionPresent.some(
|
|
2957
|
+
(flag) => bundle.selector?.optionAbsent.includes(flag)
|
|
2958
|
+
))
|
|
2959
|
+
fail(
|
|
2960
|
+
["selector"],
|
|
2961
|
+
"A selector cannot require and forbid the same option"
|
|
2962
|
+
);
|
|
2963
|
+
}
|
|
2964
|
+
if (!bundle.shippingIntent || !bundle.operations.length)
|
|
2965
|
+
fail(
|
|
2966
|
+
["operations"],
|
|
2967
|
+
"A trial needs a shipping intent and at least one operation"
|
|
2968
|
+
);
|
|
2969
|
+
if (bundle.operations.some((item) => item.kind === "promptRecipe") && bundle.operations.some((item) => item.kind === "completeOutput"))
|
|
2970
|
+
fail(
|
|
2971
|
+
["operations"],
|
|
2972
|
+
"Completed-output buffering cannot block an interactive prompt"
|
|
2973
|
+
);
|
|
2974
|
+
if (bundle.captureProfiles.includes("pty")) {
|
|
2975
|
+
for (const [index, operation] of bundle.operations.entries()) {
|
|
2976
|
+
if (operation.kind === "structuredStdin")
|
|
2977
|
+
fail(
|
|
2978
|
+
["operations", index],
|
|
2979
|
+
"PTY structured stdin cannot prove the declared EOF; use a prompt recipe"
|
|
2980
|
+
);
|
|
2981
|
+
if ((operation.kind === "streamOutput" || operation.kind === "completeOutput") && operation.stream !== "combined")
|
|
2982
|
+
fail(["operations", index], "PTY output combines stdout and stderr");
|
|
2983
|
+
}
|
|
2984
|
+
}
|
|
2985
|
+
if (bundle.operations.some((item) => item.kind === "promptRecipe") && bundle.operations.some((item) => item.kind === "structuredStdin"))
|
|
2986
|
+
fail(
|
|
2987
|
+
["operations"],
|
|
2988
|
+
"A prompt recipe and structured stdin cannot both control the same input stream"
|
|
2989
|
+
);
|
|
2990
|
+
if (bundle.operations.some(
|
|
2991
|
+
(item) => (item.kind === "streamOutput" || item.kind === "completeOutput") && item.stream === "combined"
|
|
2992
|
+
) && !bundle.baselineInvocations.some(
|
|
2993
|
+
(item) => item.targetId === target.id && item.commandId === command.id && item.artifactSha256 === target.artifactSha256 && (target.discovery !== void 0 || item.entrypointSha256 === target.entrypointSha256) && target.launchForms.includes(item.launchForm) && (item.ioProfile === "shared-descriptor" || item.ioProfile === "pty") && target.ioProfiles.includes(item.ioProfile) && bundle.captureProfiles.includes(item.ioProfile)
|
|
2994
|
+
))
|
|
2995
|
+
fail(
|
|
2996
|
+
["operations"],
|
|
2997
|
+
"Combined output requires a shared-descriptor or PTY baseline profile"
|
|
2998
|
+
);
|
|
2999
|
+
if (!bundle.baselineInvocations.some(
|
|
3000
|
+
(item) => item.targetId === target.id && item.commandId === command.id && item.artifactSha256 === target.artifactSha256 && (target.discovery !== void 0 || item.entrypointSha256 === target.entrypointSha256) && target.launchForms.includes(item.launchForm) && target.ioProfiles.includes(item.ioProfile) && bundle.captureProfiles.includes(item.ioProfile)
|
|
3001
|
+
))
|
|
3002
|
+
fail(
|
|
3003
|
+
["baselineInvocations"],
|
|
3004
|
+
"No baseline invocation matches this exact command, artifact, launch form, and I/O profile"
|
|
3005
|
+
);
|
|
3006
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3007
|
+
const changedEnvironment = new Set(
|
|
3008
|
+
bundle.operations.flatMap(
|
|
3009
|
+
(item) => item.kind === "setEnv" ? [item.key] : []
|
|
3010
|
+
)
|
|
3011
|
+
);
|
|
3012
|
+
bundle.operations.forEach((operation, index) => {
|
|
3013
|
+
const path2 = ["operations", index];
|
|
3014
|
+
if (!command.capabilities.operations.includes(operation.kind))
|
|
3015
|
+
fail(path2, `${operation.kind} is not qualified for this command`);
|
|
3016
|
+
if ("option" in operation && !command.capabilities.options.some(
|
|
3017
|
+
(option) => option.flag === operation.option
|
|
3018
|
+
))
|
|
3019
|
+
fail(path2, "Option is not supported by this version");
|
|
3020
|
+
if (operation.kind === "ensureOption" || operation.kind === "replaceOption") {
|
|
3021
|
+
const option = command.capabilities.options.find(
|
|
3022
|
+
(item) => item.flag === operation.option
|
|
3023
|
+
);
|
|
3024
|
+
if (option && option.takesValue !== (operation.value !== void 0))
|
|
3025
|
+
fail(path2, "Option value does not match its qualified arity");
|
|
3026
|
+
}
|
|
3027
|
+
if (operation.kind === "setEnv" && !command.capabilities.settings.includes(operation.key))
|
|
3028
|
+
fail(path2, "Environment setting is not supported by this version");
|
|
3029
|
+
if (operation.kind === "structuredStdin" && !command.capabilities.stdinFormats.includes(operation.format))
|
|
3030
|
+
fail(path2, "Stdin format is not qualified");
|
|
3031
|
+
if (operation.kind === "structuredStdin" && operation.payload.source === "literal" && operation.format !== "text") {
|
|
3032
|
+
const records = operation.format === "json" ? [operation.payload.value] : operation.payload.value.trimEnd().split("\n");
|
|
3033
|
+
try {
|
|
3034
|
+
if (records.some((record) => !record.trim()))
|
|
3035
|
+
throw new Error("Empty structured stdin record");
|
|
3036
|
+
for (const record of records) JSON.parse(record);
|
|
3037
|
+
} catch {
|
|
3038
|
+
fail(
|
|
3039
|
+
path2,
|
|
3040
|
+
"Structured stdin payload does not match its declared JSON format"
|
|
3041
|
+
);
|
|
3042
|
+
}
|
|
3043
|
+
}
|
|
3044
|
+
if (operation.kind === "promptRecipe" && !command.capabilities.recipeIds.includes(operation.recipeId))
|
|
3045
|
+
fail(path2, "Prompt recipe is not qualified");
|
|
3046
|
+
if (operation.kind === "generatedFile" && !command.capabilities.generatedPaths.includes(operation.path))
|
|
3047
|
+
fail(path2, "Output path is not attributed to this command");
|
|
3048
|
+
if (operation.kind === "generatedFile" && !bundle.baselineInvocations.some(
|
|
3049
|
+
(invocation) => invocation.targetId === target.id && invocation.commandId === command.id && invocation.recordFiles.some(
|
|
3050
|
+
(file) => file.path === operation.path && file.status === "generated" && file.beforeSha256 === operation.beforeSha256 && file.generatedSha256 === operation.generatedSha256
|
|
3051
|
+
)
|
|
3052
|
+
))
|
|
3053
|
+
fail(
|
|
3054
|
+
path2,
|
|
3055
|
+
"No attributed baseline file matches the preimage and real generated bytes"
|
|
3056
|
+
);
|
|
3057
|
+
if (operation.kind === "streamOutput" && !command.capabilities.outputModes.includes("streaming"))
|
|
3058
|
+
fail(path2, "Streaming output is not qualified");
|
|
3059
|
+
if (operation.kind === "completeOutput" && !command.capabilities.outputModes.includes("completed"))
|
|
3060
|
+
fail(path2, "Completed output is not qualified");
|
|
3061
|
+
if (operation.kind === "streamOutput" && operation.edit.find === operation.edit.replace || operation.kind === "generatedFile" && operation.edit.find === operation.edit.replace || operation.kind === "completeOutput" && operation.matcher.kind === "literal" && operation.matcher.text === operation.replace)
|
|
3062
|
+
fail(path2, "A trial operation must change the matched content");
|
|
3063
|
+
const sources = [
|
|
3064
|
+
"value" in operation ? operation.value : null,
|
|
3065
|
+
operation.kind === "structuredStdin" ? operation.payload : null,
|
|
3066
|
+
...operation.kind === "promptRecipe" ? operation.steps.map((step) => step.response) : []
|
|
3067
|
+
];
|
|
3068
|
+
if (sources.some(
|
|
3069
|
+
(source) => source?.source === "localFact" && !command.capabilities.localFacts.includes(source.key)
|
|
3070
|
+
))
|
|
3071
|
+
fail(path2, "Local fact is not declared non-secret for this command");
|
|
3072
|
+
if (sources.some(
|
|
3073
|
+
(source) => source?.source === "localFact" && changedEnvironment.has(source.key)
|
|
3074
|
+
))
|
|
3075
|
+
fail(
|
|
3076
|
+
path2,
|
|
3077
|
+
"A local fact cannot be changed by another operation in the same trial"
|
|
3078
|
+
);
|
|
3079
|
+
const conflict = operation.kind === "ensureOption" || operation.kind === "replaceOption" ? `option:${operation.option}` : operation.kind === "setEnv" ? `env:${operation.key}` : operation.kind === "structuredStdin" ? "stdin" : operation.kind === "promptRecipe" ? `prompt:${operation.recipeId}` : operation.kind === "generatedFile" ? `file:${operation.path}` : null;
|
|
3080
|
+
if (conflict && seen.has(conflict))
|
|
3081
|
+
fail(path2, "Conflicting operation on the same input or output");
|
|
3082
|
+
if (conflict) seen.add(conflict);
|
|
3083
|
+
});
|
|
3084
|
+
if (bundle.operations.filter((item) => item.kind === "generatedFile").length > bundle.limits.generatedFileCount)
|
|
3085
|
+
fail(
|
|
3086
|
+
["limits", "generatedFileCount"],
|
|
3087
|
+
"Generated file count exceeds limit"
|
|
3088
|
+
);
|
|
3089
|
+
});
|
|
3090
|
+
var cliTrialChangeSchema = z28.object({
|
|
3091
|
+
targetId: token,
|
|
3092
|
+
commandId: token,
|
|
3093
|
+
selector: cliInterventionSchema.shape.selector.unwrap(),
|
|
3094
|
+
operations: cliInterventionSchema.shape.operations.min(1),
|
|
3095
|
+
shippingIntent: cliInterventionSchema.shape.shippingIntent.unwrap()
|
|
3096
|
+
}).strict();
|
|
2558
3097
|
|
|
2559
3098
|
// ../packages/api-schemas/src/climb-kinds.ts
|
|
2560
3099
|
var LEVER_KINDS = [
|
|
@@ -2572,46 +3111,47 @@ var INTERVENTION_KINDS = [
|
|
|
2572
3111
|
"REWRITE_DOCS",
|
|
2573
3112
|
"PACKAGE_VERSION",
|
|
2574
3113
|
"PACKAGE_CONTENT",
|
|
2575
|
-
"CLI_VERSION"
|
|
3114
|
+
"CLI_VERSION",
|
|
3115
|
+
"CLI_INTERVENTION"
|
|
2576
3116
|
];
|
|
2577
3117
|
var LEVER_INTERVENTIONS = {
|
|
2578
3118
|
DOCS: ["REWRITE_DOCS"],
|
|
2579
3119
|
MARKETING_SITE: ["REWRITE_DOCS"],
|
|
2580
3120
|
SDK: ["PACKAGE_VERSION", "PACKAGE_CONTENT"],
|
|
2581
|
-
CLI: ["
|
|
3121
|
+
CLI: ["CLI_INTERVENTION"],
|
|
2582
3122
|
SKILL: ["ADD_SKILL", "REMOVE_SKILL", "EDIT_SKILL"]
|
|
2583
3123
|
};
|
|
2584
3124
|
|
|
2585
3125
|
// ../packages/api-schemas/src/sandbox-overrides.ts
|
|
2586
|
-
import { z as
|
|
3126
|
+
import { z as z29 } from "zod";
|
|
2587
3127
|
var SANDBOX_OVERRIDE_KINDS = ["packages", "clis"];
|
|
2588
|
-
var ecosystemSchema =
|
|
2589
|
-
var overrideFilesSchema =
|
|
2590
|
-
|
|
2591
|
-
path:
|
|
3128
|
+
var ecosystemSchema = z29.enum(["npm", "pypi", "cargo", "go", "gem"]);
|
|
3129
|
+
var overrideFilesSchema = z29.array(
|
|
3130
|
+
z29.object({
|
|
3131
|
+
path: z29.string().min(1).refine(
|
|
2592
3132
|
(path2) => !path2.startsWith("/") && !path2.includes("\\") && !path2.split("/").includes(".."),
|
|
2593
3133
|
"Use a relative path within the target"
|
|
2594
3134
|
),
|
|
2595
|
-
body:
|
|
3135
|
+
body: z29.string()
|
|
2596
3136
|
}).strict()
|
|
2597
3137
|
).min(1);
|
|
2598
3138
|
var contentShape = {
|
|
2599
|
-
version:
|
|
3139
|
+
version: z29.string().trim().min(1).optional(),
|
|
2600
3140
|
files: overrideFilesSchema.optional()
|
|
2601
3141
|
};
|
|
2602
3142
|
var hasContent = (value) => value.version !== void 0 || value.files !== void 0;
|
|
2603
|
-
var sandboxOverridesSchema =
|
|
2604
|
-
version:
|
|
2605
|
-
packages:
|
|
2606
|
-
|
|
3143
|
+
var sandboxOverridesSchema = z29.object({
|
|
3144
|
+
version: z29.literal(1),
|
|
3145
|
+
packages: z29.array(
|
|
3146
|
+
z29.object({
|
|
2607
3147
|
ecosystem: ecosystemSchema,
|
|
2608
|
-
name:
|
|
3148
|
+
name: z29.string().trim().min(1),
|
|
2609
3149
|
...contentShape
|
|
2610
3150
|
}).strict().refine(hasContent, "Specify a version or files")
|
|
2611
3151
|
).optional(),
|
|
2612
|
-
clis:
|
|
2613
|
-
|
|
2614
|
-
name:
|
|
3152
|
+
clis: z29.array(
|
|
3153
|
+
z29.object({
|
|
3154
|
+
name: z29.string().trim().min(1),
|
|
2615
3155
|
...contentShape
|
|
2616
3156
|
}).strict().refine(hasContent, "Specify a version or files")
|
|
2617
3157
|
).optional()
|
|
@@ -2638,11 +3178,11 @@ var sandboxOverridesSchema = z28.object({
|
|
|
2638
3178
|
});
|
|
2639
3179
|
|
|
2640
3180
|
// ../packages/api-schemas/src/climb.ts
|
|
2641
|
-
var climbSubjectKindSchema =
|
|
2642
|
-
var leverKindSchema =
|
|
2643
|
-
var interventionKindSchema =
|
|
2644
|
-
var skill =
|
|
2645
|
-
var empty =
|
|
3181
|
+
var climbSubjectKindSchema = z30.enum(["EVAL_SET", "VISIBILITY_PROMPT"]);
|
|
3182
|
+
var leverKindSchema = z30.enum(LEVER_KINDS);
|
|
3183
|
+
var interventionKindSchema = z30.enum(INTERVENTION_KINDS);
|
|
3184
|
+
var skill = z30.object({ skillVersionId: z30.string().min(1) }).strict();
|
|
3185
|
+
var empty = z30.object({}).strict();
|
|
2646
3186
|
function validateFixtureIntervention(value, ctx) {
|
|
2647
3187
|
const result = webFixtureSchema.safeParse({ version: 1, ...value });
|
|
2648
3188
|
if (!result.success) {
|
|
@@ -2655,71 +3195,75 @@ function validateFixtureIntervention(value, ctx) {
|
|
|
2655
3195
|
}
|
|
2656
3196
|
}
|
|
2657
3197
|
}
|
|
2658
|
-
var interventionSchema =
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
kind:
|
|
2665
|
-
before:
|
|
2666
|
-
after:
|
|
3198
|
+
var interventionSchema = z30.discriminatedUnion("kind", [
|
|
3199
|
+
z30.object({ kind: z30.literal("BASELINE"), after: empty }),
|
|
3200
|
+
z30.object({ kind: z30.literal("ADD_SKILL"), after: skill }),
|
|
3201
|
+
z30.object({ kind: z30.literal("REMOVE_SKILL"), before: skill, after: empty }),
|
|
3202
|
+
z30.object({ kind: z30.literal("EDIT_SKILL"), before: skill, after: skill }),
|
|
3203
|
+
z30.object({
|
|
3204
|
+
kind: z30.literal("REWRITE_DOCS"),
|
|
3205
|
+
before: z30.json().optional(),
|
|
3206
|
+
after: z30.object({ pages: webFixtureSchema.shape.pages.unwrap().min(1) }).strict().superRefine(validateFixtureIntervention)
|
|
2667
3207
|
}),
|
|
2668
|
-
|
|
2669
|
-
kind:
|
|
2670
|
-
after:
|
|
3208
|
+
z30.object({
|
|
3209
|
+
kind: z30.literal("PACKAGE_VERSION"),
|
|
3210
|
+
after: z30.object({
|
|
2671
3211
|
ecosystem: ecosystemSchema,
|
|
2672
|
-
name:
|
|
2673
|
-
version:
|
|
3212
|
+
name: z30.string().min(1),
|
|
3213
|
+
version: z30.string().min(1)
|
|
2674
3214
|
}).strict()
|
|
2675
3215
|
}),
|
|
2676
|
-
|
|
2677
|
-
kind:
|
|
2678
|
-
after:
|
|
3216
|
+
z30.object({
|
|
3217
|
+
kind: z30.literal("PACKAGE_CONTENT"),
|
|
3218
|
+
after: z30.object({
|
|
2679
3219
|
ecosystem: ecosystemSchema,
|
|
2680
|
-
name:
|
|
3220
|
+
name: z30.string().min(1),
|
|
2681
3221
|
files: overrideFilesSchema
|
|
2682
3222
|
}).strict()
|
|
2683
3223
|
}),
|
|
2684
|
-
|
|
2685
|
-
kind:
|
|
2686
|
-
after:
|
|
3224
|
+
z30.object({
|
|
3225
|
+
kind: z30.literal("CLI_VERSION"),
|
|
3226
|
+
after: z30.object({ name: z30.string().min(1), version: z30.string().min(1) }).strict()
|
|
3227
|
+
}),
|
|
3228
|
+
z30.object({
|
|
3229
|
+
kind: z30.literal("CLI_INTERVENTION"),
|
|
3230
|
+
after: cliInterventionSchema.refine((bundle) => bundle.mode === "trial", {
|
|
3231
|
+
message: "A CLI trial requires a trial-mode bundle"
|
|
3232
|
+
})
|
|
2687
3233
|
})
|
|
2688
3234
|
]);
|
|
2689
|
-
var climbSettingsSchema =
|
|
2690
|
-
maxRounds:
|
|
2691
|
-
sampleCount:
|
|
2692
|
-
creditCap:
|
|
2693
|
-
|
|
2694
|
-
autoAdopt: z29.boolean().default(false),
|
|
2695
|
-
targetScore: z29.number().min(0).max(1).default(1),
|
|
3235
|
+
var climbSettingsSchema = z30.object({
|
|
3236
|
+
maxRounds: z30.number().int().positive().default(5),
|
|
3237
|
+
sampleCount: z30.number().int().positive().default(1),
|
|
3238
|
+
creditCap: z30.number().int().nonnegative(),
|
|
3239
|
+
targetScore: z30.number().min(0).max(1).default(1),
|
|
2696
3240
|
// Narrow the subject's whole roster to one slot, for the baseline and every
|
|
2697
3241
|
// trial alike. A subject rostering ten models otherwise multiplies every
|
|
2698
3242
|
// trial by ten, and a one-model trial measured against a ten-model baseline
|
|
2699
3243
|
// is not a comparison. `pinnedModel` empty means the agent's default model.
|
|
2700
3244
|
pinnedAgent: schedulableAgentSchema.optional(),
|
|
2701
|
-
pinnedModel:
|
|
2702
|
-
maxTrialsPerRound:
|
|
3245
|
+
pinnedModel: z30.string().optional(),
|
|
3246
|
+
maxTrialsPerRound: z30.number().int().min(1).max(10).default(3),
|
|
2703
3247
|
// The round a continuation started at; the no-improvement counter
|
|
2704
3248
|
// restarts here.
|
|
2705
|
-
continuedAtRound:
|
|
2706
|
-
}).
|
|
3249
|
+
continuedAtRound: z30.number().int().nonnegative().optional()
|
|
3250
|
+
}).strip().refine((value) => value.pinnedAgent !== void 0 || !value.pinnedModel, {
|
|
2707
3251
|
path: ["pinnedAgent"],
|
|
2708
3252
|
message: "Pin an agent with the model"
|
|
2709
3253
|
});
|
|
2710
|
-
var climbDirectionSchema =
|
|
2711
|
-
focus:
|
|
2712
|
-
preserve:
|
|
2713
|
-
exclude:
|
|
2714
|
-
notes:
|
|
2715
|
-
});
|
|
2716
|
-
var climbDirectionPatchSchema =
|
|
2717
|
-
focus:
|
|
2718
|
-
preserve:
|
|
2719
|
-
exclude:
|
|
2720
|
-
notes:
|
|
3254
|
+
var climbDirectionSchema = z30.object({
|
|
3255
|
+
focus: z30.string().trim().max(2e3).nullable().default(null),
|
|
3256
|
+
preserve: z30.array(z30.string().trim().min(1).max(500)).max(20).default([]),
|
|
3257
|
+
exclude: z30.array(z30.string().trim().min(1).max(500)).max(20).default([]),
|
|
3258
|
+
notes: z30.string().trim().max(4e3).nullable().default(null)
|
|
3259
|
+
});
|
|
3260
|
+
var climbDirectionPatchSchema = z30.object({
|
|
3261
|
+
focus: z30.string().trim().max(2e3).nullable().optional(),
|
|
3262
|
+
preserve: z30.array(z30.string().trim().min(1).max(500)).max(20).optional(),
|
|
3263
|
+
exclude: z30.array(z30.string().trim().min(1).max(500)).max(20).optional(),
|
|
3264
|
+
notes: z30.string().trim().max(4e3).nullable().optional(),
|
|
2721
3265
|
/** One line on why this was recorded; becomes the DIRECTION row summary. */
|
|
2722
|
-
rationale:
|
|
3266
|
+
rationale: z30.string().trim().max(500).optional()
|
|
2723
3267
|
}).strict();
|
|
2724
3268
|
var CLIMB_ACTIVITY_KINDS = [
|
|
2725
3269
|
"STARTED",
|
|
@@ -2727,7 +3271,6 @@ var CLIMB_ACTIVITY_KINDS = [
|
|
|
2727
3271
|
"EXCLUDED",
|
|
2728
3272
|
"LAUNCHED",
|
|
2729
3273
|
"SETTLED",
|
|
2730
|
-
"CONFIRMING",
|
|
2731
3274
|
"DONE",
|
|
2732
3275
|
"FAILED",
|
|
2733
3276
|
"DIRECTION",
|
|
@@ -2739,73 +3282,74 @@ var CLIMB_ACTIVITY_KINDS = [
|
|
|
2739
3282
|
"CONTINUED",
|
|
2740
3283
|
"CANCELED"
|
|
2741
3284
|
];
|
|
2742
|
-
var climbActivityKindSchema =
|
|
2743
|
-
var proposerTraceStepSchema =
|
|
2744
|
-
tool:
|
|
2745
|
-
args:
|
|
2746
|
-
result:
|
|
2747
|
-
isError:
|
|
2748
|
-
});
|
|
2749
|
-
var proposerTraceSchema =
|
|
2750
|
-
steps:
|
|
2751
|
-
reasoning:
|
|
2752
|
-
durationMs:
|
|
2753
|
-
model:
|
|
2754
|
-
});
|
|
2755
|
-
var climbActivitySchema =
|
|
2756
|
-
id:
|
|
2757
|
-
climbId:
|
|
2758
|
-
roundId:
|
|
2759
|
-
trialId:
|
|
3285
|
+
var climbActivityKindSchema = z30.enum(CLIMB_ACTIVITY_KINDS);
|
|
3286
|
+
var proposerTraceStepSchema = z30.object({
|
|
3287
|
+
tool: z30.string(),
|
|
3288
|
+
args: z30.string(),
|
|
3289
|
+
result: z30.string().nullable(),
|
|
3290
|
+
isError: z30.boolean().optional()
|
|
3291
|
+
});
|
|
3292
|
+
var proposerTraceSchema = z30.object({
|
|
3293
|
+
steps: z30.array(proposerTraceStepSchema),
|
|
3294
|
+
reasoning: z30.string(),
|
|
3295
|
+
durationMs: z30.number().int().nonnegative(),
|
|
3296
|
+
model: z30.string()
|
|
3297
|
+
});
|
|
3298
|
+
var climbActivitySchema = z30.object({
|
|
3299
|
+
id: z30.string(),
|
|
3300
|
+
climbId: z30.string(),
|
|
3301
|
+
roundId: z30.string().nullable(),
|
|
3302
|
+
trialId: z30.string().nullable(),
|
|
2760
3303
|
kind: climbActivityKindSchema,
|
|
2761
|
-
actorUserId:
|
|
2762
|
-
actorName:
|
|
2763
|
-
summary:
|
|
2764
|
-
detail:
|
|
2765
|
-
createdAt:
|
|
2766
|
-
});
|
|
2767
|
-
var activityQuerySchema =
|
|
2768
|
-
after:
|
|
2769
|
-
limit:
|
|
2770
|
-
});
|
|
2771
|
-
var climbConfigSnapshotSchema =
|
|
2772
|
-
promptText:
|
|
2773
|
-
scenario:
|
|
2774
|
-
repoUrl:
|
|
2775
|
-
repoRef:
|
|
2776
|
-
profileId:
|
|
2777
|
-
profileName:
|
|
2778
|
-
profileDescription:
|
|
2779
|
-
profileModel:
|
|
2780
|
-
limits:
|
|
2781
|
-
agents:
|
|
2782
|
-
skillRefs:
|
|
2783
|
-
mcpRefs:
|
|
2784
|
-
skillVersionIds:
|
|
2785
|
-
mcpServerVersionIds:
|
|
2786
|
-
skillSet:
|
|
2787
|
-
mcpSet:
|
|
2788
|
-
connectionIds:
|
|
2789
|
-
connectionSet:
|
|
2790
|
-
agentContext:
|
|
2791
|
-
agentContextDigest:
|
|
3304
|
+
actorUserId: z30.string().nullable(),
|
|
3305
|
+
actorName: z30.string().nullable(),
|
|
3306
|
+
summary: z30.string(),
|
|
3307
|
+
detail: z30.unknown().nullable(),
|
|
3308
|
+
createdAt: z30.string()
|
|
3309
|
+
});
|
|
3310
|
+
var activityQuerySchema = z30.object({
|
|
3311
|
+
after: z30.string().datetime().optional(),
|
|
3312
|
+
limit: z30.coerce.number().int().min(1).max(500).default(500)
|
|
3313
|
+
});
|
|
3314
|
+
var climbConfigSnapshotSchema = z30.object({
|
|
3315
|
+
promptText: z30.string(),
|
|
3316
|
+
scenario: z30.enum(["GREENFIELD", "MIGRATION"]),
|
|
3317
|
+
repoUrl: z30.string().nullable(),
|
|
3318
|
+
repoRef: z30.string().nullable(),
|
|
3319
|
+
profileId: z30.string().nullable(),
|
|
3320
|
+
profileName: z30.string().nullable(),
|
|
3321
|
+
profileDescription: z30.string().nullable(),
|
|
3322
|
+
profileModel: z30.string().nullable(),
|
|
3323
|
+
limits: z30.json(),
|
|
3324
|
+
agents: z30.array(agentConfigSchema).min(1),
|
|
3325
|
+
skillRefs: z30.array(z30.string()),
|
|
3326
|
+
mcpRefs: z30.array(z30.string()),
|
|
3327
|
+
skillVersionIds: z30.array(z30.string()),
|
|
3328
|
+
mcpServerVersionIds: z30.array(z30.string()),
|
|
3329
|
+
skillSet: z30.json().nullable(),
|
|
3330
|
+
mcpSet: z30.json().nullable(),
|
|
3331
|
+
connectionIds: z30.array(z30.string()),
|
|
3332
|
+
connectionSet: z30.json().nullable(),
|
|
3333
|
+
agentContext: z30.json().nullable(),
|
|
3334
|
+
agentContextDigest: z30.string().nullable(),
|
|
2792
3335
|
webFixture: webFixtureSchema.nullable(),
|
|
3336
|
+
cliIntervention: cliInterventionSchema.nullable().default(null),
|
|
2793
3337
|
sandboxOverrides: sandboxOverridesSchema.nullable(),
|
|
2794
|
-
visibilityScenarioId:
|
|
3338
|
+
visibilityScenarioId: z30.string().nullable()
|
|
2795
3339
|
}).strict();
|
|
2796
|
-
var targetRefSchema =
|
|
2797
|
-
kind:
|
|
2798
|
-
id:
|
|
2799
|
-
url:
|
|
2800
|
-
package:
|
|
3340
|
+
var targetRefSchema = z30.object({
|
|
3341
|
+
kind: z30.string().min(1),
|
|
3342
|
+
id: z30.string().optional(),
|
|
3343
|
+
url: z30.url().optional(),
|
|
3344
|
+
package: z30.string().optional()
|
|
2801
3345
|
}).strict();
|
|
2802
|
-
var trialInputSchema =
|
|
2803
|
-
title:
|
|
2804
|
-
hypothesis:
|
|
3346
|
+
var trialInputSchema = z30.object({
|
|
3347
|
+
title: z30.string().trim().min(1).max(200),
|
|
3348
|
+
hypothesis: z30.string().max(4e3).optional(),
|
|
2805
3349
|
lever: leverKindSchema,
|
|
2806
3350
|
targetRef: targetRefSchema.optional(),
|
|
2807
3351
|
intervention: interventionSchema,
|
|
2808
|
-
authorMode:
|
|
3352
|
+
authorMode: z30.enum(["MANUAL", "DIRECT", "PROPOSE"]).default("MANUAL")
|
|
2809
3353
|
}).strict().superRefine((value, ctx) => {
|
|
2810
3354
|
const allowed = LEVER_INTERVENTIONS[value.lever];
|
|
2811
3355
|
if (!allowed.includes(value.intervention.kind))
|
|
@@ -2824,7 +3368,7 @@ var OPTIMIZATION_STATUSES = [
|
|
|
2824
3368
|
"DONE",
|
|
2825
3369
|
"ARCHIVED"
|
|
2826
3370
|
];
|
|
2827
|
-
var optimizationStatusSchema =
|
|
3371
|
+
var optimizationStatusSchema = z31.enum(OPTIMIZATION_STATUSES);
|
|
2828
3372
|
var ROUND_STATUSES = [
|
|
2829
3373
|
"PROPOSING",
|
|
2830
3374
|
"PROPOSED",
|
|
@@ -2853,61 +3397,61 @@ var OPTIMIZATION_EFFORTS = [
|
|
|
2853
3397
|
{ value: "HIGH", label: "Thorough", sampleCount: 2, maxRounds: 8 },
|
|
2854
3398
|
{ value: "XHIGH", label: "Exhaustive", sampleCount: 3, maxRounds: 12 }
|
|
2855
3399
|
];
|
|
2856
|
-
var optimizationEffortSchema =
|
|
3400
|
+
var optimizationEffortSchema = z31.enum(
|
|
2857
3401
|
OPTIMIZATION_EFFORTS.map((preset) => preset.value)
|
|
2858
3402
|
);
|
|
2859
|
-
var listOptimizationsQuerySchema =
|
|
2860
|
-
status:
|
|
3403
|
+
var listOptimizationsQuerySchema = z31.object({
|
|
3404
|
+
status: z31.string().optional().transform(
|
|
2861
3405
|
(value) => value ? value.split(",").map((part) => part.trim().toUpperCase()).filter(Boolean) : void 0
|
|
2862
|
-
).pipe(
|
|
3406
|
+
).pipe(z31.array(optimizationStatusSchema).optional()),
|
|
2863
3407
|
subjectKind: climbSubjectKindSchema.optional(),
|
|
2864
|
-
subjectId:
|
|
2865
|
-
limit:
|
|
3408
|
+
subjectId: z31.string().min(1).optional(),
|
|
3409
|
+
limit: z31.coerce.number().int().min(1).max(200).default(50),
|
|
2866
3410
|
/** Drafts are workspaces, not runs; older clients never see them. */
|
|
2867
|
-
includeDrafts:
|
|
3411
|
+
includeDrafts: z31.union([z31.boolean(), z31.string()]).optional().transform((value) => value === true || value === "true" || value === "1")
|
|
2868
3412
|
});
|
|
2869
|
-
var optimizationSummarySchema =
|
|
2870
|
-
id:
|
|
2871
|
-
name:
|
|
3413
|
+
var optimizationSummarySchema = z31.object({
|
|
3414
|
+
id: z31.string(),
|
|
3415
|
+
name: z31.string(),
|
|
2872
3416
|
status: optimizationStatusSchema,
|
|
2873
|
-
mode:
|
|
3417
|
+
mode: z31.enum(["MANUAL", "AUTO"]),
|
|
2874
3418
|
/** Autonomous launches every proposed round; interactive waits for one. */
|
|
2875
|
-
autonomous:
|
|
2876
|
-
stopReason:
|
|
3419
|
+
autonomous: z31.boolean(),
|
|
3420
|
+
stopReason: z31.string().nullable(),
|
|
2877
3421
|
subjectKind: climbSubjectKindSchema,
|
|
2878
|
-
subjectId:
|
|
2879
|
-
subjectName:
|
|
3422
|
+
subjectId: z31.string(),
|
|
3423
|
+
subjectName: z31.string(),
|
|
2880
3424
|
lever: leverKindSchema.nullable(),
|
|
2881
|
-
round:
|
|
2882
|
-
maxRounds:
|
|
3425
|
+
round: z31.number().int(),
|
|
3426
|
+
maxRounds: z31.number().int(),
|
|
2883
3427
|
/** Starting baseline score, 0..1. */
|
|
2884
|
-
before:
|
|
2885
|
-
/** Score of the
|
|
2886
|
-
after:
|
|
2887
|
-
/** Percentage-point change from the
|
|
2888
|
-
delta:
|
|
3428
|
+
before: z31.number().nullable(),
|
|
3429
|
+
/** Score of the marked winner, or best BETTER trial, 0..1. */
|
|
3430
|
+
after: z31.number().nullable(),
|
|
3431
|
+
/** Percentage-point change from the baseline to the winner or best BETTER trial. */
|
|
3432
|
+
delta: z31.number().nullable(),
|
|
2889
3433
|
/** Score after each round, baseline first. */
|
|
2890
|
-
path:
|
|
2891
|
-
creditsSpent:
|
|
2892
|
-
creditsReserved:
|
|
2893
|
-
summary:
|
|
2894
|
-
href:
|
|
2895
|
-
createdAt:
|
|
2896
|
-
updatedAt:
|
|
2897
|
-
});
|
|
2898
|
-
var listOptimizationsResponseSchema =
|
|
2899
|
-
items:
|
|
2900
|
-
});
|
|
2901
|
-
var targetSchema =
|
|
3434
|
+
path: z31.array(z31.number()),
|
|
3435
|
+
creditsSpent: z31.number().int(),
|
|
3436
|
+
creditsReserved: z31.number().int(),
|
|
3437
|
+
summary: z31.string().nullable(),
|
|
3438
|
+
href: z31.string(),
|
|
3439
|
+
createdAt: z31.string(),
|
|
3440
|
+
updatedAt: z31.string()
|
|
3441
|
+
});
|
|
3442
|
+
var listOptimizationsResponseSchema = z31.object({
|
|
3443
|
+
items: z31.array(optimizationSummarySchema)
|
|
3444
|
+
});
|
|
3445
|
+
var targetSchema = z31.object({
|
|
2902
3446
|
agent: schedulableAgentSchema.optional(),
|
|
2903
3447
|
/** Empty or omitted means the agent's default model. */
|
|
2904
|
-
model:
|
|
3448
|
+
model: z31.string().optional()
|
|
2905
3449
|
});
|
|
2906
3450
|
var planShape = {
|
|
2907
3451
|
effort: optimizationEffortSchema.optional(),
|
|
2908
|
-
sampleCount:
|
|
2909
|
-
maxRounds:
|
|
2910
|
-
maxTrialsPerRound:
|
|
3452
|
+
sampleCount: z31.number().int().min(1).max(100).optional(),
|
|
3453
|
+
maxRounds: z31.number().int().min(1).max(100).optional(),
|
|
3454
|
+
maxTrialsPerRound: z31.number().int().min(1).max(10).optional()
|
|
2911
3455
|
};
|
|
2912
3456
|
function refinePlan(value, ctx) {
|
|
2913
3457
|
if (value.effort && (value.sampleCount || value.maxRounds))
|
|
@@ -2923,19 +3467,18 @@ function refinePlan(value, ctx) {
|
|
|
2923
3467
|
message: "Pin an agent with the model"
|
|
2924
3468
|
});
|
|
2925
3469
|
}
|
|
2926
|
-
var startOptimizationBodySchema =
|
|
3470
|
+
var startOptimizationBodySchema = z31.object({
|
|
2927
3471
|
subjectKind: climbSubjectKindSchema,
|
|
2928
|
-
subjectId:
|
|
2929
|
-
name:
|
|
3472
|
+
subjectId: z31.string().min(1),
|
|
3473
|
+
name: z31.string().trim().min(1).max(200).optional(),
|
|
2930
3474
|
...planShape,
|
|
2931
|
-
creditCap:
|
|
2932
|
-
|
|
2933
|
-
targetScore: z30.number().min(0).max(1).optional(),
|
|
3475
|
+
creditCap: z31.number().int().nonnegative().optional(),
|
|
3476
|
+
targetScore: z31.number().min(0).max(1).optional(),
|
|
2934
3477
|
target: targetSchema.extend({ agent: schedulableAgentSchema })
|
|
2935
3478
|
}).strict().superRefine(refinePlan);
|
|
2936
|
-
var estimateOptimizationBodySchema =
|
|
3479
|
+
var estimateOptimizationBodySchema = z31.object({
|
|
2937
3480
|
subjectKind: climbSubjectKindSchema,
|
|
2938
|
-
subjectId:
|
|
3481
|
+
subjectId: z31.string().min(1),
|
|
2939
3482
|
...planShape,
|
|
2940
3483
|
target: targetSchema.optional()
|
|
2941
3484
|
}).strict().superRefine(refinePlan);
|
|
@@ -2947,98 +3490,100 @@ var RUN_CREDIT_BASES = [
|
|
|
2947
3490
|
"byok",
|
|
2948
3491
|
"reserve"
|
|
2949
3492
|
];
|
|
2950
|
-
var runCreditBasisSchema =
|
|
2951
|
-
var optimizationEstimateSchema =
|
|
2952
|
-
sampleCount:
|
|
2953
|
-
maxRounds:
|
|
2954
|
-
runsPerTrial:
|
|
2955
|
-
executionCredits:
|
|
2956
|
-
judgeCredits:
|
|
2957
|
-
leverPassCredits:
|
|
2958
|
-
trialCredits:
|
|
3493
|
+
var runCreditBasisSchema = z31.enum(RUN_CREDIT_BASES);
|
|
3494
|
+
var optimizationEstimateSchema = z31.object({
|
|
3495
|
+
sampleCount: z31.number().int(),
|
|
3496
|
+
maxRounds: z31.number().int(),
|
|
3497
|
+
runsPerTrial: z31.number(),
|
|
3498
|
+
executionCredits: z31.number(),
|
|
3499
|
+
judgeCredits: z31.number(),
|
|
3500
|
+
leverPassCredits: z31.number(),
|
|
3501
|
+
trialCredits: z31.number(),
|
|
2959
3502
|
/** Estimate, not a reservation or enforced spending cap. */
|
|
2960
|
-
estimate:
|
|
3503
|
+
estimate: z31.number().int(),
|
|
2961
3504
|
/** True when no price or history covered a model: a bound, not a forecast. */
|
|
2962
|
-
approximate:
|
|
3505
|
+
approximate: z31.boolean().default(false),
|
|
2963
3506
|
/** Which rung priced the run, so the UI can say why a figure is what it is. */
|
|
2964
3507
|
basis: runCreditBasisSchema.optional()
|
|
2965
3508
|
});
|
|
2966
|
-
var diagnoseBodySchema =
|
|
3509
|
+
var diagnoseBodySchema = z31.object({
|
|
2967
3510
|
subjectKind: climbSubjectKindSchema,
|
|
2968
|
-
subjectId:
|
|
3511
|
+
subjectId: z31.string().min(1)
|
|
2969
3512
|
}).strict();
|
|
2970
|
-
var diagnosisSchema =
|
|
2971
|
-
id:
|
|
2972
|
-
status:
|
|
3513
|
+
var diagnosisSchema = z31.object({
|
|
3514
|
+
id: z31.string(),
|
|
3515
|
+
status: z31.enum(["GENERATING", "READY", "FAILED"]),
|
|
2973
3516
|
subjectKind: climbSubjectKindSchema,
|
|
2974
|
-
subjectId:
|
|
2975
|
-
runCount:
|
|
2976
|
-
summary:
|
|
3517
|
+
subjectId: z31.string(),
|
|
3518
|
+
runCount: z31.number().int(),
|
|
3519
|
+
summary: z31.string().nullable(),
|
|
2977
3520
|
recommendedLever: leverKindSchema.nullable(),
|
|
2978
|
-
recommendedTarget:
|
|
2979
|
-
clusters:
|
|
2980
|
-
|
|
2981
|
-
title:
|
|
2982
|
-
failureMode:
|
|
2983
|
-
criterionIds:
|
|
2984
|
-
runIds:
|
|
3521
|
+
recommendedTarget: z31.unknown().nullable(),
|
|
3522
|
+
clusters: z31.array(
|
|
3523
|
+
z31.object({
|
|
3524
|
+
title: z31.string(),
|
|
3525
|
+
failureMode: z31.string(),
|
|
3526
|
+
criterionIds: z31.array(z31.string()),
|
|
3527
|
+
runIds: z31.array(z31.string()),
|
|
2985
3528
|
lever: leverKindSchema.nullable(),
|
|
2986
|
-
targetRef:
|
|
2987
|
-
confidence:
|
|
3529
|
+
targetRef: z31.unknown().nullable(),
|
|
3530
|
+
confidence: z31.number()
|
|
2988
3531
|
})
|
|
2989
3532
|
),
|
|
2990
|
-
unsupportedLevers:
|
|
2991
|
-
createdAt:
|
|
2992
|
-
});
|
|
2993
|
-
var trialRunSchema =
|
|
2994
|
-
id:
|
|
2995
|
-
status:
|
|
2996
|
-
passed:
|
|
2997
|
-
criteria:
|
|
2998
|
-
|
|
3533
|
+
unsupportedLevers: z31.array(leverKindSchema).default([]),
|
|
3534
|
+
createdAt: z31.string()
|
|
3535
|
+
});
|
|
3536
|
+
var trialRunSchema = z31.object({
|
|
3537
|
+
id: z31.string(),
|
|
3538
|
+
status: z31.string(),
|
|
3539
|
+
passed: z31.boolean().nullable(),
|
|
3540
|
+
criteria: z31.array(
|
|
3541
|
+
z31.object({ name: z31.string(), passed: z31.boolean(), reasoning: z31.string() })
|
|
2999
3542
|
),
|
|
3000
|
-
levers:
|
|
3001
|
-
|
|
3002
|
-
lever:
|
|
3003
|
-
attributed:
|
|
3004
|
-
note:
|
|
3543
|
+
levers: z31.array(
|
|
3544
|
+
z31.object({
|
|
3545
|
+
lever: z31.string(),
|
|
3546
|
+
attributed: z31.boolean(),
|
|
3547
|
+
note: z31.string().nullable()
|
|
3005
3548
|
})
|
|
3006
3549
|
)
|
|
3007
3550
|
});
|
|
3008
|
-
var trialSchema =
|
|
3009
|
-
id:
|
|
3010
|
-
climbId:
|
|
3011
|
-
roundNumber:
|
|
3012
|
-
key:
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3551
|
+
var trialSchema = z31.object({
|
|
3552
|
+
id: z31.string(),
|
|
3553
|
+
climbId: z31.string(),
|
|
3554
|
+
roundNumber: z31.number().int(),
|
|
3555
|
+
key: z31.string(),
|
|
3556
|
+
/** @deprecated All trials branch from the baseline. */
|
|
3557
|
+
parentTrialId: z31.string().nullable(),
|
|
3558
|
+
/** @deprecated Always the baseline key for non-baseline trials. */
|
|
3559
|
+
parentKey: z31.string().nullable(),
|
|
3560
|
+
title: z31.string(),
|
|
3561
|
+
hypothesis: z31.string().nullable(),
|
|
3017
3562
|
lever: leverKindSchema.nullable(),
|
|
3018
3563
|
kind: interventionKindSchema,
|
|
3019
|
-
status:
|
|
3020
|
-
verdict:
|
|
3021
|
-
score:
|
|
3022
|
-
baselineScore:
|
|
3564
|
+
status: z31.enum(TRIAL_STATUSES),
|
|
3565
|
+
verdict: z31.enum(TRIAL_VERDICTS).nullable(),
|
|
3566
|
+
score: z31.number().nullable(),
|
|
3567
|
+
baselineScore: z31.number().nullable(),
|
|
3023
3568
|
/** Judged sessions behind `score`. */
|
|
3024
|
-
sessions:
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
adopted:
|
|
3028
|
-
adoptedAt:
|
|
3029
|
-
outcome:
|
|
3569
|
+
sessions: z31.number().int(),
|
|
3570
|
+
isBaseline: z31.boolean(),
|
|
3571
|
+
/** The user marked this trial as the winner; it does not move the baseline. */
|
|
3572
|
+
adopted: z31.boolean(),
|
|
3573
|
+
adoptedAt: z31.string().nullable(),
|
|
3574
|
+
outcome: z31.string().nullable(),
|
|
3030
3575
|
/** The kind-specific `after` payload (skill version id, pages, package). */
|
|
3031
|
-
after:
|
|
3032
|
-
before:
|
|
3033
|
-
runs:
|
|
3034
|
-
createdAt:
|
|
3035
|
-
});
|
|
3036
|
-
var roundSchema =
|
|
3037
|
-
id:
|
|
3038
|
-
number:
|
|
3039
|
-
status:
|
|
3040
|
-
narrative:
|
|
3041
|
-
trials:
|
|
3576
|
+
after: z31.unknown(),
|
|
3577
|
+
before: z31.unknown(),
|
|
3578
|
+
runs: z31.array(trialRunSchema),
|
|
3579
|
+
createdAt: z31.string()
|
|
3580
|
+
});
|
|
3581
|
+
var roundSchema = z31.object({
|
|
3582
|
+
id: z31.string(),
|
|
3583
|
+
number: z31.number().int(),
|
|
3584
|
+
status: z31.enum(ROUND_STATUSES),
|
|
3585
|
+
narrative: z31.string().nullable(),
|
|
3586
|
+
trials: z31.array(trialSchema)
|
|
3042
3587
|
});
|
|
3043
3588
|
var OPTIMIZATION_NEXT_ACTIONS = [
|
|
3044
3589
|
"START",
|
|
@@ -3049,474 +3594,880 @@ var OPTIMIZATION_NEXT_ACTIONS = [
|
|
|
3049
3594
|
"RESUME",
|
|
3050
3595
|
"NONE"
|
|
3051
3596
|
];
|
|
3052
|
-
var optimizationNextActionSchema =
|
|
3597
|
+
var optimizationNextActionSchema = z31.enum(OPTIMIZATION_NEXT_ACTIONS);
|
|
3053
3598
|
var optimizationDetailSchema = optimizationSummarySchema.extend({
|
|
3054
|
-
baselineTrialId:
|
|
3599
|
+
baselineTrialId: z31.string().nullable(),
|
|
3055
3600
|
/** The one thing a client does next. WAIT and NONE need nobody. */
|
|
3056
3601
|
nextAction: optimizationNextActionSchema,
|
|
3057
|
-
settings:
|
|
3058
|
-
sampleCount:
|
|
3059
|
-
maxRounds:
|
|
3060
|
-
maxTrialsPerRound:
|
|
3061
|
-
creditCap:
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
pinnedModel: z30.string().nullable(),
|
|
3067
|
-
continuedAtRound: z30.number().int().nullable()
|
|
3602
|
+
settings: z31.object({
|
|
3603
|
+
sampleCount: z31.number().int(),
|
|
3604
|
+
maxRounds: z31.number().int(),
|
|
3605
|
+
maxTrialsPerRound: z31.number().int(),
|
|
3606
|
+
creditCap: z31.number().int(),
|
|
3607
|
+
targetScore: z31.number(),
|
|
3608
|
+
pinnedAgent: z31.string().nullable(),
|
|
3609
|
+
pinnedModel: z31.string().nullable(),
|
|
3610
|
+
continuedAtRound: z31.number().int().nullable()
|
|
3068
3611
|
}),
|
|
3069
3612
|
direction: climbDirectionSchema,
|
|
3070
|
-
rounds:
|
|
3613
|
+
rounds: z31.array(roundSchema)
|
|
3071
3614
|
});
|
|
3072
|
-
var listTrialsResponseSchema =
|
|
3073
|
-
items:
|
|
3615
|
+
var listTrialsResponseSchema = z31.object({
|
|
3616
|
+
items: z31.array(trialSchema)
|
|
3074
3617
|
});
|
|
3075
|
-
var trialDocumentSchema =
|
|
3076
|
-
path:
|
|
3077
|
-
before:
|
|
3078
|
-
after:
|
|
3618
|
+
var trialDocumentSchema = z31.object({
|
|
3619
|
+
path: z31.string(),
|
|
3620
|
+
before: z31.string(),
|
|
3621
|
+
after: z31.string()
|
|
3079
3622
|
});
|
|
3080
|
-
var trialContentSchema =
|
|
3081
|
-
trialId:
|
|
3623
|
+
var trialContentSchema = z31.object({
|
|
3624
|
+
trialId: z31.string(),
|
|
3082
3625
|
kind: interventionKindSchema,
|
|
3083
|
-
documents:
|
|
3626
|
+
documents: z31.array(trialDocumentSchema),
|
|
3084
3627
|
/** Full page bodies for a docs rewrite, ready to paste; null otherwise. */
|
|
3085
|
-
copyableBody:
|
|
3086
|
-
configSnapshot:
|
|
3628
|
+
copyableBody: z31.string().nullable(),
|
|
3629
|
+
configSnapshot: z31.unknown()
|
|
3087
3630
|
});
|
|
3088
|
-
var createTrialBodySchema =
|
|
3631
|
+
var createTrialBodySchema = z31.object({
|
|
3089
3632
|
trial: trialInputSchema,
|
|
3090
3633
|
files: overrideFilesSchema.optional(),
|
|
3091
|
-
fork:
|
|
3092
|
-
launch:
|
|
3634
|
+
fork: z31.object({ runId: z31.string().min(1), boundarySeq: z31.string().min(1) }).strict().optional(),
|
|
3635
|
+
launch: z31.boolean().default(false),
|
|
3093
3636
|
/** Replays return the trial the first call created instead of a second
|
|
3094
3637
|
* one. Scoped to the optimization. */
|
|
3095
|
-
idempotencyKey:
|
|
3638
|
+
idempotencyKey: z31.string().min(1).max(200).optional()
|
|
3096
3639
|
}).strict();
|
|
3097
|
-
var skillFilesResponseSchema =
|
|
3098
|
-
versionId:
|
|
3640
|
+
var skillFilesResponseSchema = z31.object({
|
|
3641
|
+
versionId: z31.string(),
|
|
3099
3642
|
files: overrideFilesSchema
|
|
3100
3643
|
});
|
|
3101
|
-
var pageBodyResponseSchema =
|
|
3102
|
-
url:
|
|
3103
|
-
body:
|
|
3104
|
-
contentType:
|
|
3644
|
+
var pageBodyResponseSchema = z31.object({
|
|
3645
|
+
url: z31.string(),
|
|
3646
|
+
body: z31.string(),
|
|
3647
|
+
contentType: z31.string().optional()
|
|
3105
3648
|
});
|
|
3106
|
-
var launchTrialResponseSchema =
|
|
3107
|
-
trialId:
|
|
3108
|
-
batchIds:
|
|
3109
|
-
runIds:
|
|
3649
|
+
var launchTrialResponseSchema = z31.object({
|
|
3650
|
+
trialId: z31.string(),
|
|
3651
|
+
batchIds: z31.array(z31.string()),
|
|
3652
|
+
runIds: z31.array(z31.string())
|
|
3110
3653
|
});
|
|
3111
|
-
var createDraftOptimizationBodySchema =
|
|
3654
|
+
var createDraftOptimizationBodySchema = z31.object({
|
|
3112
3655
|
subjectKind: climbSubjectKindSchema,
|
|
3113
|
-
subjectId:
|
|
3114
|
-
name:
|
|
3656
|
+
subjectId: z31.string().min(1),
|
|
3657
|
+
name: z31.string().trim().min(1).max(200).optional(),
|
|
3115
3658
|
...planShape,
|
|
3116
3659
|
target: targetSchema.optional(),
|
|
3117
|
-
autonomous:
|
|
3660
|
+
autonomous: z31.boolean().optional()
|
|
3118
3661
|
}).strict().superRefine(refinePlan);
|
|
3119
3662
|
var planPatchShape = {
|
|
3120
|
-
name:
|
|
3663
|
+
name: z31.string().trim().min(1).max(200).optional(),
|
|
3121
3664
|
...planShape,
|
|
3122
3665
|
target: targetSchema.optional()
|
|
3123
3666
|
};
|
|
3124
|
-
var optimizationPlanPatchSchema =
|
|
3125
|
-
var startDraftOptimizationBodySchema =
|
|
3667
|
+
var optimizationPlanPatchSchema = z31.object(planPatchShape).strict().superRefine(refinePlan);
|
|
3668
|
+
var startDraftOptimizationBodySchema = z31.object({
|
|
3126
3669
|
...planPatchShape,
|
|
3127
|
-
creditCap:
|
|
3670
|
+
creditCap: z31.number().int().nonnegative().optional()
|
|
3128
3671
|
}).strict().superRefine(refinePlan);
|
|
3129
|
-
var setModeBodySchema =
|
|
3130
|
-
var launchRoundResponseSchema =
|
|
3131
|
-
roundNumber:
|
|
3132
|
-
launched:
|
|
3672
|
+
var setModeBodySchema = z31.object({ autonomous: z31.boolean() }).strict();
|
|
3673
|
+
var launchRoundResponseSchema = z31.object({
|
|
3674
|
+
roundNumber: z31.number().int().nullable(),
|
|
3675
|
+
launched: z31.array(z31.string()),
|
|
3133
3676
|
/** True when nothing was left to launch: someone else got there first. */
|
|
3134
|
-
alreadyLaunched:
|
|
3677
|
+
alreadyLaunched: z31.boolean()
|
|
3135
3678
|
});
|
|
3136
|
-
var resolveChatResponseSchema =
|
|
3137
|
-
chatId:
|
|
3138
|
-
href:
|
|
3679
|
+
var resolveChatResponseSchema = z31.object({
|
|
3680
|
+
chatId: z31.string(),
|
|
3681
|
+
href: z31.string()
|
|
3139
3682
|
});
|
|
3140
|
-
var continueOptimizationBodySchema =
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
maxTrialsPerRound: z30.number().int().min(1).max(3).optional()
|
|
3683
|
+
var continueOptimizationBodySchema = z31.object({
|
|
3684
|
+
rounds: z31.number().int().min(1).max(20).default(1),
|
|
3685
|
+
maxTrialsPerRound: z31.number().int().min(1).max(10).optional()
|
|
3144
3686
|
}).strict();
|
|
3145
|
-
var patchOptimizationBodySchema =
|
|
3687
|
+
var patchOptimizationBodySchema = z31.object({
|
|
3146
3688
|
...planPatchShape,
|
|
3147
3689
|
direction: climbDirectionPatchSchema.optional(),
|
|
3148
|
-
autonomous:
|
|
3690
|
+
autonomous: z31.boolean().optional()
|
|
3149
3691
|
}).strict().superRefine(refinePlan);
|
|
3150
3692
|
|
|
3151
3693
|
// ../packages/api-schemas/src/orgs.ts
|
|
3152
|
-
import { z as
|
|
3153
|
-
var createOrgRequestSchema =
|
|
3694
|
+
import { z as z32 } from "zod";
|
|
3695
|
+
var createOrgRequestSchema = z32.object({
|
|
3154
3696
|
/** Workspace display name; the slug is derived server-side. */
|
|
3155
|
-
name:
|
|
3697
|
+
name: z32.string().trim().min(1).max(200),
|
|
3156
3698
|
/** Optional company site (logo source; stamped onto the org). */
|
|
3157
|
-
website:
|
|
3699
|
+
website: z32.string().trim().max(300).optional()
|
|
3158
3700
|
});
|
|
3159
|
-
var createOrgResponseSchema =
|
|
3160
|
-
slug:
|
|
3161
|
-
name:
|
|
3701
|
+
var createOrgResponseSchema = z32.object({
|
|
3702
|
+
slug: z32.string(),
|
|
3703
|
+
name: z32.string(),
|
|
3162
3704
|
// LOCAL orgs never have a Gauge publicId.
|
|
3163
|
-
publicId:
|
|
3705
|
+
publicId: z32.string().nullable()
|
|
3164
3706
|
});
|
|
3165
|
-
var orgSettingsSchema =
|
|
3166
|
-
slug:
|
|
3167
|
-
name:
|
|
3168
|
-
publicId:
|
|
3169
|
-
website:
|
|
3707
|
+
var orgSettingsSchema = z32.object({
|
|
3708
|
+
slug: z32.string(),
|
|
3709
|
+
name: z32.string(),
|
|
3710
|
+
publicId: z32.string().nullable(),
|
|
3711
|
+
website: z32.string().nullable(),
|
|
3170
3712
|
/** Per-run max-spend override in USD; null = the server default. */
|
|
3171
|
-
maxSpendUsd:
|
|
3172
|
-
overageEnabled:
|
|
3713
|
+
maxSpendUsd: z32.number().nullable(),
|
|
3714
|
+
overageEnabled: z32.boolean(),
|
|
3173
3715
|
/** Per-cycle overage cap in credits; null = uncapped. */
|
|
3174
|
-
overageCapCredits:
|
|
3716
|
+
overageCapCredits: z32.number().int().nullable(),
|
|
3175
3717
|
/** Domain claimed for same-domain auto-join; null = off. */
|
|
3176
|
-
autoJoinDomain:
|
|
3718
|
+
autoJoinDomain: z32.string().nullable()
|
|
3177
3719
|
});
|
|
3178
|
-
var orgMemberSchema =
|
|
3179
|
-
userId:
|
|
3180
|
-
name:
|
|
3181
|
-
email:
|
|
3182
|
-
joinedAt:
|
|
3720
|
+
var orgMemberSchema = z32.object({
|
|
3721
|
+
userId: z32.string(),
|
|
3722
|
+
name: z32.string().nullable(),
|
|
3723
|
+
email: z32.string().nullable(),
|
|
3724
|
+
joinedAt: z32.string()
|
|
3183
3725
|
});
|
|
3184
|
-
var listMembersResponseSchema =
|
|
3185
|
-
items:
|
|
3726
|
+
var listMembersResponseSchema = z32.object({
|
|
3727
|
+
items: z32.array(orgMemberSchema)
|
|
3186
3728
|
});
|
|
3187
3729
|
|
|
3188
3730
|
// ../packages/api-schemas/src/personas.ts
|
|
3189
|
-
import { z as
|
|
3190
|
-
var personaSchema =
|
|
3191
|
-
id:
|
|
3192
|
-
name:
|
|
3193
|
-
description:
|
|
3731
|
+
import { z as z33 } from "zod";
|
|
3732
|
+
var personaSchema = z33.object({
|
|
3733
|
+
id: z33.string(),
|
|
3734
|
+
name: z33.string(),
|
|
3735
|
+
description: z33.string(),
|
|
3194
3736
|
/** The judge model this persona calls (Claude only). */
|
|
3195
|
-
model:
|
|
3196
|
-
createdAt:
|
|
3197
|
-
updatedAt:
|
|
3737
|
+
model: z33.string(),
|
|
3738
|
+
createdAt: z33.string(),
|
|
3739
|
+
updatedAt: z33.string()
|
|
3198
3740
|
});
|
|
3199
|
-
var listPersonasResponseSchema =
|
|
3200
|
-
items:
|
|
3741
|
+
var listPersonasResponseSchema = z33.object({
|
|
3742
|
+
items: z33.array(personaSchema)
|
|
3201
3743
|
});
|
|
3202
|
-
var createPersonaBodySchema =
|
|
3203
|
-
name:
|
|
3204
|
-
description:
|
|
3744
|
+
var createPersonaBodySchema = z33.object({
|
|
3745
|
+
name: z33.string().min(1).max(120),
|
|
3746
|
+
description: z33.string().min(1)
|
|
3205
3747
|
});
|
|
3206
|
-
var updatePersonaBodySchema =
|
|
3207
|
-
name:
|
|
3208
|
-
description:
|
|
3748
|
+
var updatePersonaBodySchema = z33.object({
|
|
3749
|
+
name: z33.string().min(1).max(120).optional(),
|
|
3750
|
+
description: z33.string().min(1).optional()
|
|
3209
3751
|
});
|
|
3210
3752
|
|
|
3211
3753
|
// ../packages/api-schemas/src/prompts.ts
|
|
3212
|
-
import { z as
|
|
3754
|
+
import { z as z36 } from "zod";
|
|
3213
3755
|
|
|
3214
3756
|
// ../packages/api-schemas/src/tags.ts
|
|
3215
|
-
import { z as
|
|
3216
|
-
var tagSchema =
|
|
3217
|
-
id:
|
|
3218
|
-
name:
|
|
3757
|
+
import { z as z34 } from "zod";
|
|
3758
|
+
var tagSchema = z34.object({
|
|
3759
|
+
id: z34.string(),
|
|
3760
|
+
name: z34.string(),
|
|
3219
3761
|
/** A Mantine palette name (e.g. "blue", "grape"); unknown values are
|
|
3220
3762
|
* normalized server-side to the default ("gray"). */
|
|
3221
|
-
color:
|
|
3763
|
+
color: z34.string()
|
|
3222
3764
|
});
|
|
3223
|
-
var createTagBodySchema =
|
|
3224
|
-
name:
|
|
3225
|
-
color:
|
|
3765
|
+
var createTagBodySchema = z34.object({
|
|
3766
|
+
name: z34.string().min(1),
|
|
3767
|
+
color: z34.string().optional()
|
|
3226
3768
|
});
|
|
3227
|
-
var updateTagBodySchema =
|
|
3228
|
-
name:
|
|
3229
|
-
color:
|
|
3769
|
+
var updateTagBodySchema = z34.object({
|
|
3770
|
+
name: z34.string().min(1).optional(),
|
|
3771
|
+
color: z34.string().optional()
|
|
3230
3772
|
});
|
|
3231
3773
|
var tagListResponseSchema = listResponseSchema(tagSchema);
|
|
3232
3774
|
|
|
3233
3775
|
// ../packages/api-schemas/src/topics.ts
|
|
3234
|
-
import { z as
|
|
3235
|
-
var topicSchema =
|
|
3236
|
-
id:
|
|
3237
|
-
name:
|
|
3776
|
+
import { z as z35 } from "zod";
|
|
3777
|
+
var topicSchema = z35.object({
|
|
3778
|
+
id: z35.string(),
|
|
3779
|
+
name: z35.string()
|
|
3238
3780
|
});
|
|
3239
|
-
var createTopicBodySchema =
|
|
3240
|
-
name:
|
|
3781
|
+
var createTopicBodySchema = z35.object({
|
|
3782
|
+
name: z35.string().min(1, "name is required")
|
|
3241
3783
|
});
|
|
3242
3784
|
var topicListResponseSchema = listResponseSchema(topicSchema);
|
|
3243
3785
|
|
|
3244
3786
|
// ../packages/api-schemas/src/prompts.ts
|
|
3245
|
-
var promptSummarySchema =
|
|
3246
|
-
id:
|
|
3247
|
-
text:
|
|
3248
|
-
isBranded:
|
|
3787
|
+
var promptSummarySchema = z36.object({
|
|
3788
|
+
id: z36.string(),
|
|
3789
|
+
text: z36.string(),
|
|
3790
|
+
isBranded: z36.boolean(),
|
|
3249
3791
|
/** Battery-spec identity (idempotent applies); null for UI/CLI-created prompts. */
|
|
3250
|
-
externalKey:
|
|
3792
|
+
externalKey: z36.string().nullable(),
|
|
3251
3793
|
topic: topicSchema.nullable(),
|
|
3252
|
-
tags:
|
|
3253
|
-
createdAt:
|
|
3794
|
+
tags: z36.array(tagSchema),
|
|
3795
|
+
createdAt: z36.string()
|
|
3254
3796
|
});
|
|
3255
3797
|
var promptListResponseSchema = listResponseSchema(promptSummarySchema);
|
|
3256
3798
|
var listPromptsQuerySchema = listQuerySchema.extend({
|
|
3257
|
-
topic:
|
|
3258
|
-
tag:
|
|
3259
|
-
q:
|
|
3260
|
-
});
|
|
3261
|
-
var createPromptBodySchema =
|
|
3262
|
-
text:
|
|
3263
|
-
isBranded:
|
|
3264
|
-
topic:
|
|
3265
|
-
tags:
|
|
3266
|
-
});
|
|
3267
|
-
var updatePromptBodySchema =
|
|
3268
|
-
text:
|
|
3269
|
-
isBranded:
|
|
3270
|
-
topicId:
|
|
3271
|
-
tags:
|
|
3272
|
-
});
|
|
3273
|
-
var runPromptBodySchema =
|
|
3799
|
+
topic: z36.string().min(1).optional(),
|
|
3800
|
+
tag: z36.string().min(1).optional(),
|
|
3801
|
+
q: z36.string().min(1).optional()
|
|
3802
|
+
});
|
|
3803
|
+
var createPromptBodySchema = z36.object({
|
|
3804
|
+
text: z36.string().min(1, "text is required"),
|
|
3805
|
+
isBranded: z36.boolean().default(false),
|
|
3806
|
+
topic: z36.string().min(1).optional(),
|
|
3807
|
+
tags: z36.array(z36.string().min(1)).default([])
|
|
3808
|
+
});
|
|
3809
|
+
var updatePromptBodySchema = z36.object({
|
|
3810
|
+
text: z36.string().min(1).optional(),
|
|
3811
|
+
isBranded: z36.boolean().optional(),
|
|
3812
|
+
topicId: z36.string().min(1).nullish(),
|
|
3813
|
+
tags: z36.array(z36.string().min(1)).optional()
|
|
3814
|
+
});
|
|
3815
|
+
var runPromptBodySchema = z36.object({
|
|
3274
3816
|
/** Preferred paired representation; each agent may pin its own model tiers. */
|
|
3275
|
-
agentConfigs:
|
|
3817
|
+
agentConfigs: z36.array(agentConfigInputSchema).min(1).optional(),
|
|
3276
3818
|
/** Legacy agent set (no per-agent pins), kept during the API transition. */
|
|
3277
|
-
agents:
|
|
3278
|
-
repoUrl:
|
|
3279
|
-
repoRef:
|
|
3280
|
-
samplesPerCycle:
|
|
3819
|
+
agents: z36.array(z36.enum(["CLAUDE_CODE", "CODEX_CLI", "CURSOR"])).min(1, "Select at least one agent").default(["CLAUDE_CODE", "CODEX_CLI"]),
|
|
3820
|
+
repoUrl: z36.string().min(1).nullish(),
|
|
3821
|
+
repoRef: z36.string().min(1).nullish(),
|
|
3822
|
+
samplesPerCycle: z36.number().int().min(1).max(10).default(1),
|
|
3281
3823
|
webFixture: webFixtureSchema.optional()
|
|
3282
3824
|
});
|
|
3283
|
-
var runPromptResponseSchema =
|
|
3284
|
-
batchIds:
|
|
3285
|
-
runIds:
|
|
3286
|
-
blocked:
|
|
3825
|
+
var runPromptResponseSchema = z36.object({
|
|
3826
|
+
batchIds: z36.array(z36.string()),
|
|
3827
|
+
runIds: z36.array(z36.string()),
|
|
3828
|
+
blocked: z36.enum(["insufficient_credits", "needs_provider_key"]).optional()
|
|
3287
3829
|
});
|
|
3288
3830
|
|
|
3289
3831
|
// ../packages/api-schemas/src/repos.ts
|
|
3290
|
-
import { z as
|
|
3832
|
+
import { z as z37 } from "zod";
|
|
3291
3833
|
var REPO_SIZE_BUCKETS = [
|
|
3292
3834
|
"small",
|
|
3293
3835
|
"medium",
|
|
3294
3836
|
"large",
|
|
3295
3837
|
"xlarge"
|
|
3296
3838
|
];
|
|
3297
|
-
var repoSizeBucketSchema =
|
|
3298
|
-
var repoSchema =
|
|
3299
|
-
url:
|
|
3300
|
-
name:
|
|
3301
|
-
isSystem:
|
|
3302
|
-
languages:
|
|
3303
|
-
frameworks:
|
|
3839
|
+
var repoSizeBucketSchema = z37.enum(REPO_SIZE_BUCKETS);
|
|
3840
|
+
var repoSchema = z37.object({
|
|
3841
|
+
url: z37.string(),
|
|
3842
|
+
name: z37.string(),
|
|
3843
|
+
isSystem: z37.boolean(),
|
|
3844
|
+
languages: z37.array(z37.string()),
|
|
3845
|
+
frameworks: z37.array(z37.string()),
|
|
3304
3846
|
/** Faceted labels (complexity:L2, buildable:yes, stack:nextjs, …). */
|
|
3305
|
-
tags:
|
|
3306
|
-
primaryLanguage:
|
|
3307
|
-
sizeKb:
|
|
3847
|
+
tags: z37.array(z37.string()),
|
|
3848
|
+
primaryLanguage: z37.string(),
|
|
3849
|
+
sizeKb: z37.number(),
|
|
3308
3850
|
sizeBucket: repoSizeBucketSchema.nullable(),
|
|
3309
|
-
stars:
|
|
3851
|
+
stars: z37.number()
|
|
3310
3852
|
});
|
|
3311
3853
|
var repoListResponseSchema = listResponseSchema(repoSchema);
|
|
3312
|
-
var addRepoBodySchema =
|
|
3313
|
-
url:
|
|
3854
|
+
var addRepoBodySchema = z37.object({
|
|
3855
|
+
url: z37.string().min(1, "url is required"),
|
|
3314
3856
|
/**
|
|
3315
3857
|
* Reserved (accepted, not yet persisted): the catalog keys repos by URL only
|
|
3316
3858
|
* — `fact_repo` has no ref column and metadata sync always reads the default
|
|
3317
3859
|
* branch. Kept in the wire format so `gauge repos add --ref` is stable once
|
|
3318
3860
|
* the catalog learns refs.
|
|
3319
3861
|
*/
|
|
3320
|
-
ref:
|
|
3862
|
+
ref: z37.string().min(1).optional(),
|
|
3321
3863
|
/** Display name; defaults to the `owner/repo` label of the URL. */
|
|
3322
|
-
name:
|
|
3864
|
+
name: z37.string().min(1).optional(),
|
|
3323
3865
|
/** Manual override; defaults to frameworks detected from the repo's manifests. */
|
|
3324
|
-
frameworks:
|
|
3325
|
-
tags:
|
|
3866
|
+
frameworks: z37.array(z37.string()).optional(),
|
|
3867
|
+
tags: z37.array(z37.string()).optional()
|
|
3326
3868
|
});
|
|
3327
|
-
var updateRepoBodySchema =
|
|
3328
|
-
url:
|
|
3329
|
-
name:
|
|
3330
|
-
frameworks:
|
|
3331
|
-
tags:
|
|
3869
|
+
var updateRepoBodySchema = z37.object({
|
|
3870
|
+
url: z37.string().min(1, "url is required"),
|
|
3871
|
+
name: z37.string().min(1).optional(),
|
|
3872
|
+
frameworks: z37.array(z37.string()).optional(),
|
|
3873
|
+
tags: z37.array(z37.string()).optional()
|
|
3332
3874
|
});
|
|
3333
|
-
var resyncRepoBodySchema =
|
|
3334
|
-
url:
|
|
3875
|
+
var resyncRepoBodySchema = z37.object({
|
|
3876
|
+
url: z37.string().min(1, "url is required")
|
|
3335
3877
|
});
|
|
3336
|
-
var deleteRepoQuerySchema =
|
|
3337
|
-
url:
|
|
3878
|
+
var deleteRepoQuerySchema = z37.object({
|
|
3879
|
+
url: z37.string().min(1, "url is required")
|
|
3338
3880
|
});
|
|
3339
|
-
var repoSyncResultSchema =
|
|
3881
|
+
var repoSyncResultSchema = z37.object({
|
|
3340
3882
|
repo: repoSchema,
|
|
3341
|
-
synced:
|
|
3883
|
+
synced: z37.boolean()
|
|
3884
|
+
});
|
|
3885
|
+
|
|
3886
|
+
// ../packages/api-schemas/src/runAddons.ts
|
|
3887
|
+
import { z as z38 } from "zod";
|
|
3888
|
+
var runAddonsSchema = z38.object({
|
|
3889
|
+
browser: z38.object({
|
|
3890
|
+
session: z38.string().min(1),
|
|
3891
|
+
projectId: z38.number().int().positive(),
|
|
3892
|
+
region: z38.literal("us").default("us")
|
|
3893
|
+
}).strict().optional()
|
|
3894
|
+
}).strict();
|
|
3895
|
+
|
|
3896
|
+
// ../packages/api-schemas/src/runDefinitions.ts
|
|
3897
|
+
import { z as z40 } from "zod";
|
|
3898
|
+
|
|
3899
|
+
// ../packages/api-schemas/src/runPlan.ts
|
|
3900
|
+
import { z as z39 } from "zod";
|
|
3901
|
+
var runPlanCriterionSchema = z39.object({
|
|
3902
|
+
id: z39.string().min(1),
|
|
3903
|
+
name: z39.string(),
|
|
3904
|
+
rubric: z39.string()
|
|
3905
|
+
});
|
|
3906
|
+
var criteriaSnapshotV1Schema = z39.object({
|
|
3907
|
+
version: z39.literal(1),
|
|
3908
|
+
// File-authored criteria have no saved EvalSet; their identity is frozen
|
|
3909
|
+
// in the request and result rows retain the criterion text without an FK.
|
|
3910
|
+
evalSetId: z39.string().min(1).nullable(),
|
|
3911
|
+
criteria: z39.array(runPlanCriterionSchema)
|
|
3912
|
+
});
|
|
3913
|
+
var runPlanCaseSchema = z39.object({
|
|
3914
|
+
id: z39.string().min(1),
|
|
3915
|
+
prompt: z39.object({ id: z39.string().min(1), text: z39.string() }),
|
|
3916
|
+
workspace: z39.object({
|
|
3917
|
+
repoUrl: z39.string().nullable(),
|
|
3918
|
+
repoRef: z39.string().nullable()
|
|
3919
|
+
}),
|
|
3920
|
+
agents: z39.array(
|
|
3921
|
+
agentConfigSchema.extend({
|
|
3922
|
+
// Validated against the current catalog request schema when lowered.
|
|
3923
|
+
catalogSelections: z39.record(z39.string(), z39.unknown()).optional()
|
|
3924
|
+
})
|
|
3925
|
+
),
|
|
3926
|
+
samples: z39.number().int().min(1).max(50),
|
|
3927
|
+
evalSetId: z39.string().min(1).optional(),
|
|
3928
|
+
criteriaSnapshot: criteriaSnapshotV1Schema.optional(),
|
|
3929
|
+
sourceRepoCommit: z39.string().nullable().optional(),
|
|
3930
|
+
experiment: z39.string().nullable().optional(),
|
|
3931
|
+
membershipTagIds: z39.array(z39.string()).optional(),
|
|
3932
|
+
profile: z39.object({
|
|
3933
|
+
id: z39.string(),
|
|
3934
|
+
name: z39.string(),
|
|
3935
|
+
description: z39.string(),
|
|
3936
|
+
model: z39.string()
|
|
3937
|
+
}).nullable().optional(),
|
|
3938
|
+
skillSet: skillSetSchema.optional(),
|
|
3939
|
+
mcpSet: mcpSetSchema.optional(),
|
|
3940
|
+
connectionSet: connectionSetSchema.optional(),
|
|
3941
|
+
// The server's agentContextSnapshotV1Schema validates this at lowering.
|
|
3942
|
+
agentContext: z39.unknown().optional(),
|
|
3943
|
+
agentContextDigest: z39.string().nullable().optional(),
|
|
3944
|
+
addons: runAddonsSchema.optional(),
|
|
3945
|
+
/** Names of resources the coordinator must prepare before expansion. */
|
|
3946
|
+
inputs: z39.array(z39.string().min(1)).default([])
|
|
3947
|
+
});
|
|
3948
|
+
var builtDirectoryResourceSchema = z39.object({
|
|
3949
|
+
name: z39.string().min(1),
|
|
3950
|
+
type: z39.literal("built-directory"),
|
|
3951
|
+
source: z39.object({
|
|
3952
|
+
type: z39.literal("git-checkout"),
|
|
3953
|
+
repoUrl: z39.string().min(1),
|
|
3954
|
+
commit: z39.string().regex(/^[a-f0-9]{40}$/i)
|
|
3955
|
+
}),
|
|
3956
|
+
build: z39.object({
|
|
3957
|
+
runtime: z39.literal("node-24"),
|
|
3958
|
+
run: z39.string().min(1),
|
|
3959
|
+
output: z39.string().min(1),
|
|
3960
|
+
/** Declared command name plus args, invoked from a clean cwd before a session. */
|
|
3961
|
+
smoke: z39.array(z39.string().min(1)).min(1)
|
|
3962
|
+
}),
|
|
3963
|
+
expose: z39.object({
|
|
3964
|
+
type: z39.literal("command"),
|
|
3965
|
+
commands: z39.record(z39.string().min(1), z39.array(z39.string().min(1)).min(1))
|
|
3966
|
+
})
|
|
3967
|
+
}).superRefine((resource, ctx) => {
|
|
3968
|
+
if (!(resource.build.smoke[0] in resource.expose.commands))
|
|
3969
|
+
ctx.addIssue({
|
|
3970
|
+
code: "custom",
|
|
3971
|
+
path: ["build", "smoke", 0],
|
|
3972
|
+
message: "Smoke command must name an exposed command"
|
|
3973
|
+
});
|
|
3974
|
+
});
|
|
3975
|
+
var bundleRelativeScriptSchema = z39.string().min(1).refine(
|
|
3976
|
+
(path2) => !path2.startsWith("/") && !path2.includes("\\") && !path2.includes("\0") && !path2.split("/").includes("..") && path2.split("/").some((part) => part !== "" && part !== "."),
|
|
3977
|
+
"Install script must be a bundle-relative path without traversal"
|
|
3978
|
+
);
|
|
3979
|
+
var preparedDirectoryPrepareSchema = z39.union([
|
|
3980
|
+
z39.object({ run: z39.string().min(1) }).strict(),
|
|
3981
|
+
z39.object({
|
|
3982
|
+
type: z39.literal("npm-cli"),
|
|
3983
|
+
build: z39.string().min(1).optional()
|
|
3984
|
+
}).strict()
|
|
3985
|
+
]);
|
|
3986
|
+
var preparedDirectoryResourceSchema = z39.object({
|
|
3987
|
+
name: z39.string().min(1),
|
|
3988
|
+
type: z39.literal("prepared-directory"),
|
|
3989
|
+
source: z39.object({
|
|
3990
|
+
type: z39.literal("git-checkout"),
|
|
3991
|
+
repoUrl: z39.string().min(1),
|
|
3992
|
+
commit: z39.string().regex(/^[a-f0-9]{40}$/i)
|
|
3993
|
+
}).strict(),
|
|
3994
|
+
prepare: preparedDirectoryPrepareSchema,
|
|
3995
|
+
install: bundleRelativeScriptSchema.optional()
|
|
3996
|
+
}).strict().superRefine((resource, ctx) => {
|
|
3997
|
+
if ("type" in resource.prepare && resource.install)
|
|
3998
|
+
ctx.addIssue({
|
|
3999
|
+
code: "custom",
|
|
4000
|
+
path: ["install"],
|
|
4001
|
+
message: "npm-cli products cannot declare an install script"
|
|
4002
|
+
});
|
|
4003
|
+
});
|
|
4004
|
+
var runPlanResourceSchema = z39.union([
|
|
4005
|
+
builtDirectoryResourceSchema,
|
|
4006
|
+
preparedDirectoryResourceSchema
|
|
4007
|
+
]);
|
|
4008
|
+
var runPlanSchema = z39.object({
|
|
4009
|
+
version: z39.literal(1),
|
|
4010
|
+
/** Existing launches keep their current measurement semantics. */
|
|
4011
|
+
measurementMode: z39.enum(["EXISTING", "AGENT_EXPERIENCE"]),
|
|
4012
|
+
cases: z39.array(runPlanCaseSchema).min(1),
|
|
4013
|
+
resources: z39.array(runPlanResourceSchema).default([]),
|
|
4014
|
+
/** Server-populated context retained across deferred preparation. */
|
|
4015
|
+
launch: z39.object({
|
|
4016
|
+
origin: z39.object({
|
|
4017
|
+
initiator: z39.literal("API"),
|
|
4018
|
+
userId: z39.string().min(1),
|
|
4019
|
+
email: z39.string().nullable()
|
|
4020
|
+
}).strict(),
|
|
4021
|
+
funding: z39.enum(["ORG", "PLATFORM"]),
|
|
4022
|
+
queueClass: z39.literal("INTERACTIVE")
|
|
4023
|
+
}).strict().optional(),
|
|
4024
|
+
provenance: z39.object({
|
|
4025
|
+
definition: z39.object({
|
|
4026
|
+
repoUrl: z39.string().min(1),
|
|
4027
|
+
commit: z39.string().regex(/^[a-f0-9]{40}$/i),
|
|
4028
|
+
digest: z39.string().min(1),
|
|
4029
|
+
/** File submissions are asserted by the caller until server-side
|
|
4030
|
+
* committed-byte fetching is available. */
|
|
4031
|
+
verification: z39.literal("SUBMITTED_UNVERIFIED").optional()
|
|
4032
|
+
}).optional(),
|
|
4033
|
+
liveDependencies: z39.array(
|
|
4034
|
+
z39.object({
|
|
4035
|
+
name: z39.string().min(1),
|
|
4036
|
+
url: z39.string().url().optional()
|
|
4037
|
+
})
|
|
4038
|
+
).default([])
|
|
4039
|
+
})
|
|
4040
|
+
}).superRefine((plan, ctx) => {
|
|
4041
|
+
const resourceNames = new Set(
|
|
4042
|
+
plan.resources.map((resource) => resource.name)
|
|
4043
|
+
);
|
|
4044
|
+
for (const [index, item] of plan.cases.entries()) {
|
|
4045
|
+
if (plan.measurementMode === "AGENT_EXPERIENCE" && !item.agents.length)
|
|
4046
|
+
ctx.addIssue({
|
|
4047
|
+
code: "custom",
|
|
4048
|
+
path: ["cases", index, "agents"],
|
|
4049
|
+
message: "Agent Experience cases require a selected agent"
|
|
4050
|
+
});
|
|
4051
|
+
if (plan.measurementMode === "AGENT_EXPERIENCE" && !item.criteriaSnapshot?.criteria.length)
|
|
4052
|
+
ctx.addIssue({
|
|
4053
|
+
code: "custom",
|
|
4054
|
+
path: ["cases", index, "criteriaSnapshot"],
|
|
4055
|
+
message: "Agent Experience cases require criteria"
|
|
4056
|
+
});
|
|
4057
|
+
if (item.criteriaSnapshot && item.criteriaSnapshot.evalSetId !== (item.evalSetId ?? null))
|
|
4058
|
+
ctx.addIssue({
|
|
4059
|
+
code: "custom",
|
|
4060
|
+
path: ["cases", index, "criteriaSnapshot", "evalSetId"],
|
|
4061
|
+
message: "Criteria snapshot must belong to the case's eval"
|
|
4062
|
+
});
|
|
4063
|
+
for (const input of item.inputs) {
|
|
4064
|
+
if (!resourceNames.has(input))
|
|
4065
|
+
ctx.addIssue({
|
|
4066
|
+
code: "custom",
|
|
4067
|
+
path: ["cases", index, "inputs"],
|
|
4068
|
+
message: `Unknown resource: ${input}`
|
|
4069
|
+
});
|
|
4070
|
+
}
|
|
4071
|
+
}
|
|
4072
|
+
if (new Set(plan.cases.map((item) => item.id)).size !== plan.cases.length)
|
|
4073
|
+
ctx.addIssue({
|
|
4074
|
+
code: "custom",
|
|
4075
|
+
path: ["cases"],
|
|
4076
|
+
message: "Case ids must be unique"
|
|
4077
|
+
});
|
|
4078
|
+
if (resourceNames.size !== plan.resources.length)
|
|
4079
|
+
ctx.addIssue({
|
|
4080
|
+
code: "custom",
|
|
4081
|
+
path: ["resources"],
|
|
4082
|
+
message: "Resource names must be unique"
|
|
4083
|
+
});
|
|
4084
|
+
});
|
|
4085
|
+
|
|
4086
|
+
// ../packages/api-schemas/src/runDefinitions.ts
|
|
4087
|
+
var uncredentialedRepoUrlSchema = z40.string().min(1).refine((raw) => {
|
|
4088
|
+
if (!/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) return true;
|
|
4089
|
+
try {
|
|
4090
|
+
const url = new URL(raw);
|
|
4091
|
+
return (url.protocol === "ssh:" || !url.username) && !url.password && !url.search && !url.hash;
|
|
4092
|
+
} catch {
|
|
4093
|
+
return false;
|
|
4094
|
+
}
|
|
4095
|
+
}, "Repository URL must not contain embedded credentials or query parameters");
|
|
4096
|
+
var runDefinitionConfigSchema = z40.object({
|
|
4097
|
+
agents: z40.array(agentConfigInputSchema).min(1),
|
|
4098
|
+
sampleCount: z40.number().int().min(1).max(50).optional(),
|
|
4099
|
+
repoUrl: uncredentialedRepoUrlSchema.nullable().optional(),
|
|
4100
|
+
repoRef: z40.string().min(1).nullable().optional(),
|
|
4101
|
+
profileId: z40.string().min(1).nullable().optional(),
|
|
4102
|
+
skillRefs: assetRefListSchema.optional(),
|
|
4103
|
+
mcpRefs: assetRefListSchema.optional(),
|
|
4104
|
+
connectionIds: z40.array(z40.string().min(1)).max(32).optional(),
|
|
4105
|
+
addons: runAddonsSchema.optional()
|
|
4106
|
+
}).strict();
|
|
4107
|
+
var githubCasePathSchema = z40.string().regex(
|
|
4108
|
+
/^evals\/[A-Za-z0-9_./-]+\.md$/,
|
|
4109
|
+
"Expected an explicit evals/*.md path"
|
|
4110
|
+
).refine(
|
|
4111
|
+
(path2) => path2.split("/").every((part) => part !== "" && part !== "." && part !== ".."),
|
|
4112
|
+
"Case path must not contain traversal or empty segments"
|
|
4113
|
+
);
|
|
4114
|
+
var runDefinitionFileConfigSchema = z40.object({
|
|
4115
|
+
version: z40.literal(1).default(1),
|
|
4116
|
+
org: z40.string().min(1).optional(),
|
|
4117
|
+
product: z40.object({
|
|
4118
|
+
type: z40.literal("npm-cli"),
|
|
4119
|
+
build: z40.string().min(1).optional()
|
|
4120
|
+
}).strict().optional(),
|
|
4121
|
+
prepare: z40.string().min(1).optional(),
|
|
4122
|
+
install: bundleRelativeScriptSchema.optional(),
|
|
4123
|
+
github: z40.object({
|
|
4124
|
+
enabled: z40.literal(true),
|
|
4125
|
+
files: z40.array(githubCasePathSchema).min(1).max(20).refine(
|
|
4126
|
+
(paths) => new Set(paths).size === paths.length,
|
|
4127
|
+
"Case paths must be unique"
|
|
4128
|
+
)
|
|
4129
|
+
}).strict().optional()
|
|
4130
|
+
}).strict().superRefine((config, ctx) => {
|
|
4131
|
+
if (config.product && config.prepare)
|
|
4132
|
+
ctx.addIssue({
|
|
4133
|
+
code: "custom",
|
|
4134
|
+
path: ["product"],
|
|
4135
|
+
message: "product and prepare are alternatives"
|
|
4136
|
+
});
|
|
4137
|
+
if (config.product && config.install)
|
|
4138
|
+
ctx.addIssue({
|
|
4139
|
+
code: "custom",
|
|
4140
|
+
path: ["install"],
|
|
4141
|
+
message: "npm-cli products cannot declare an install script"
|
|
4142
|
+
});
|
|
4143
|
+
});
|
|
4144
|
+
var runDefinitionCriterionSchema = z40.object({ name: z40.string().min(1), rubric: z40.string().min(1) }).strict();
|
|
4145
|
+
var runDefinitionFrontmatterSchema = z40.object({
|
|
4146
|
+
criteria: z40.array(runDefinitionCriterionSchema).min(1),
|
|
4147
|
+
config: runDefinitionConfigSchema
|
|
4148
|
+
}).strict();
|
|
4149
|
+
var runDefinitionCaseSchema = z40.object({
|
|
4150
|
+
path: z40.string().min(1),
|
|
4151
|
+
prompt: z40.string().min(1),
|
|
4152
|
+
criteria: z40.array(runDefinitionCriterionSchema).min(1),
|
|
4153
|
+
config: runDefinitionConfigSchema
|
|
4154
|
+
}).strict();
|
|
4155
|
+
var runDefinitionSubmissionSchema = z40.object({
|
|
4156
|
+
version: z40.literal(1),
|
|
4157
|
+
source: z40.object({
|
|
4158
|
+
repoUrl: uncredentialedRepoUrlSchema,
|
|
4159
|
+
commit: z40.string().regex(/^[a-f0-9]{40}$/i),
|
|
4160
|
+
digest: z40.string().regex(/^sha256:[a-f0-9]{64}$/),
|
|
4161
|
+
verification: z40.literal("SUBMITTED_UNVERIFIED")
|
|
4162
|
+
}).strict(),
|
|
4163
|
+
resources: z40.array(preparedDirectoryResourceSchema).max(1).default([]),
|
|
4164
|
+
cases: z40.array(runDefinitionCaseSchema).min(1)
|
|
4165
|
+
}).strict().superRefine((definition, ctx) => {
|
|
4166
|
+
const paths = /* @__PURE__ */ new Set();
|
|
4167
|
+
for (const [index, item] of definition.cases.entries()) {
|
|
4168
|
+
if (paths.has(item.path))
|
|
4169
|
+
ctx.addIssue({
|
|
4170
|
+
code: "custom",
|
|
4171
|
+
path: ["cases", index, "path"],
|
|
4172
|
+
message: "Case paths must be unique"
|
|
4173
|
+
});
|
|
4174
|
+
paths.add(item.path);
|
|
4175
|
+
}
|
|
4176
|
+
for (const [index, resource] of definition.resources.entries()) {
|
|
4177
|
+
if (resource.name !== "project")
|
|
4178
|
+
ctx.addIssue({
|
|
4179
|
+
code: "custom",
|
|
4180
|
+
path: ["resources", index, "name"],
|
|
4181
|
+
message: "The project preparation must use the reserved name project"
|
|
4182
|
+
});
|
|
4183
|
+
if (resource.source.repoUrl !== definition.source.repoUrl || resource.source.commit !== definition.source.commit)
|
|
4184
|
+
ctx.addIssue({
|
|
4185
|
+
code: "custom",
|
|
4186
|
+
path: ["resources", index, "source"],
|
|
4187
|
+
message: "Project preparation must use the definition repository and commit"
|
|
4188
|
+
});
|
|
4189
|
+
if (!resource.source.repoUrl.startsWith("https://"))
|
|
4190
|
+
ctx.addIssue({
|
|
4191
|
+
code: "custom",
|
|
4192
|
+
path: ["resources", index, "source", "repoUrl"],
|
|
4193
|
+
message: "Project preparation requires a public HTTPS Git source"
|
|
4194
|
+
});
|
|
4195
|
+
if (!uncredentialedRepoUrlSchema.safeParse(resource.source.repoUrl).success)
|
|
4196
|
+
ctx.addIssue({
|
|
4197
|
+
code: "custom",
|
|
4198
|
+
path: ["resources", index, "source", "repoUrl"],
|
|
4199
|
+
message: "Repository URL must not contain embedded credentials or query parameters"
|
|
4200
|
+
});
|
|
4201
|
+
}
|
|
4202
|
+
});
|
|
4203
|
+
var runDefinitionPlanResponseSchema = z40.object({
|
|
4204
|
+
source: runDefinitionSubmissionSchema.shape.source,
|
|
4205
|
+
measurementMode: z40.literal("AGENT_EXPERIENCE"),
|
|
4206
|
+
cases: z40.array(
|
|
4207
|
+
z40.object({
|
|
4208
|
+
path: z40.string(),
|
|
4209
|
+
agents: z40.array(agentConfigSchema),
|
|
4210
|
+
samples: z40.number().int().nonnegative(),
|
|
4211
|
+
expectedRuns: z40.number().int().nonnegative(),
|
|
4212
|
+
workspace: z40.object({
|
|
4213
|
+
repoUrl: z40.string().nullable(),
|
|
4214
|
+
repoRef: z40.string().nullable()
|
|
4215
|
+
}),
|
|
4216
|
+
criteriaCount: z40.number().int().nonnegative(),
|
|
4217
|
+
inputs: z40.array(z40.string())
|
|
4218
|
+
})
|
|
4219
|
+
),
|
|
4220
|
+
resources: z40.array(
|
|
4221
|
+
z40.object({ name: z40.string(), type: z40.literal("prepared-directory") })
|
|
4222
|
+
),
|
|
4223
|
+
expectedRuns: z40.number().int().nonnegative(),
|
|
4224
|
+
canRun: z40.boolean()
|
|
4225
|
+
});
|
|
4226
|
+
var runDefinitionRunResponseSchema = z40.object({
|
|
4227
|
+
runRequestId: z40.string(),
|
|
4228
|
+
batchIds: z40.array(z40.string()),
|
|
4229
|
+
runIds: z40.array(z40.string()),
|
|
4230
|
+
requestedRuns: z40.number().int().nonnegative(),
|
|
4231
|
+
createdRuns: z40.number().int().nonnegative(),
|
|
4232
|
+
aggregateUrl: z40.string()
|
|
4233
|
+
});
|
|
4234
|
+
|
|
4235
|
+
// ../packages/api-schemas/src/runRequests.ts
|
|
4236
|
+
import { z as z41 } from "zod";
|
|
4237
|
+
var runRequestStatusSchema = z41.enum([
|
|
4238
|
+
"PENDING",
|
|
4239
|
+
"PREPARING",
|
|
4240
|
+
"RUNNING",
|
|
4241
|
+
"COMPLETED",
|
|
4242
|
+
"FAILED",
|
|
4243
|
+
"CANCELED"
|
|
4244
|
+
]);
|
|
4245
|
+
var runRequestResultSchema = z41.enum([
|
|
4246
|
+
"PENDING",
|
|
4247
|
+
"PASSED",
|
|
4248
|
+
"QUALITY_FAILED",
|
|
4249
|
+
"EXECUTION_FAILED",
|
|
4250
|
+
"PREPARATION_FAILED",
|
|
4251
|
+
"JUDGE_FAILED",
|
|
4252
|
+
"INCOMPLETE",
|
|
4253
|
+
"CANCELED"
|
|
4254
|
+
]);
|
|
4255
|
+
var runRequestCriterionResultSchema = z41.object({
|
|
4256
|
+
criterionId: z41.string().nullable(),
|
|
4257
|
+
name: z41.string(),
|
|
4258
|
+
passed: z41.boolean(),
|
|
4259
|
+
reasoning: z41.string().nullable()
|
|
4260
|
+
});
|
|
4261
|
+
var runRequestRunSchema = z41.object({
|
|
4262
|
+
id: z41.string(),
|
|
4263
|
+
batchId: z41.string(),
|
|
4264
|
+
status: z41.string(),
|
|
4265
|
+
agent: agentSchema,
|
|
4266
|
+
model: z41.string(),
|
|
4267
|
+
result: runRequestResultSchema,
|
|
4268
|
+
judge: z41.object({ status: z41.string() }).nullable(),
|
|
4269
|
+
criterionResults: z41.array(runRequestCriterionResultSchema)
|
|
4270
|
+
});
|
|
4271
|
+
var runRequestResponseSchema = z41.object({
|
|
4272
|
+
id: z41.string(),
|
|
4273
|
+
status: runRequestStatusSchema,
|
|
4274
|
+
result: runRequestResultSchema,
|
|
4275
|
+
settled: z41.boolean(),
|
|
4276
|
+
counts: z41.object({
|
|
4277
|
+
requestedRuns: z41.number().int().nonnegative(),
|
|
4278
|
+
createdRuns: z41.number().int().nonnegative(),
|
|
4279
|
+
terminalRuns: z41.number().int().nonnegative(),
|
|
4280
|
+
judgedRuns: z41.number().int().nonnegative(),
|
|
4281
|
+
passedRuns: z41.number().int().nonnegative(),
|
|
4282
|
+
qualityFailedRuns: z41.number().int().nonnegative(),
|
|
4283
|
+
executionFailedRuns: z41.number().int().nonnegative(),
|
|
4284
|
+
judgeFailedRuns: z41.number().int().nonnegative(),
|
|
4285
|
+
pendingRuns: z41.number().int().nonnegative()
|
|
4286
|
+
}),
|
|
4287
|
+
batches: z41.array(
|
|
4288
|
+
z41.object({ id: z41.string(), runs: z41.array(runRequestRunSchema) })
|
|
4289
|
+
),
|
|
4290
|
+
createdAt: z41.string(),
|
|
4291
|
+
updatedAt: z41.string(),
|
|
4292
|
+
finishedAt: z41.string().nullable()
|
|
3342
4293
|
});
|
|
3343
4294
|
|
|
3344
4295
|
// ../packages/api-schemas/src/summary.ts
|
|
3345
|
-
import { z as
|
|
4296
|
+
import { z as z42 } from "zod";
|
|
3346
4297
|
function cardResultSchema(data) {
|
|
3347
|
-
return
|
|
3348
|
-
|
|
3349
|
-
|
|
4298
|
+
return z42.union([
|
|
4299
|
+
z42.object({ ok: z42.literal(true), data }),
|
|
4300
|
+
z42.object({ ok: z42.literal(false), error: z42.string() })
|
|
3350
4301
|
]);
|
|
3351
4302
|
}
|
|
3352
|
-
var summaryActionSchema =
|
|
3353
|
-
id:
|
|
3354
|
-
title:
|
|
3355
|
-
priority:
|
|
3356
|
-
status:
|
|
3357
|
-
evidenceCount:
|
|
3358
|
-
});
|
|
3359
|
-
var summaryAeCardSchema =
|
|
3360
|
-
id:
|
|
3361
|
-
name:
|
|
3362
|
-
judgedRuns:
|
|
3363
|
-
passRate:
|
|
4303
|
+
var summaryActionSchema = z42.object({
|
|
4304
|
+
id: z42.string(),
|
|
4305
|
+
title: z42.string(),
|
|
4306
|
+
priority: z42.string(),
|
|
4307
|
+
status: z42.string(),
|
|
4308
|
+
evidenceCount: z42.number().int()
|
|
4309
|
+
});
|
|
4310
|
+
var summaryAeCardSchema = z42.object({
|
|
4311
|
+
id: z42.string(),
|
|
4312
|
+
name: z42.string(),
|
|
4313
|
+
judgedRuns: z42.number().int(),
|
|
4314
|
+
passRate: z42.number().nullable(),
|
|
3364
4315
|
/** pp delta vs the prior 30d; null with no prior. */
|
|
3365
|
-
deltaPP:
|
|
3366
|
-
});
|
|
3367
|
-
var summaryVisibilityCardSchema =
|
|
3368
|
-
name:
|
|
3369
|
-
owned:
|
|
3370
|
-
name:
|
|
3371
|
-
rank:
|
|
3372
|
-
chosenRate:
|
|
3373
|
-
mentionRate:
|
|
4316
|
+
deltaPP: z42.number().nullable()
|
|
4317
|
+
});
|
|
4318
|
+
var summaryVisibilityCardSchema = z42.object({
|
|
4319
|
+
name: z42.string(),
|
|
4320
|
+
owned: z42.object({
|
|
4321
|
+
name: z42.string(),
|
|
4322
|
+
rank: z42.number(),
|
|
4323
|
+
chosenRate: z42.number(),
|
|
4324
|
+
mentionRate: z42.number()
|
|
3374
4325
|
}).nullable(),
|
|
3375
|
-
chosenDeltaPP:
|
|
4326
|
+
chosenDeltaPP: z42.number().nullable()
|
|
3376
4327
|
});
|
|
3377
|
-
var homeSummarySchema =
|
|
3378
|
-
hasAnyRuns:
|
|
3379
|
-
lastSynthesisAt:
|
|
3380
|
-
actions: cardResultSchema(
|
|
4328
|
+
var homeSummarySchema = z42.object({
|
|
4329
|
+
hasAnyRuns: z42.boolean(),
|
|
4330
|
+
lastSynthesisAt: z42.string().nullable(),
|
|
4331
|
+
actions: cardResultSchema(z42.array(summaryActionSchema)),
|
|
3381
4332
|
visibility: cardResultSchema(
|
|
3382
|
-
|
|
3383
|
-
cards:
|
|
3384
|
-
marketsTotal:
|
|
3385
|
-
promptCount:
|
|
4333
|
+
z42.object({
|
|
4334
|
+
cards: z42.array(summaryVisibilityCardSchema),
|
|
4335
|
+
marketsTotal: z42.number().int(),
|
|
4336
|
+
promptCount: z42.number().int()
|
|
3386
4337
|
})
|
|
3387
4338
|
),
|
|
3388
4339
|
agentExperience: cardResultSchema(
|
|
3389
|
-
|
|
3390
|
-
cards:
|
|
3391
|
-
total:
|
|
3392
|
-
setCount:
|
|
4340
|
+
z42.object({
|
|
4341
|
+
cards: z42.array(summaryAeCardSchema),
|
|
4342
|
+
total: z42.number().int(),
|
|
4343
|
+
setCount: z42.number().int()
|
|
3393
4344
|
})
|
|
3394
4345
|
),
|
|
3395
|
-
notableSessions: cardResultSchema(
|
|
4346
|
+
notableSessions: cardResultSchema(z42.array(z42.unknown()))
|
|
3396
4347
|
});
|
|
3397
4348
|
|
|
3398
4349
|
// ../packages/api-schemas/src/tokens.ts
|
|
3399
|
-
import { z as
|
|
3400
|
-
var apiTokenSchema =
|
|
3401
|
-
id:
|
|
3402
|
-
name:
|
|
4350
|
+
import { z as z43 } from "zod";
|
|
4351
|
+
var apiTokenSchema = z43.object({
|
|
4352
|
+
id: z43.string(),
|
|
4353
|
+
name: z43.string(),
|
|
3403
4354
|
/** Last 4 chars of the plaintext, for recognizability (non-secret). */
|
|
3404
|
-
last4:
|
|
3405
|
-
expiresAt:
|
|
3406
|
-
lastUsedAt:
|
|
3407
|
-
createdAt:
|
|
3408
|
-
revokedAt:
|
|
3409
|
-
});
|
|
3410
|
-
var createTokenRequestSchema =
|
|
3411
|
-
name:
|
|
4355
|
+
last4: z43.string(),
|
|
4356
|
+
expiresAt: z43.iso.datetime().nullable(),
|
|
4357
|
+
lastUsedAt: z43.iso.datetime().nullable(),
|
|
4358
|
+
createdAt: z43.iso.datetime(),
|
|
4359
|
+
revokedAt: z43.iso.datetime().nullable()
|
|
4360
|
+
});
|
|
4361
|
+
var createTokenRequestSchema = z43.object({
|
|
4362
|
+
name: z43.string().min(1).max(100),
|
|
3412
4363
|
/** ISO-8601 datetime; omit for no expiry. */
|
|
3413
|
-
expiresAt:
|
|
4364
|
+
expiresAt: z43.iso.datetime({ offset: true }).optional()
|
|
3414
4365
|
});
|
|
3415
4366
|
var createTokenResponseSchema = apiTokenSchema.extend({
|
|
3416
|
-
token:
|
|
4367
|
+
token: z43.string()
|
|
3417
4368
|
});
|
|
3418
4369
|
var listTokensResponseSchema = listResponseSchema(apiTokenSchema);
|
|
3419
4370
|
|
|
3420
4371
|
// ../packages/api-schemas/src/visibility.ts
|
|
3421
|
-
import { z as
|
|
4372
|
+
import { z as z44 } from "zod";
|
|
3422
4373
|
var VISIBILITY_PROMPT_KINDS = ["HEAD_TO_HEAD", "RECOMMEND"];
|
|
3423
|
-
var visibilityPromptKindSchema =
|
|
4374
|
+
var visibilityPromptKindSchema = z44.enum(VISIBILITY_PROMPT_KINDS);
|
|
3424
4375
|
var visibilityPromptSchema = visibilityPromptSettingsSchema.extend({
|
|
3425
|
-
id:
|
|
3426
|
-
name:
|
|
4376
|
+
id: z44.string(),
|
|
4377
|
+
name: z44.string(),
|
|
3427
4378
|
kind: visibilityPromptKindSchema,
|
|
3428
|
-
promptText:
|
|
4379
|
+
promptText: z44.string(),
|
|
3429
4380
|
/** HEAD_TO_HEAD is always branded; RECOMMEND may be either. */
|
|
3430
|
-
branded:
|
|
3431
|
-
brandA:
|
|
3432
|
-
brandB:
|
|
3433
|
-
market:
|
|
3434
|
-
position:
|
|
3435
|
-
createdAt:
|
|
4381
|
+
branded: z44.boolean(),
|
|
4382
|
+
brandA: z44.string().nullable(),
|
|
4383
|
+
brandB: z44.string().nullable(),
|
|
4384
|
+
market: z44.object({ id: z44.string(), slug: z44.string(), name: z44.string() }).nullable(),
|
|
4385
|
+
position: z44.number().int(),
|
|
4386
|
+
createdAt: z44.string(),
|
|
3436
4387
|
/** User tag names on the hidden backing prompt (Surface:* excluded). */
|
|
3437
|
-
tags:
|
|
3438
|
-
runCount:
|
|
3439
|
-
lastRunAt:
|
|
3440
|
-
scenarios:
|
|
4388
|
+
tags: z44.array(z44.string()),
|
|
4389
|
+
runCount: z44.number().int(),
|
|
4390
|
+
lastRunAt: z44.string().nullable(),
|
|
4391
|
+
scenarios: z44.array(visibilityScenarioSchema)
|
|
3441
4392
|
});
|
|
3442
|
-
var listVisibilityPromptsResponseSchema =
|
|
3443
|
-
items:
|
|
4393
|
+
var listVisibilityPromptsResponseSchema = z44.object({
|
|
4394
|
+
items: z44.array(visibilityPromptSchema)
|
|
3444
4395
|
});
|
|
3445
|
-
var createVisibilityPromptBodySchema =
|
|
3446
|
-
name:
|
|
3447
|
-
promptText:
|
|
4396
|
+
var createVisibilityPromptBodySchema = z44.object({
|
|
4397
|
+
name: z44.string().min(1),
|
|
4398
|
+
promptText: z44.string().min(1),
|
|
3448
4399
|
kind: visibilityPromptKindSchema,
|
|
3449
|
-
branded:
|
|
4400
|
+
branded: z44.boolean().optional(),
|
|
3450
4401
|
/** Required pair for HEAD_TO_HEAD (order-independent). */
|
|
3451
|
-
brandA:
|
|
3452
|
-
brandB:
|
|
4402
|
+
brandA: z44.string().nullish(),
|
|
4403
|
+
brandB: z44.string().nullish(),
|
|
3453
4404
|
/** Measured market id from the org's tracked set. */
|
|
3454
|
-
marketId:
|
|
3455
|
-
tags:
|
|
4405
|
+
marketId: z44.string().nullish(),
|
|
4406
|
+
tags: z44.array(z44.string()).optional(),
|
|
3456
4407
|
agents: visibilityPromptSettingsInputShape.agents.optional(),
|
|
3457
4408
|
skillRefs: visibilityPromptSettingsInputShape.skillRefs,
|
|
3458
4409
|
mcpRefs: visibilityPromptSettingsInputShape.mcpRefs,
|
|
3459
4410
|
connectionIds: visibilityPromptSettingsInputShape.connectionIds,
|
|
3460
4411
|
cadence: visibilityPromptSettingsInputShape.cadence,
|
|
3461
4412
|
sampleCount: visibilityPromptSettingsInputShape.sampleCount,
|
|
3462
|
-
scenarios:
|
|
4413
|
+
scenarios: z44.array(visibilityScenarioInputSchema).optional().default([])
|
|
3463
4414
|
}).strict();
|
|
3464
|
-
var synthesizeResponseSchema =
|
|
4415
|
+
var synthesizeResponseSchema = z44.object({
|
|
3465
4416
|
/** False when the pre-check found no new evidence and nothing was queued. */
|
|
3466
|
-
queued:
|
|
3467
|
-
});
|
|
3468
|
-
var updateVisibilityPromptBodySchema =
|
|
3469
|
-
name:
|
|
3470
|
-
promptText:
|
|
3471
|
-
branded:
|
|
3472
|
-
brandA:
|
|
3473
|
-
brandB:
|
|
3474
|
-
marketId:
|
|
3475
|
-
tags:
|
|
4417
|
+
queued: z44.boolean()
|
|
4418
|
+
});
|
|
4419
|
+
var updateVisibilityPromptBodySchema = z44.object({
|
|
4420
|
+
name: z44.string().min(1).optional(),
|
|
4421
|
+
promptText: z44.string().min(1).optional(),
|
|
4422
|
+
branded: z44.boolean().optional(),
|
|
4423
|
+
brandA: z44.string().nullish(),
|
|
4424
|
+
brandB: z44.string().nullish(),
|
|
4425
|
+
marketId: z44.string().nullish(),
|
|
4426
|
+
tags: z44.array(z44.string()).optional(),
|
|
3476
4427
|
...visibilityPromptSettingsPatchShape,
|
|
3477
4428
|
/** Replaces the whole scenario set when present. */
|
|
3478
|
-
scenarios:
|
|
4429
|
+
scenarios: z44.array(visibilityScenarioInputSchema).optional()
|
|
3479
4430
|
}).strict();
|
|
3480
|
-
var runVisibilityPromptResponseSchema =
|
|
3481
|
-
batchIds:
|
|
3482
|
-
runIds:
|
|
4431
|
+
var runVisibilityPromptResponseSchema = z44.object({
|
|
4432
|
+
batchIds: z44.array(z44.string()),
|
|
4433
|
+
runIds: z44.array(z44.string())
|
|
3483
4434
|
});
|
|
3484
|
-
var visibilityPromptRunSchema =
|
|
3485
|
-
id:
|
|
3486
|
-
status:
|
|
4435
|
+
var visibilityPromptRunSchema = z44.object({
|
|
4436
|
+
id: z44.string(),
|
|
4437
|
+
status: z44.string(),
|
|
3487
4438
|
agent: agentSchema,
|
|
3488
|
-
model:
|
|
3489
|
-
resolvedModel:
|
|
3490
|
-
createdAt:
|
|
3491
|
-
startedAt:
|
|
3492
|
-
finishedAt:
|
|
3493
|
-
failureReason:
|
|
3494
|
-
verdict:
|
|
3495
|
-
status:
|
|
3496
|
-
error:
|
|
3497
|
-
payload:
|
|
4439
|
+
model: z44.string().nullable(),
|
|
4440
|
+
resolvedModel: z44.string().nullable(),
|
|
4441
|
+
createdAt: z44.string(),
|
|
4442
|
+
startedAt: z44.string().nullable(),
|
|
4443
|
+
finishedAt: z44.string().nullable(),
|
|
4444
|
+
failureReason: z44.string().nullable(),
|
|
4445
|
+
verdict: z44.object({
|
|
4446
|
+
status: z44.string(),
|
|
4447
|
+
error: z44.string().nullable(),
|
|
4448
|
+
payload: z44.unknown().nullable()
|
|
3498
4449
|
}).nullable()
|
|
3499
4450
|
});
|
|
3500
|
-
var listVisibilityPromptRunsResponseSchema =
|
|
3501
|
-
items:
|
|
4451
|
+
var listVisibilityPromptRunsResponseSchema = z44.object({
|
|
4452
|
+
items: z44.array(visibilityPromptRunSchema)
|
|
3502
4453
|
});
|
|
3503
|
-
var orgMarketSchema =
|
|
3504
|
-
marketId:
|
|
3505
|
-
slug:
|
|
3506
|
-
name:
|
|
4454
|
+
var orgMarketSchema = z44.object({
|
|
4455
|
+
marketId: z44.string(),
|
|
4456
|
+
slug: z44.string(),
|
|
4457
|
+
name: z44.string(),
|
|
3507
4458
|
/** The one line saying what the market measures. */
|
|
3508
|
-
capability:
|
|
4459
|
+
capability: z44.string(),
|
|
3509
4460
|
/** SUGGESTED (the matcher) | USER (a human picked it). */
|
|
3510
|
-
source:
|
|
4461
|
+
source: z44.string()
|
|
3511
4462
|
});
|
|
3512
4463
|
var addableMarketSchema = orgMarketSchema.omit({ source: true });
|
|
3513
|
-
var listMarketsResponseSchema =
|
|
3514
|
-
tracked:
|
|
4464
|
+
var listMarketsResponseSchema = z44.object({
|
|
4465
|
+
tracked: z44.array(orgMarketSchema),
|
|
3515
4466
|
/** Published markets the org does not track yet (closed set — no free-text
|
|
3516
4467
|
* market creation). */
|
|
3517
|
-
addable:
|
|
4468
|
+
addable: z44.array(addableMarketSchema)
|
|
3518
4469
|
});
|
|
3519
|
-
var addMarketBodySchema =
|
|
4470
|
+
var addMarketBodySchema = z44.object({ marketId: z44.string().min(1) });
|
|
3520
4471
|
|
|
3521
4472
|
// src/version.ts
|
|
3522
4473
|
import { createRequire } from "module";
|
|
@@ -3584,9 +4535,9 @@ function resolveToken() {
|
|
|
3584
4535
|
return null;
|
|
3585
4536
|
}
|
|
3586
4537
|
}
|
|
3587
|
-
function saveToken(
|
|
4538
|
+
function saveToken(token2) {
|
|
3588
4539
|
mkdirSync2(configDir(), { recursive: true, mode: 448 });
|
|
3589
|
-
writeFileSync2(credentialsPath(), `${
|
|
4540
|
+
writeFileSync2(credentialsPath(), `${token2.trim()}
|
|
3590
4541
|
`, {
|
|
3591
4542
|
encoding: "utf8",
|
|
3592
4543
|
mode: 384
|
|
@@ -3796,9 +4747,10 @@ function messageOf(error) {
|
|
|
3796
4747
|
}
|
|
3797
4748
|
|
|
3798
4749
|
// src/lib/context.ts
|
|
3799
|
-
function resolveOrg(command) {
|
|
4750
|
+
function resolveOrg(command, fileOrg) {
|
|
3800
4751
|
const { org } = command.optsWithGlobals();
|
|
3801
4752
|
if (org) return org;
|
|
4753
|
+
if (fileOrg) return fileOrg;
|
|
3802
4754
|
const { defaultOrg } = readConfig();
|
|
3803
4755
|
if (defaultOrg) return defaultOrg;
|
|
3804
4756
|
throw new CliError(
|
|
@@ -3900,7 +4852,7 @@ __export(apply_exports, {
|
|
|
3900
4852
|
register: () => register2
|
|
3901
4853
|
});
|
|
3902
4854
|
import { readFileSync as readFileSync3 } from "fs";
|
|
3903
|
-
import { z as
|
|
4855
|
+
import { z as z45 } from "zod";
|
|
3904
4856
|
function assertRequiredExperiment(spec, required) {
|
|
3905
4857
|
if (required.length === 0) {
|
|
3906
4858
|
throw new CliError(
|
|
@@ -4052,7 +5004,7 @@ function register2(program2) {
|
|
|
4052
5004
|
).option("--schema", "print the battery spec JSON Schema and exit").option("--example", "print a minimal example battery spec and exit").action(
|
|
4053
5005
|
async (opts, command) => {
|
|
4054
5006
|
if (opts.schema) {
|
|
4055
|
-
printJson(
|
|
5007
|
+
printJson(z45.toJSONSchema(batteries_exports.batterySpecSchema));
|
|
4056
5008
|
return;
|
|
4057
5009
|
}
|
|
4058
5010
|
if (opts.example) {
|
|
@@ -4164,7 +5116,7 @@ async function browserLogin() {
|
|
|
4164
5116
|
const codeVerifier = randomBytes(32).toString("base64url");
|
|
4165
5117
|
const codeChallenge = createHash("sha256").update(codeVerifier).digest("base64url");
|
|
4166
5118
|
const server = createServer();
|
|
4167
|
-
const codePromise = new Promise((
|
|
5119
|
+
const codePromise = new Promise((resolve2, reject) => {
|
|
4168
5120
|
server.on("request", (req, res) => {
|
|
4169
5121
|
const url = new URL(req.url ?? "/", "http://127.0.0.1");
|
|
4170
5122
|
if (url.pathname !== "/callback") {
|
|
@@ -4187,15 +5139,15 @@ async function browserLogin() {
|
|
|
4187
5139
|
"content-type": "text/plain; charset=utf-8"
|
|
4188
5140
|
});
|
|
4189
5141
|
res.end("Authorized. Return to your terminal.");
|
|
4190
|
-
|
|
5142
|
+
resolve2(code);
|
|
4191
5143
|
});
|
|
4192
5144
|
server.on(
|
|
4193
5145
|
"error",
|
|
4194
5146
|
(e) => reject(new CliError("network", `Loopback listener failed: ${e.message}`))
|
|
4195
5147
|
);
|
|
4196
5148
|
});
|
|
4197
|
-
await new Promise((
|
|
4198
|
-
server.listen(0, "127.0.0.1",
|
|
5149
|
+
await new Promise((resolve2) => {
|
|
5150
|
+
server.listen(0, "127.0.0.1", resolve2);
|
|
4199
5151
|
});
|
|
4200
5152
|
const address = server.address();
|
|
4201
5153
|
if (!address || typeof address === "string") {
|
|
@@ -4246,9 +5198,9 @@ async function browserLogin() {
|
|
|
4246
5198
|
);
|
|
4247
5199
|
}
|
|
4248
5200
|
const exchange = parsed.data;
|
|
4249
|
-
const
|
|
4250
|
-
const me = await fetchMe(new ApiClient({ token }));
|
|
4251
|
-
saveToken(
|
|
5201
|
+
const token2 = exchange.accessToken;
|
|
5202
|
+
const me = await fetchMe(new ApiClient({ token: token2 }));
|
|
5203
|
+
saveToken(token2);
|
|
4252
5204
|
console.log(`Token saved to ${credentialsPath()}`);
|
|
4253
5205
|
printIdentity(me);
|
|
4254
5206
|
} finally {
|
|
@@ -4352,10 +5304,10 @@ async function deviceLogin(opts) {
|
|
|
4352
5304
|
"The sign-in request expired before it was approved. Re-run it to try again."
|
|
4353
5305
|
);
|
|
4354
5306
|
case "approved": {
|
|
4355
|
-
const
|
|
5307
|
+
const token2 = result.accessToken;
|
|
4356
5308
|
try {
|
|
4357
|
-
const me = await fetchMe(new ApiClient({ token }));
|
|
4358
|
-
return { token, me };
|
|
5309
|
+
const me = await fetchMe(new ApiClient({ token: token2 }));
|
|
5310
|
+
return { token: token2, me };
|
|
4359
5311
|
} catch (e) {
|
|
4360
5312
|
throw new CliError(
|
|
4361
5313
|
"network",
|
|
@@ -4392,15 +5344,15 @@ async function withTokenLogin() {
|
|
|
4392
5344
|
if (process.stdin.isTTY) {
|
|
4393
5345
|
console.error("Paste your API token, then press Enter:");
|
|
4394
5346
|
}
|
|
4395
|
-
const
|
|
4396
|
-
if (!
|
|
5347
|
+
const token2 = (await readStdin()).trim();
|
|
5348
|
+
if (!token2.startsWith(TOKEN_PREFIX)) {
|
|
4397
5349
|
throw new CliError(
|
|
4398
5350
|
"usage",
|
|
4399
5351
|
`Expected a token starting with ${TOKEN_PREFIX} on stdin`
|
|
4400
5352
|
);
|
|
4401
5353
|
}
|
|
4402
|
-
const me = await fetchMe(new ApiClient({ token }));
|
|
4403
|
-
saveToken(
|
|
5354
|
+
const me = await fetchMe(new ApiClient({ token: token2 }));
|
|
5355
|
+
saveToken(token2);
|
|
4404
5356
|
console.log(`Token saved to ${credentialsPath()}`);
|
|
4405
5357
|
printIdentity(me);
|
|
4406
5358
|
}
|
|
@@ -4468,8 +5420,8 @@ function register3(program2) {
|
|
|
4468
5420
|
throw new CliError("usage", "--email requires --device.");
|
|
4469
5421
|
}
|
|
4470
5422
|
if (opts.device) {
|
|
4471
|
-
const { token, me } = await deviceLogin({ email: opts.email });
|
|
4472
|
-
saveToken(
|
|
5423
|
+
const { token: token2, me } = await deviceLogin({ email: opts.email });
|
|
5424
|
+
saveToken(token2);
|
|
4473
5425
|
console.log(`Token saved to ${credentialsPath()}`);
|
|
4474
5426
|
printIdentity(me);
|
|
4475
5427
|
} else if (opts.withToken) {
|
|
@@ -4824,10 +5776,10 @@ function splitList(raw) {
|
|
|
4824
5776
|
return raw.split(",").map((v) => v.trim()).filter(Boolean);
|
|
4825
5777
|
}
|
|
4826
5778
|
function parsePackage(raw) {
|
|
4827
|
-
const
|
|
4828
|
-
if (
|
|
4829
|
-
const name = raw.slice(0,
|
|
4830
|
-
const ecosystem = raw.slice(
|
|
5779
|
+
const sep3 = raw.lastIndexOf(":");
|
|
5780
|
+
if (sep3 === -1) return { name: raw.trim(), ecosystem: null };
|
|
5781
|
+
const name = raw.slice(0, sep3).trim();
|
|
5782
|
+
const ecosystem = raw.slice(sep3 + 1).trim();
|
|
4831
5783
|
if (!name) throw new CliError("usage", `Invalid package "${raw}"`);
|
|
4832
5784
|
if (!ECOSYSTEMS.includes(ecosystem)) {
|
|
4833
5785
|
throw new CliError(
|
|
@@ -4963,7 +5915,7 @@ async function waitForCanary(id) {
|
|
|
4963
5915
|
while (true) {
|
|
4964
5916
|
const canary = await client.get(`${path}/${id}`);
|
|
4965
5917
|
if (terminal.has(canary.status)) return canary;
|
|
4966
|
-
await new Promise((
|
|
5918
|
+
await new Promise((resolve2) => setTimeout(resolve2, 1e4));
|
|
4967
5919
|
}
|
|
4968
5920
|
}
|
|
4969
5921
|
function register7(program2, opts = {}) {
|
|
@@ -5182,6 +6134,7 @@ __export(evals_exports2, {
|
|
|
5182
6134
|
register: () => register11
|
|
5183
6135
|
});
|
|
5184
6136
|
import { readFileSync as readFileSync6 } from "fs";
|
|
6137
|
+
import { Option as Option2 } from "commander";
|
|
5185
6138
|
|
|
5186
6139
|
// src/lib/confirm.ts
|
|
5187
6140
|
import { createInterface as createInterface2 } from "readline";
|
|
@@ -5195,7 +6148,7 @@ async function confirmOrAbort(question, yes) {
|
|
|
5195
6148
|
}
|
|
5196
6149
|
const rl = createInterface2({ input: process.stdin, output: process.stderr });
|
|
5197
6150
|
const answer = await new Promise(
|
|
5198
|
-
(
|
|
6151
|
+
(resolve2) => rl.question(`${question} [y/N] `, resolve2)
|
|
5199
6152
|
);
|
|
5200
6153
|
rl.close();
|
|
5201
6154
|
if (!/^y(es)?$/i.test(answer.trim())) {
|
|
@@ -5392,6 +6345,181 @@ function cadenceLabel(cadence) {
|
|
|
5392
6345
|
return cadence === "NONE" ? "manual" : cadence.toLowerCase();
|
|
5393
6346
|
}
|
|
5394
6347
|
|
|
6348
|
+
// src/lib/runDefinitionFiles.ts
|
|
6349
|
+
import { execFileSync } from "child_process";
|
|
6350
|
+
import { createHash as createHash2 } from "crypto";
|
|
6351
|
+
import { isAbsolute, relative, resolve, sep } from "path";
|
|
6352
|
+
import { parseDocument } from "yaml";
|
|
6353
|
+
function git(cwd, args) {
|
|
6354
|
+
try {
|
|
6355
|
+
return execFileSync("git", args, {
|
|
6356
|
+
cwd,
|
|
6357
|
+
encoding: "buffer",
|
|
6358
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
6359
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
6360
|
+
});
|
|
6361
|
+
} catch (error) {
|
|
6362
|
+
const detail = error && typeof error === "object" && "stderr" in error ? String(error.stderr).trim() : String(error);
|
|
6363
|
+
throw new CliError("usage", `git ${args[0]} failed: ${detail}`);
|
|
6364
|
+
}
|
|
6365
|
+
}
|
|
6366
|
+
function committedBlob(root, commit, path2) {
|
|
6367
|
+
return git(root, ["show", `${commit}:${path2}`]);
|
|
6368
|
+
}
|
|
6369
|
+
function decode(blob, path2) {
|
|
6370
|
+
try {
|
|
6371
|
+
return new TextDecoder("utf-8", { fatal: true }).decode(blob);
|
|
6372
|
+
} catch {
|
|
6373
|
+
throw new CliError("usage", `${path2} is not UTF-8 text`);
|
|
6374
|
+
}
|
|
6375
|
+
}
|
|
6376
|
+
function parseYaml(text, path2) {
|
|
6377
|
+
const document = parseDocument(text, { uniqueKeys: true });
|
|
6378
|
+
if (document.errors.length) {
|
|
6379
|
+
throw new CliError(
|
|
6380
|
+
"usage",
|
|
6381
|
+
`${path2}: ${document.errors.map((error) => error.message).join("; ")}`
|
|
6382
|
+
);
|
|
6383
|
+
}
|
|
6384
|
+
return document.toJS();
|
|
6385
|
+
}
|
|
6386
|
+
function caseFromMarkdown(path2, source) {
|
|
6387
|
+
const match = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)([\s\S]*)$/.exec(source);
|
|
6388
|
+
if (!match)
|
|
6389
|
+
throw new CliError(
|
|
6390
|
+
"usage",
|
|
6391
|
+
`${path2}: expected YAML frontmatter between --- lines`
|
|
6392
|
+
);
|
|
6393
|
+
const parsed = runDefinitionFrontmatterSchema.safeParse(
|
|
6394
|
+
parseYaml(match[1], path2)
|
|
6395
|
+
);
|
|
6396
|
+
if (!parsed.success)
|
|
6397
|
+
throw new CliError("usage", `${path2}: ${parsed.error.message}`);
|
|
6398
|
+
const prompt = match[2].trim();
|
|
6399
|
+
if (!prompt) throw new CliError("usage", `${path2}: Markdown body is empty`);
|
|
6400
|
+
return { path: path2, prompt, ...parsed.data };
|
|
6401
|
+
}
|
|
6402
|
+
function relativeGitPath(root, cwd, input) {
|
|
6403
|
+
if (!input || input.includes("\0") || /[*?[\]{}]/.test(input))
|
|
6404
|
+
throw new CliError("usage", `expected an explicit case path: ${input}`);
|
|
6405
|
+
const absolute = resolve(cwd, input);
|
|
6406
|
+
const path2 = relative(root, absolute);
|
|
6407
|
+
if (!path2 || path2 === ".." || path2.startsWith(`..${sep}`) || isAbsolute(path2))
|
|
6408
|
+
throw new CliError(
|
|
6409
|
+
"usage",
|
|
6410
|
+
`case path is outside this Git repository: ${input}`
|
|
6411
|
+
);
|
|
6412
|
+
const gitPath = path2.split(sep).join("/");
|
|
6413
|
+
if (!gitPath.endsWith(".md"))
|
|
6414
|
+
throw new CliError("usage", `case path must end in .md: ${input}`);
|
|
6415
|
+
return gitPath;
|
|
6416
|
+
}
|
|
6417
|
+
function loadCommittedRunDefinitionWithContext(files, cwd = process.cwd()) {
|
|
6418
|
+
if (files.length === 0)
|
|
6419
|
+
throw new CliError("usage", "--files requires at least one Markdown path");
|
|
6420
|
+
const root = git(cwd, ["rev-parse", "--show-toplevel"]).toString("utf8").trim();
|
|
6421
|
+
const commit = git(root, ["rev-parse", "--verify", "HEAD^{commit}"]).toString("utf8").trim();
|
|
6422
|
+
let repoUrl;
|
|
6423
|
+
try {
|
|
6424
|
+
repoUrl = git(root, ["remote", "get-url", "origin"]).toString("utf8").trim();
|
|
6425
|
+
} catch {
|
|
6426
|
+
throw new CliError(
|
|
6427
|
+
"usage",
|
|
6428
|
+
"Git origin is required to identify the committed definition repository"
|
|
6429
|
+
);
|
|
6430
|
+
}
|
|
6431
|
+
if (!uncredentialedRepoUrlSchema.safeParse(repoUrl).success)
|
|
6432
|
+
throw new CliError(
|
|
6433
|
+
"usage",
|
|
6434
|
+
"Git origin must be a repository URL without embedded credentials or query parameters"
|
|
6435
|
+
);
|
|
6436
|
+
const paths = files.map((file) => relativeGitPath(root, cwd, file));
|
|
6437
|
+
if (new Set(paths).size !== paths.length)
|
|
6438
|
+
throw new CliError("usage", "case paths must be unique");
|
|
6439
|
+
const blobs = [];
|
|
6440
|
+
let config;
|
|
6441
|
+
const hasConfig = git(root, ["ls-tree", "--name-only", commit, "--", "gauge.json"]).toString("utf8").trim() === "gauge.json";
|
|
6442
|
+
if (hasConfig) {
|
|
6443
|
+
try {
|
|
6444
|
+
const bytes = committedBlob(root, commit, "gauge.json");
|
|
6445
|
+
blobs.push({ path: "gauge.json", bytes });
|
|
6446
|
+
config = runDefinitionFileConfigSchema.parse(
|
|
6447
|
+
JSON.parse(decode(bytes, "gauge.json"))
|
|
6448
|
+
);
|
|
6449
|
+
} catch (error) {
|
|
6450
|
+
throw new CliError(
|
|
6451
|
+
"usage",
|
|
6452
|
+
`gauge.json: ${error instanceof Error ? error.message : error}`
|
|
6453
|
+
);
|
|
6454
|
+
}
|
|
6455
|
+
}
|
|
6456
|
+
const hasConventionalPrepare = git(root, ["ls-tree", "--name-only", commit, "--", ".gauge/prepare.sh"]).toString("utf8").trim() === ".gauge/prepare.sh";
|
|
6457
|
+
if (hasConventionalPrepare) {
|
|
6458
|
+
blobs.push({
|
|
6459
|
+
path: ".gauge/prepare.sh",
|
|
6460
|
+
bytes: committedBlob(root, commit, ".gauge/prepare.sh")
|
|
6461
|
+
});
|
|
6462
|
+
}
|
|
6463
|
+
if (config?.product && hasConventionalPrepare)
|
|
6464
|
+
throw new CliError(
|
|
6465
|
+
"usage",
|
|
6466
|
+
"gauge.json: product and .gauge/prepare.sh are alternatives"
|
|
6467
|
+
);
|
|
6468
|
+
const prepare = config?.prepare ?? (hasConventionalPrepare ? "bash .gauge/prepare.sh" : void 0);
|
|
6469
|
+
const preparation = config?.product ?? (prepare ? { run: prepare } : void 0);
|
|
6470
|
+
if (config?.install && !prepare)
|
|
6471
|
+
throw new CliError("usage", "gauge.json: install requires prepare");
|
|
6472
|
+
if (preparation && !repoUrl.startsWith("https://"))
|
|
6473
|
+
throw new CliError(
|
|
6474
|
+
"usage",
|
|
6475
|
+
"Project preparation requires a public HTTPS Git origin"
|
|
6476
|
+
);
|
|
6477
|
+
const cases = paths.map((path2) => {
|
|
6478
|
+
const bytes = committedBlob(root, commit, path2);
|
|
6479
|
+
blobs.push({ path: path2, bytes });
|
|
6480
|
+
return caseFromMarkdown(path2, decode(bytes, path2));
|
|
6481
|
+
});
|
|
6482
|
+
const hash = createHash2("sha256");
|
|
6483
|
+
hash.update("gauge-run-definition-v1\0");
|
|
6484
|
+
for (const entry of blobs.sort(
|
|
6485
|
+
(a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0
|
|
6486
|
+
)) {
|
|
6487
|
+
hash.update(entry.path);
|
|
6488
|
+
hash.update("\0");
|
|
6489
|
+
hash.update(String(entry.bytes.length));
|
|
6490
|
+
hash.update("\0");
|
|
6491
|
+
hash.update(entry.bytes);
|
|
6492
|
+
hash.update("\0");
|
|
6493
|
+
}
|
|
6494
|
+
const submission = runDefinitionSubmissionSchema.safeParse({
|
|
6495
|
+
version: 1,
|
|
6496
|
+
source: {
|
|
6497
|
+
repoUrl,
|
|
6498
|
+
commit,
|
|
6499
|
+
digest: `sha256:${hash.digest("hex")}`,
|
|
6500
|
+
verification: "SUBMITTED_UNVERIFIED"
|
|
6501
|
+
},
|
|
6502
|
+
resources: preparation ? [
|
|
6503
|
+
{
|
|
6504
|
+
name: "project",
|
|
6505
|
+
type: "prepared-directory",
|
|
6506
|
+
source: { type: "git-checkout", repoUrl, commit },
|
|
6507
|
+
prepare: preparation,
|
|
6508
|
+
...config?.install ? { install: config.install } : {}
|
|
6509
|
+
}
|
|
6510
|
+
] : [],
|
|
6511
|
+
cases
|
|
6512
|
+
});
|
|
6513
|
+
if (!submission.success)
|
|
6514
|
+
throw new CliError("usage", submission.error.message);
|
|
6515
|
+
return { definition: submission.data, org: config?.org };
|
|
6516
|
+
}
|
|
6517
|
+
|
|
6518
|
+
// src/lib/staff.ts
|
|
6519
|
+
function staffCommandsVisible(config = readConfig()) {
|
|
6520
|
+
return config.staff === true;
|
|
6521
|
+
}
|
|
6522
|
+
|
|
5395
6523
|
// src/commands/evals.ts
|
|
5396
6524
|
function evalSetsPath(org) {
|
|
5397
6525
|
return `/api/v1/orgs/${encodeURIComponent(org)}/eval-sets`;
|
|
@@ -5470,6 +6598,40 @@ function printDetail2(e) {
|
|
|
5470
6598
|
}
|
|
5471
6599
|
function register11(program2) {
|
|
5472
6600
|
const group = program2.command("evals").description("Manage eval sets (judged agent-experience measurements)");
|
|
6601
|
+
group.command("plan").description(
|
|
6602
|
+
"Validate committed Markdown cases and preview a Git-native run"
|
|
6603
|
+
).requiredOption("--files <paths...>", "explicit committed Markdown paths").action(async (opts, cmd) => {
|
|
6604
|
+
const { definition, org: fileOrg } = loadCommittedRunDefinitionWithContext(opts.files);
|
|
6605
|
+
const org = resolveOrg(cmd, fileOrg);
|
|
6606
|
+
const result = await createClient().post(
|
|
6607
|
+
`/api/v1/orgs/${encodeURIComponent(org)}/run-definitions/plan`,
|
|
6608
|
+
definition
|
|
6609
|
+
);
|
|
6610
|
+
if (resolveOutput(cmd) === "json") {
|
|
6611
|
+
printJson(result);
|
|
6612
|
+
return;
|
|
6613
|
+
}
|
|
6614
|
+
console.log(
|
|
6615
|
+
`Definition: ${result.source.commit} (${result.source.digest})`
|
|
6616
|
+
);
|
|
6617
|
+
console.log(
|
|
6618
|
+
"Provenance: submitted by CLI; server has not verified Git bytes"
|
|
6619
|
+
);
|
|
6620
|
+
console.log(`Expected sessions: ${result.expectedRuns}`);
|
|
6621
|
+
for (const item of result.cases)
|
|
6622
|
+
console.log(
|
|
6623
|
+
` ${item.path}: ${item.expectedRuns} sessions, ${item.criteriaCount} criteria`
|
|
6624
|
+
);
|
|
6625
|
+
if (result.resources.length) {
|
|
6626
|
+
console.log(
|
|
6627
|
+
`Preparations: ${result.resources.map((r) => r.name).join(", ")}`
|
|
6628
|
+
);
|
|
6629
|
+
if (!result.canRun)
|
|
6630
|
+
console.log(
|
|
6631
|
+
"Resource-backed runs cannot launch until preparation is available."
|
|
6632
|
+
);
|
|
6633
|
+
}
|
|
6634
|
+
});
|
|
5473
6635
|
group.command("list").description("List the org's eval sets with pass-rate stats").option(
|
|
5474
6636
|
"--agents <agents>",
|
|
5475
6637
|
"filter stats to these agents, e.g. claude-code,codex"
|
|
@@ -5589,24 +6751,56 @@ function register11(program2) {
|
|
|
5589
6751
|
);
|
|
5590
6752
|
if (resolveOutput(cmd) !== "json") console.log(`Deleted eval set ${id}`);
|
|
5591
6753
|
});
|
|
5592
|
-
group.command("run
|
|
5593
|
-
"Run an eval set now with its stored configuration (
|
|
5594
|
-
).
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
"
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
|
|
5607
|
-
|
|
6754
|
+
group.command("run [id]").description(
|
|
6755
|
+
staffCommandsVisible() ? "Run an eval set now with its stored configuration (superusers run without billing the org by default)" : "Run an eval set now with its stored configuration"
|
|
6756
|
+
).addOption(
|
|
6757
|
+
new Option2(
|
|
6758
|
+
"--bill-to-org",
|
|
6759
|
+
"bill this run to the org's credits instead (superusers)"
|
|
6760
|
+
).hideHelp(!staffCommandsVisible())
|
|
6761
|
+
).option("--files <paths...>", "explicit committed Markdown paths").option("--yes", "skip the confirmation").action(
|
|
6762
|
+
async (id, opts, cmd) => {
|
|
6763
|
+
if (opts.files !== void 0) {
|
|
6764
|
+
if (id !== void 0)
|
|
6765
|
+
throw new CliError(
|
|
6766
|
+
"usage",
|
|
6767
|
+
"pass either an eval ID or --files, not both"
|
|
6768
|
+
);
|
|
6769
|
+
const { definition, org: fileOrg } = loadCommittedRunDefinitionWithContext(opts.files);
|
|
6770
|
+
const org2 = resolveOrg(cmd, fileOrg);
|
|
6771
|
+
await confirmOrAbort("Run these committed eval cases now?", opts.yes);
|
|
6772
|
+
const result2 = await createClient().post(
|
|
6773
|
+
`/api/v1/orgs/${encodeURIComponent(org2)}/run-definitions/run`,
|
|
6774
|
+
definition,
|
|
6775
|
+
opts.billToOrg ? { billToOrg: true } : void 0
|
|
6776
|
+
);
|
|
6777
|
+
if (resolveOutput(cmd) === "json") printJson(result2);
|
|
6778
|
+
else {
|
|
6779
|
+
console.log(
|
|
6780
|
+
`Started ${result2.createdRuns}/${result2.requestedRuns} sessions in request ${result2.runRequestId}`
|
|
6781
|
+
);
|
|
6782
|
+
console.log(`Wait: gauge run-requests wait ${result2.runRequestId}`);
|
|
6783
|
+
}
|
|
6784
|
+
return;
|
|
6785
|
+
}
|
|
6786
|
+
if (id === void 0)
|
|
6787
|
+
throw new CliError("usage", "pass an eval ID or --files <paths...>");
|
|
6788
|
+
const org = resolveOrg(cmd);
|
|
6789
|
+
await confirmOrAbort(
|
|
6790
|
+
"Run this eval's stored configuration now?",
|
|
6791
|
+
opts.yes
|
|
5608
6792
|
);
|
|
5609
|
-
|
|
6793
|
+
const result = await createClient().post(
|
|
6794
|
+
`${evalSetsPath(org)}/${encodeURIComponent(id)}/run`,
|
|
6795
|
+
opts.billToOrg ? { billToOrg: true } : {}
|
|
6796
|
+
);
|
|
6797
|
+
if (resolveOutput(cmd) === "json") printJson(result);
|
|
6798
|
+
else
|
|
6799
|
+
console.log(
|
|
6800
|
+
`Started ${result.runIds.length} run${result.runIds.length === 1 ? "" : "s"} in batch ${result.batchIds[0] ?? ""}`
|
|
6801
|
+
);
|
|
6802
|
+
}
|
|
6803
|
+
);
|
|
5610
6804
|
group.command("runs <id>").description("The eval set's run log with per-criterion verdicts").option("--limit <n>", "max runs (default 100)").action(async (id, opts, cmd) => {
|
|
5611
6805
|
const org = resolveOrg(cmd);
|
|
5612
6806
|
const res = await createClient().get(
|
|
@@ -5686,7 +6880,7 @@ function rewriteContentType(path2, explicit) {
|
|
|
5686
6880
|
return REWRITE_FILE_CONTENT_TYPES[extname(path2).toLowerCase()];
|
|
5687
6881
|
}
|
|
5688
6882
|
function sleep2(ms) {
|
|
5689
|
-
return new Promise((
|
|
6883
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
5690
6884
|
}
|
|
5691
6885
|
function toRow3(e) {
|
|
5692
6886
|
return {
|
|
@@ -5723,7 +6917,7 @@ function printDetail3(e) {
|
|
|
5723
6917
|
}
|
|
5724
6918
|
function register12(program2) {
|
|
5725
6919
|
const group = program2.command("experiments").description(
|
|
5726
|
-
"Author, launch, and read
|
|
6920
|
+
"Author, launch, and read content experiments (legacy: orgs with Optimizations reject writes; use `gauge optimizations`)"
|
|
5727
6921
|
);
|
|
5728
6922
|
group.command("list").description("List the org's experiments").option(
|
|
5729
6923
|
"--status <statuses>",
|
|
@@ -5769,13 +6963,13 @@ function register12(program2) {
|
|
|
5769
6963
|
);
|
|
5770
6964
|
});
|
|
5771
6965
|
group.command("create").description(
|
|
5772
|
-
"Open (or resume) an experiment draft and start the
|
|
6966
|
+
"Open (or resume) an experiment draft and start the plan round"
|
|
5773
6967
|
).requiredOption("--run <runId>", "source run id").requiredOption(
|
|
5774
6968
|
"--boundary <seq>",
|
|
5775
6969
|
"opaque boundarySeq from `experiments boundaries`"
|
|
5776
6970
|
).option("--problem <text>", "problem statement (derived if omitted)").option(
|
|
5777
6971
|
"--mode <mode>",
|
|
5778
|
-
"who writes the rewrites: propose (default,
|
|
6972
|
+
"who writes the rewrites: propose (default, directions are proposed then each is written), direct (one --change instruction, one write), manual (no generation \u2014 the rewrite is seeded with the page the agent read)",
|
|
5779
6973
|
"propose"
|
|
5780
6974
|
).option(
|
|
5781
6975
|
"--change <text>",
|
|
@@ -5959,13 +7153,13 @@ function register12(program2) {
|
|
|
5959
7153
|
}
|
|
5960
7154
|
}
|
|
5961
7155
|
);
|
|
5962
|
-
group.command("replan <id>").description("Discard the directions and
|
|
7156
|
+
group.command("replan <id>").description("Discard the directions and propose fresh ones").action(async (id, _opts, cmd) => {
|
|
5963
7157
|
const org = resolveOrg(cmd);
|
|
5964
7158
|
await createClient().post(`${itemPath(org, id)}/replan`, {});
|
|
5965
7159
|
if (resolveOutput(cmd) !== "json")
|
|
5966
7160
|
console.log("Replanning \u2014 follow with `gauge experiments watch`");
|
|
5967
7161
|
});
|
|
5968
|
-
group.command("write <id>").description("
|
|
7162
|
+
group.command("write <id>").description("Write rewrites for directions that lack one").action(async (id, _opts, cmd) => {
|
|
5969
7163
|
const org = resolveOrg(cmd);
|
|
5970
7164
|
const result = await createClient().post(
|
|
5971
7165
|
`${itemPath(org, id)}/write`,
|
|
@@ -6411,7 +7605,7 @@ function register15(program2) {
|
|
|
6411
7605
|
});
|
|
6412
7606
|
group.command("add").description(
|
|
6413
7607
|
"Add a server, or append a version to an existing --name (versions are immutable)"
|
|
6414
|
-
).requiredOption("--name <handle>", "
|
|
7608
|
+
).requiredOption("--name <handle>", "the handle (the agent config key)").option("--command <cmd>", "stdio launch command, e.g. npx").option(
|
|
6415
7609
|
"--arg <arg>",
|
|
6416
7610
|
"stdio argument (repeatable)",
|
|
6417
7611
|
collect,
|
|
@@ -6484,7 +7678,7 @@ function register15(program2) {
|
|
|
6484
7678
|
}
|
|
6485
7679
|
);
|
|
6486
7680
|
group.command("rm <ref>").description(
|
|
6487
|
-
"
|
|
7681
|
+
"Remove a server; the handle stays reserved and finished runs keep their snapshot"
|
|
6488
7682
|
).action(async (ref, _opts, cmd) => {
|
|
6489
7683
|
const org = resolveOrg(cmd);
|
|
6490
7684
|
const result = await createClient().delete(serverPath(org, ref));
|
|
@@ -6695,10 +7889,11 @@ __export(optimizations_exports2, {
|
|
|
6695
7889
|
readSkillDir: () => readSkillDir,
|
|
6696
7890
|
register: () => register19,
|
|
6697
7891
|
trialBodyFromOpts: () => trialBodyFromOpts,
|
|
6698
|
-
trialTemplate: () => trialTemplate
|
|
7892
|
+
trialTemplate: () => trialTemplate,
|
|
7893
|
+
watchStops: () => watchStops
|
|
6699
7894
|
});
|
|
6700
7895
|
import { readdirSync, readFileSync as readFileSync9, statSync } from "fs";
|
|
6701
|
-
import { join as join4, relative, sep } from "path";
|
|
7896
|
+
import { join as join4, relative as relative2, sep as sep2 } from "path";
|
|
6702
7897
|
var TERMINAL2 = /* @__PURE__ */ new Set(["DONE", "ARCHIVED"]);
|
|
6703
7898
|
var EFFORTS = ["low", "medium", "high", "xhigh"];
|
|
6704
7899
|
function basePath(org) {
|
|
@@ -6714,7 +7909,7 @@ function readText2(path2) {
|
|
|
6714
7909
|
return readFileSync9(path2 === "-" ? 0 : path2, "utf8");
|
|
6715
7910
|
}
|
|
6716
7911
|
function sleep3(ms) {
|
|
6717
|
-
return new Promise((
|
|
7912
|
+
return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
6718
7913
|
}
|
|
6719
7914
|
function parseSubject(opts) {
|
|
6720
7915
|
if (opts.eval && opts.preference)
|
|
@@ -6767,11 +7962,14 @@ var NEXT_ACTION_HINT = {
|
|
|
6767
7962
|
LAUNCH_ROUND: "launch the saved round: `gauge optimizations launch <id>`",
|
|
6768
7963
|
ADOPT: "adopt the winning trial: `gauge optimizations adopt <id> <trial>`",
|
|
6769
7964
|
RESUME: "resume it: `gauge optimizations resume <id>`",
|
|
6770
|
-
NONE: "
|
|
7965
|
+
NONE: "nothing to do in Gauge"
|
|
6771
7966
|
};
|
|
6772
7967
|
function awaitingInput(detail) {
|
|
6773
7968
|
return detail.nextAction !== "WAIT" && detail.nextAction !== "NONE";
|
|
6774
7969
|
}
|
|
7970
|
+
function watchStops(detail, follow) {
|
|
7971
|
+
return TERMINAL2.has(detail.status) || detail.nextAction === "NONE" || !follow && awaitingInput(detail);
|
|
7972
|
+
}
|
|
6775
7973
|
function readObject(path2) {
|
|
6776
7974
|
let value;
|
|
6777
7975
|
try {
|
|
@@ -6813,7 +8011,7 @@ function readSkillDir(dir, read = readText2, walk = listFiles) {
|
|
|
6813
8011
|
if (!paths.length)
|
|
6814
8012
|
throw new CliError("usage", `no files under ${dir} to send`);
|
|
6815
8013
|
const files = paths.map((path2) => ({
|
|
6816
|
-
path:
|
|
8014
|
+
path: relative2(dir, path2).split(sep2).join("/"),
|
|
6817
8015
|
body: read(path2)
|
|
6818
8016
|
}));
|
|
6819
8017
|
if (!files.some((file) => file.path === "SKILL.md"))
|
|
@@ -6883,7 +8081,6 @@ function trialBodyFromOpts(opts, read = readText2, walk) {
|
|
|
6883
8081
|
}
|
|
6884
8082
|
if (!opts.title) throw new CliError("usage", "--title is required");
|
|
6885
8083
|
const lever = opts.lever?.toUpperCase();
|
|
6886
|
-
const kind = opts.kind?.toUpperCase();
|
|
6887
8084
|
let intervention;
|
|
6888
8085
|
let files;
|
|
6889
8086
|
if (opts.page || opts.bodyFile) {
|
|
@@ -6893,12 +8090,6 @@ function trialBodyFromOpts(opts, read = readText2, walk) {
|
|
|
6893
8090
|
kind: "REWRITE_DOCS",
|
|
6894
8091
|
after: { pages: [{ url: opts.page, body: read(opts.bodyFile) }] }
|
|
6895
8092
|
};
|
|
6896
|
-
} else if (opts.removeSkill) {
|
|
6897
|
-
intervention = {
|
|
6898
|
-
kind: "REMOVE_SKILL",
|
|
6899
|
-
before: { skillVersionId: opts.removeSkill },
|
|
6900
|
-
after: {}
|
|
6901
|
-
};
|
|
6902
8093
|
} else if (opts.editSkill) {
|
|
6903
8094
|
if (!opts.skillDir)
|
|
6904
8095
|
throw new CliError(
|
|
@@ -6911,21 +8102,10 @@ function trialBodyFromOpts(opts, read = readText2, walk) {
|
|
|
6911
8102
|
after: { skillVersionId: opts.editSkill }
|
|
6912
8103
|
};
|
|
6913
8104
|
files = readSkillDir(opts.skillDir, read, walk);
|
|
6914
|
-
} else if (opts.skillVersion) {
|
|
6915
|
-
if (opts.skillDir)
|
|
6916
|
-
throw new CliError(
|
|
6917
|
-
"usage",
|
|
6918
|
-
"--skill-dir edits a skill: pass --edit-skill <id>, not --skill-version"
|
|
6919
|
-
);
|
|
6920
|
-
intervention = {
|
|
6921
|
-
kind: kind ?? "ADD_SKILL",
|
|
6922
|
-
after: { skillVersionId: opts.skillVersion },
|
|
6923
|
-
...kind === "EDIT_SKILL" ? { before: { skillVersionId: opts.skillVersion } } : {}
|
|
6924
|
-
};
|
|
6925
8105
|
} else {
|
|
6926
8106
|
throw new CliError(
|
|
6927
8107
|
"usage",
|
|
6928
|
-
"describe the change: --
|
|
8108
|
+
"describe the change: --edit-skill <id> --skill-dir <path>, --page <url> --body-file <path>, or --file <trial.json>"
|
|
6929
8109
|
);
|
|
6930
8110
|
}
|
|
6931
8111
|
const trial = {
|
|
@@ -6944,18 +8124,6 @@ function trialBodyFromOpts(opts, read = readText2, walk) {
|
|
|
6944
8124
|
};
|
|
6945
8125
|
}
|
|
6946
8126
|
var TRIAL_TEMPLATES = {
|
|
6947
|
-
ADD_SKILL: {
|
|
6948
|
-
trial: {
|
|
6949
|
-
title: "Add the troubleshooting skill",
|
|
6950
|
-
hypothesis: "The agent misses the retry rule without it",
|
|
6951
|
-
lever: "SKILL",
|
|
6952
|
-
intervention: {
|
|
6953
|
-
kind: "ADD_SKILL",
|
|
6954
|
-
after: { skillVersionId: "sv_REPLACE_ME" }
|
|
6955
|
-
},
|
|
6956
|
-
authorMode: "MANUAL"
|
|
6957
|
-
}
|
|
6958
|
-
},
|
|
6959
8127
|
EDIT_SKILL: {
|
|
6960
8128
|
trial: {
|
|
6961
8129
|
title: "Name the env var in the first line",
|
|
@@ -6972,19 +8140,6 @@ var TRIAL_TEMPLATES = {
|
|
|
6972
8140
|
{ path: "SKILL.md", body: "# Skill\n\nThe complete rewritten file.\n" }
|
|
6973
8141
|
]
|
|
6974
8142
|
},
|
|
6975
|
-
REMOVE_SKILL: {
|
|
6976
|
-
trial: {
|
|
6977
|
-
title: "Drop the legacy skill",
|
|
6978
|
-
hypothesis: "It contradicts the quickstart",
|
|
6979
|
-
lever: "SKILL",
|
|
6980
|
-
intervention: {
|
|
6981
|
-
kind: "REMOVE_SKILL",
|
|
6982
|
-
before: { skillVersionId: "sv_REPLACE_ME" },
|
|
6983
|
-
after: {}
|
|
6984
|
-
},
|
|
6985
|
-
authorMode: "MANUAL"
|
|
6986
|
-
}
|
|
6987
|
-
},
|
|
6988
8143
|
REWRITE_DOCS: {
|
|
6989
8144
|
trial: {
|
|
6990
8145
|
title: "Move the auth note above the example",
|
|
@@ -7041,12 +8196,11 @@ function trialRow(t) {
|
|
|
7041
8196
|
title: t.title.length > 48 ? `${t.title.slice(0, 45)}...` : t.title,
|
|
7042
8197
|
surface: t.lever ?? "",
|
|
7043
8198
|
change: t.kind,
|
|
7044
|
-
"builds on": t.parentKey ?? "",
|
|
7045
8199
|
status: t.status,
|
|
7046
8200
|
score: pct(t.score),
|
|
7047
8201
|
result: t.verdict ?? "",
|
|
7048
8202
|
sessions: t.sessions,
|
|
7049
|
-
|
|
8203
|
+
winner: t.adopted ? "yes" : "",
|
|
7050
8204
|
id: t.id
|
|
7051
8205
|
};
|
|
7052
8206
|
}
|
|
@@ -7066,7 +8220,7 @@ function printDetail4(o) {
|
|
|
7066
8220
|
`target: ${o.settings.pinnedAgent ?? "(subject roster)"}${o.settings.pinnedModel ? `:${o.settings.pinnedModel}` : ""}`
|
|
7067
8221
|
);
|
|
7068
8222
|
console.log(
|
|
7069
|
-
`plan: ${o.settings.sampleCount} session${o.settings.sampleCount === 1 ? "" : "s"} per trial, up to ${o.settings.maxRounds} rounds
|
|
8223
|
+
`plan: ${o.settings.sampleCount} session${o.settings.sampleCount === 1 ? "" : "s"} per trial, up to ${o.settings.maxRounds} rounds`
|
|
7070
8224
|
);
|
|
7071
8225
|
console.log(
|
|
7072
8226
|
`credits: ${o.creditsSpent} spent; ${o.settings.creditCap} estimated (not a spending cap)`
|
|
@@ -7089,9 +8243,8 @@ function printTrial(t) {
|
|
|
7089
8243
|
console.log(`title: ${t.title}`);
|
|
7090
8244
|
if (t.hypothesis) console.log(`hypothesis: ${t.hypothesis}`);
|
|
7091
8245
|
console.log(`change: ${t.kind}${t.lever ? ` on ${t.lever}` : ""}`);
|
|
7092
|
-
console.log(`builds on: ${t.parentKey ?? "\u2014"}`);
|
|
7093
8246
|
console.log(
|
|
7094
|
-
`status: ${t.status}${t.verdict ? ` \xB7 ${t.verdict}` : ""}${t.adopted ? " \xB7
|
|
8247
|
+
`status: ${t.status}${t.verdict ? ` \xB7 ${t.verdict}` : ""}${t.adopted ? " \xB7 winner" : ""}`
|
|
7095
8248
|
);
|
|
7096
8249
|
console.log(
|
|
7097
8250
|
`score: ${pct(t.baselineScore)} \u2192 ${pct(t.score)} over ${t.sessions} session${t.sessions === 1 ? "" : "s"}`
|
|
@@ -7165,7 +8318,7 @@ function addPlanOptions(command) {
|
|
|
7165
8318
|
}
|
|
7166
8319
|
function register19(program2) {
|
|
7167
8320
|
const group = program2.command("optimizations").alias("optimize").description(
|
|
7168
|
-
"Start, steer, and read optimizations: Gauge changes one owned surface per trial and
|
|
8321
|
+
"Start, steer, and read optimizations: Gauge changes one owned surface per trial and reports the best measured change against the baseline"
|
|
7169
8322
|
);
|
|
7170
8323
|
group.command("list").description("List the org's optimizations, newest activity first").option(
|
|
7171
8324
|
"--status <statuses>",
|
|
@@ -7226,10 +8379,10 @@ function register19(program2) {
|
|
|
7226
8379
|
"Measure a baseline; author and explicitly launch candidate rounds afterward"
|
|
7227
8380
|
)
|
|
7228
8381
|
)
|
|
7229
|
-
).option("--name <name>", "
|
|
8382
|
+
).option("--name <name>", "optimization name (default: from the subject)").option(
|
|
7230
8383
|
"--credit-cap <n>",
|
|
7231
8384
|
"record this credit estimate instead (not an enforced cap)"
|
|
7232
|
-
).option("--target-score <0-1>", "stop once the score reaches this").option("--
|
|
8385
|
+
).option("--target-score <0-1>", "stop once the score reaches this").option("--yes", "skip the confirmation").action(
|
|
7233
8386
|
async (opts, cmd) => {
|
|
7234
8387
|
const org = resolveOrg(cmd);
|
|
7235
8388
|
if (!opts.target)
|
|
@@ -7258,7 +8411,6 @@ function register19(program2) {
|
|
|
7258
8411
|
...plan,
|
|
7259
8412
|
name: opts.name,
|
|
7260
8413
|
creditCap,
|
|
7261
|
-
autoAdopt: opts.autoAdopt,
|
|
7262
8414
|
targetScore
|
|
7263
8415
|
});
|
|
7264
8416
|
if (resolveOutput(cmd) === "json") printJson(detail);
|
|
@@ -7392,9 +8544,8 @@ function register19(program2) {
|
|
|
7392
8544
|
result.alreadyLaunched ? "Round already launched." : `Launched ${result.launched.length} trials in round ${result.roundNumber}.`
|
|
7393
8545
|
);
|
|
7394
8546
|
});
|
|
7395
|
-
group.command("continue <id>").description(
|
|
7396
|
-
"
|
|
7397
|
-
"starting trial id or key, e.g. B0 or R1.1"
|
|
8547
|
+
group.command("continue <id>").description(
|
|
8548
|
+
"Add rounds to a finished optimization against its fixed baseline"
|
|
7398
8549
|
).option("--rounds <n>", "additional rounds (1\u201320, default 1)").option("--trials-per-round <n>", "maximum candidates per round (1\u20133)").option("--yes", "skip the confirmation").action(
|
|
7399
8550
|
async (id, opts, cmd) => {
|
|
7400
8551
|
const rounds = parseCount(opts.rounds, "--rounds", 20) ?? 1;
|
|
@@ -7404,12 +8555,12 @@ function register19(program2) {
|
|
|
7404
8555
|
3
|
|
7405
8556
|
);
|
|
7406
8557
|
await confirmOrAbort(
|
|
7407
|
-
`Continue ${id}
|
|
8558
|
+
`Continue ${id} for up to ${rounds} additional rounds? Saved trials still require launch.`,
|
|
7408
8559
|
opts.yes
|
|
7409
8560
|
);
|
|
7410
8561
|
const detail = await createClient().post(
|
|
7411
8562
|
`${itemPath2(resolveOrg(cmd), id)}/continue`,
|
|
7412
|
-
{
|
|
8563
|
+
{ rounds, maxTrialsPerRound }
|
|
7413
8564
|
);
|
|
7414
8565
|
if (resolveOutput(cmd) === "json") printJson(detail);
|
|
7415
8566
|
else printDetail4(detail);
|
|
@@ -7461,7 +8612,7 @@ function register19(program2) {
|
|
|
7461
8612
|
if (resolveOutput(cmd) === "json") printJson({ deleted: id });
|
|
7462
8613
|
else console.log(`Deleted ${id}.`);
|
|
7463
8614
|
});
|
|
7464
|
-
group.command("complete <id>").description("Complete an optimization: it reads as Finished, not canceled").option("--adopt <trialId>", "
|
|
8615
|
+
group.command("complete <id>").description("Complete an optimization: it reads as Finished, not canceled").option("--adopt <trialId>", "mark this settled trial as the winner").option("--yes", "skip the confirmation").action(
|
|
7465
8616
|
async (id, opts, cmd) => {
|
|
7466
8617
|
const org = resolveOrg(cmd);
|
|
7467
8618
|
await confirmOrAbort(
|
|
@@ -7494,13 +8645,13 @@ function register19(program2) {
|
|
|
7494
8645
|
for (; ; ) {
|
|
7495
8646
|
const detail = await client.get(itemPath2(org, id));
|
|
7496
8647
|
const trials2 = detail.rounds.flatMap((round) => round.trials);
|
|
7497
|
-
const line = `${detail.status} \xB7 round ${detail.round}/${detail.maxRounds} \xB7 ${pct(detail.before)} \u2192 ${pct(detail.after)} \xB7 ${detail.creditsSpent} credits spent \xB7 ${trials2.filter((t) => t.status === "RUNNING").map((t) => t.key).join(",") || "idle"}`;
|
|
8648
|
+
const line = `${detail.status}${detail.stopReason ? ` (${detail.stopReason})` : ""} \xB7 round ${detail.round}/${detail.maxRounds} \xB7 ${pct(detail.before)} \u2192 ${pct(detail.after)} \xB7 ${detail.creditsSpent} credits spent \xB7 ${trials2.filter((t) => t.status === "RUNNING").map((t) => t.key).join(",") || "idle"}`;
|
|
7498
8649
|
if (line !== last) {
|
|
7499
8650
|
last = line;
|
|
7500
8651
|
console.error(line);
|
|
7501
8652
|
}
|
|
7502
8653
|
const needsInput = awaitingInput(detail);
|
|
7503
|
-
if (
|
|
8654
|
+
if (watchStops(detail, !!opts.follow)) {
|
|
7504
8655
|
if (resolveOutput(cmd) === "json") printJson(detail);
|
|
7505
8656
|
else printDetail4(detail);
|
|
7506
8657
|
if (needsInput) process.exitCode = NEEDS_INPUT_EXIT;
|
|
@@ -7525,15 +8676,12 @@ function register19(program2) {
|
|
|
7525
8676
|
"--file <path>",
|
|
7526
8677
|
"authored trial JSON, as `trials template` prints it: one {trial, files?} or an array; '-' for stdin"
|
|
7527
8678
|
).option("--pick <n>", "which trial to take from an array (default 1)").option("--title <text>", "what the change is").option("--hypothesis <text>", "why it should help").option("--lever <lever>", "DOCS, MARKETING_SITE, SKILL, SDK, CLI").option(
|
|
7528
|
-
"--kind <kind>",
|
|
7529
|
-
"override the change kind the other flags imply; rarely needed"
|
|
7530
|
-
).option("--skill-version <id>", "skill version to add").option(
|
|
7531
8679
|
"--edit-skill <id>",
|
|
7532
8680
|
"skill version to rewrite (with --skill-dir); see `gauge optimizations skill-files <id>`"
|
|
7533
8681
|
).option(
|
|
7534
8682
|
"--skill-dir <path>",
|
|
7535
8683
|
"local directory holding the complete edited bundle, SKILL.md included"
|
|
7536
|
-
).option("--
|
|
8684
|
+
).option("--page <url>", "docs page to rewrite (with --body-file)").option("--body-file <path>", "the rewritten page body; '-' for stdin").option("--fork-run <runId>", "fork-at-fetch source run for a docs rewrite").option(
|
|
7537
8685
|
"--fork-boundary <seq>",
|
|
7538
8686
|
"fork-at-fetch boundary from `gauge experiments boundaries`"
|
|
7539
8687
|
).option(
|
|
@@ -7563,10 +8711,7 @@ function register19(program2) {
|
|
|
7563
8711
|
);
|
|
7564
8712
|
}
|
|
7565
8713
|
);
|
|
7566
|
-
trials.command("template").description("Print a trial JSON skeleton for `trials add --file`").requiredOption(
|
|
7567
|
-
"--kind <kind>",
|
|
7568
|
-
"ADD_SKILL, EDIT_SKILL, REMOVE_SKILL, or REWRITE_DOCS"
|
|
7569
|
-
).action((opts) => {
|
|
8714
|
+
trials.command("template").description("Print a trial JSON skeleton for `trials add --file`").requiredOption("--kind <kind>", "EDIT_SKILL or REWRITE_DOCS").action((opts) => {
|
|
7570
8715
|
printJson(trialTemplate(opts.kind));
|
|
7571
8716
|
});
|
|
7572
8717
|
trials.command("run <id> <trialId>").description("Run a draft trial now (spends credits)").option("--yes", "skip the confirmation").action(
|
|
@@ -7615,12 +8760,12 @@ function register19(program2) {
|
|
|
7615
8760
|
}
|
|
7616
8761
|
);
|
|
7617
8762
|
group.command("adopt <id> <trialId>").description(
|
|
7618
|
-
"
|
|
8763
|
+
"Mark a settled trial as the winner without changing the baseline"
|
|
7619
8764
|
).option("--yes", "skip the confirmation").action(
|
|
7620
8765
|
async (id, trialId, opts, cmd) => {
|
|
7621
8766
|
const org = resolveOrg(cmd);
|
|
7622
8767
|
await confirmOrAbort(
|
|
7623
|
-
`
|
|
8768
|
+
`Mark ${trialId} as the winner on ${id}?`,
|
|
7624
8769
|
opts.yes
|
|
7625
8770
|
);
|
|
7626
8771
|
const detail = await createClient().post(
|
|
@@ -7628,14 +8773,11 @@ function register19(program2) {
|
|
|
7628
8773
|
{}
|
|
7629
8774
|
);
|
|
7630
8775
|
if (resolveOutput(cmd) === "json") printJson(detail);
|
|
7631
|
-
else
|
|
7632
|
-
console.log(
|
|
7633
|
-
`Adopted ${trialId}; baseline is now ${pct(detail.after)}`
|
|
7634
|
-
);
|
|
8776
|
+
else console.log(`Marked ${trialId} as the winner.`);
|
|
7635
8777
|
}
|
|
7636
8778
|
);
|
|
7637
8779
|
group.command("skill-files <versionId>").description(
|
|
7638
|
-
"The text files of a skill version, the base
|
|
8780
|
+
"The text files of a skill version, the base a skill-edit trial edits"
|
|
7639
8781
|
).action(async (versionId, _opts, cmd) => {
|
|
7640
8782
|
const org = resolveOrg(cmd);
|
|
7641
8783
|
const res = await createClient().get(
|
|
@@ -7649,7 +8791,7 @@ function register19(program2) {
|
|
|
7649
8791
|
}
|
|
7650
8792
|
});
|
|
7651
8793
|
group.command("page <id>").description(
|
|
7652
|
-
"The baseline body of a docs page, the base a
|
|
8794
|
+
"The baseline body of a docs page, the base a docs-rewrite trial edits"
|
|
7653
8795
|
).requiredOption("--url <url>", "the page URL").action(async (id, opts, cmd) => {
|
|
7654
8796
|
const org = resolveOrg(cmd);
|
|
7655
8797
|
const res = await createClient().get(
|
|
@@ -8194,10 +9336,48 @@ function register24(program2) {
|
|
|
8194
9336
|
});
|
|
8195
9337
|
}
|
|
8196
9338
|
|
|
9339
|
+
// src/commands/runRequests.ts
|
|
9340
|
+
var runRequests_exports2 = {};
|
|
9341
|
+
__export(runRequests_exports2, {
|
|
9342
|
+
register: () => register25
|
|
9343
|
+
});
|
|
9344
|
+
var pollIntervalMs = 5e3;
|
|
9345
|
+
function register25(program2) {
|
|
9346
|
+
program2.command("run-requests").description("Read a durable launch and its aggregate verdict").command("wait <id>").description("Wait for all sessions and judging in a launch request").action(async (id, _opts, cmd) => {
|
|
9347
|
+
const org = resolveOrg(cmd);
|
|
9348
|
+
const path2 = `/api/v1/orgs/${encodeURIComponent(org)}/run-requests/${encodeURIComponent(id)}`;
|
|
9349
|
+
const client = createClient();
|
|
9350
|
+
let lastStatus = "";
|
|
9351
|
+
while (true) {
|
|
9352
|
+
const result = await client.get(path2);
|
|
9353
|
+
if (resolveOutput(cmd) !== "json" && result.status !== lastStatus) {
|
|
9354
|
+
console.error(`Run request ${id}: ${result.status}`);
|
|
9355
|
+
lastStatus = result.status;
|
|
9356
|
+
}
|
|
9357
|
+
if (result.settled) {
|
|
9358
|
+
if (resolveOutput(cmd) === "json") printJson(result);
|
|
9359
|
+
else {
|
|
9360
|
+
console.log(
|
|
9361
|
+
`${result.result}: ${result.counts.passedRuns}/${result.counts.createdRuns} passed sessions (${result.counts.judgedRuns} judged)`
|
|
9362
|
+
);
|
|
9363
|
+
console.log(`Request: ${result.id}`);
|
|
9364
|
+
}
|
|
9365
|
+
if (result.result !== "PASSED")
|
|
9366
|
+
throw new CliError(
|
|
9367
|
+
"internal",
|
|
9368
|
+
`Run request ${id}: ${result.result}`
|
|
9369
|
+
);
|
|
9370
|
+
return;
|
|
9371
|
+
}
|
|
9372
|
+
await new Promise((resolve2) => setTimeout(resolve2, pollIntervalMs));
|
|
9373
|
+
}
|
|
9374
|
+
});
|
|
9375
|
+
}
|
|
9376
|
+
|
|
8197
9377
|
// src/commands/runs.ts
|
|
8198
9378
|
var runs_exports2 = {};
|
|
8199
9379
|
__export(runs_exports2, {
|
|
8200
|
-
register: () =>
|
|
9380
|
+
register: () => register26
|
|
8201
9381
|
});
|
|
8202
9382
|
function runPath(id) {
|
|
8203
9383
|
return `/api/v1/runs/${encodeURIComponent(id)}`;
|
|
@@ -8457,7 +9637,7 @@ async function streamRun(id, json, opts) {
|
|
|
8457
9637
|
if (drainTimer) clearTimeout(drainTimer);
|
|
8458
9638
|
}
|
|
8459
9639
|
}
|
|
8460
|
-
function
|
|
9640
|
+
function register26(program2) {
|
|
8461
9641
|
const runs = program2.command("runs").description("Inspect and manage runs");
|
|
8462
9642
|
runs.command("list").description("List runs (newest first)").option("--status <status>", "filter by status, e.g. running,queued").option("--batch <id>", "filter by batch").option("--prompt <id>", "filter by prompt").option("--experiment <experiment>", "filter by experiment label").option("--eval-set <id>", "filter by eval set").option("--since <window>", "e.g. 24h, 7d, or an ISO date").option("--limit <n>", "page size (1-200)", "50").option("--cursor <cursor>", "resume from a previous page's nextCursor").action(
|
|
8463
9643
|
async (opts, command) => {
|
|
@@ -8510,6 +9690,21 @@ function register25(program2) {
|
|
|
8510
9690
|
}
|
|
8511
9691
|
printInsight(insight);
|
|
8512
9692
|
});
|
|
9693
|
+
runs.command("regrade <id>").description(
|
|
9694
|
+
"Re-judge the run against its eval set's current criteria (no new session)"
|
|
9695
|
+
).action(async (id, _opts, command) => {
|
|
9696
|
+
const result = await createClient().post(
|
|
9697
|
+
`${runPath(id)}/regrade`
|
|
9698
|
+
);
|
|
9699
|
+
if (resolveOutput(command) === "json") {
|
|
9700
|
+
printJson(result);
|
|
9701
|
+
return;
|
|
9702
|
+
}
|
|
9703
|
+
console.log(
|
|
9704
|
+
`Re-judging run ${result.runId} against ${result.criteria} ${result.criteria === 1 ? "criterion" : "criteria"} of eval ${result.evalSetId}.`
|
|
9705
|
+
);
|
|
9706
|
+
console.log("Verdicts land on the run; `gauge runs get <id>` to check.");
|
|
9707
|
+
});
|
|
8513
9708
|
runs.command("watch <id>").description(
|
|
8514
9709
|
"Stream normalized trace events live (SSE; exits when the run finishes)"
|
|
8515
9710
|
).action(async (id, _opts, command) => {
|
|
@@ -8581,7 +9776,7 @@ function register25(program2) {
|
|
|
8581
9776
|
var skills_exports2 = {};
|
|
8582
9777
|
__export(skills_exports2, {
|
|
8583
9778
|
buildSkillEditBody: () => buildSkillEditBody,
|
|
8584
|
-
register: () =>
|
|
9779
|
+
register: () => register27
|
|
8585
9780
|
});
|
|
8586
9781
|
function skillsPath(org) {
|
|
8587
9782
|
return `/api/v1/orgs/${encodeURIComponent(org)}/skills`;
|
|
@@ -8643,7 +9838,7 @@ async function buildSkillEditBody(opts, fetchCurrentSource) {
|
|
|
8643
9838
|
throw new CliError("usage", "nothing to change");
|
|
8644
9839
|
return body;
|
|
8645
9840
|
}
|
|
8646
|
-
function
|
|
9841
|
+
function register27(program2) {
|
|
8647
9842
|
const group = program2.command("skills").description("Manage agent skills used by eval and visibility runs");
|
|
8648
9843
|
group.command("list").description("List the org's skills with their newest version").action(async (_opts, cmd) => {
|
|
8649
9844
|
const org = resolveOrg(cmd);
|
|
@@ -8682,7 +9877,7 @@ function register26(program2) {
|
|
|
8682
9877
|
` ${skill2.name}@${v.label} ${v.contentDigest.slice(0, 19)} ${v.createdAt}`
|
|
8683
9878
|
);
|
|
8684
9879
|
});
|
|
8685
|
-
group.command("add <reference>").description("Ingest a skill from owner/repo[/skill][#ref] or a GitHub URL").option("--as <handle>", "
|
|
9880
|
+
group.command("add <reference>").description("Ingest a skill from owner/repo[/skill][#ref] or a GitHub URL").option("--as <handle>", "the handle (default: the directory basename)").option(
|
|
8686
9881
|
"--label <label>",
|
|
8687
9882
|
"version label (a typed label always mints; omitted dedups by digest)"
|
|
8688
9883
|
).option("--skill <name>", "select one skill from a source holding several").option("--note <text>", "note shown on the index").option(
|
|
@@ -8747,7 +9942,7 @@ function register26(program2) {
|
|
|
8747
9942
|
);
|
|
8748
9943
|
});
|
|
8749
9944
|
group.command("rm <ref>").description(
|
|
8750
|
-
"
|
|
9945
|
+
"Remove a skill; the handle stays reserved and finished runs keep their snapshot"
|
|
8751
9946
|
).action(async (ref, _opts, cmd) => {
|
|
8752
9947
|
const org = resolveOrg(cmd);
|
|
8753
9948
|
const result = await createClient().delete(skillPath(org, ref));
|
|
@@ -8784,7 +9979,7 @@ function register26(program2) {
|
|
|
8784
9979
|
// src/commands/stats.ts
|
|
8785
9980
|
var stats_exports2 = {};
|
|
8786
9981
|
__export(stats_exports2, {
|
|
8787
|
-
register: () =>
|
|
9982
|
+
register: () => register28
|
|
8788
9983
|
});
|
|
8789
9984
|
function addFilterOptions(cmd) {
|
|
8790
9985
|
return cmd.option("--prompt <id>", "filter to a single prompt").option("--tag <ids>", "comma-separated tag ids").option("--repo <urls>", "comma-separated base repo urls").option("--language <langs>", "comma-separated repo languages").option("--framework <names>", "comma-separated repo frameworks").option("--size <buckets>", "comma-separated codebase-size buckets").option("--branded", "only branded prompts").option("--unbranded", "only non-branded prompts").option("--limit <n>", "max rows (default 50)").option("--full", "return all rows (up to 1000)");
|
|
@@ -8867,7 +10062,7 @@ var domainRow = (r) => ({
|
|
|
8867
10062
|
rate: pct3(r.rate),
|
|
8868
10063
|
requests: r.requests
|
|
8869
10064
|
});
|
|
8870
|
-
function
|
|
10065
|
+
function register28(program2) {
|
|
8871
10066
|
const group = program2.command("stats").description("Read-only analytics over your runs");
|
|
8872
10067
|
addFilterOptions(
|
|
8873
10068
|
group.command("rankings").description("Brands ranked by install rate")
|
|
@@ -8899,7 +10094,7 @@ function register27(program2) {
|
|
|
8899
10094
|
// src/commands/status.ts
|
|
8900
10095
|
var status_exports = {};
|
|
8901
10096
|
__export(status_exports, {
|
|
8902
|
-
register: () =>
|
|
10097
|
+
register: () => register29
|
|
8903
10098
|
});
|
|
8904
10099
|
function pct4(rate) {
|
|
8905
10100
|
return rate == null ? "\u2014" : `${Math.round(rate * 100)}%`;
|
|
@@ -8909,7 +10104,7 @@ function delta2(pp) {
|
|
|
8909
10104
|
const rounded = Math.round(pp);
|
|
8910
10105
|
return rounded === 0 ? "" : ` (${rounded > 0 ? "+" : ""}${rounded}pp)`;
|
|
8911
10106
|
}
|
|
8912
|
-
function
|
|
10107
|
+
function register29(program2) {
|
|
8913
10108
|
program2.command("status").description("One-shot org summary: actions, visibility, agent experience").action(async (_opts, cmd) => {
|
|
8914
10109
|
const org = resolveOrg(cmd);
|
|
8915
10110
|
const s = await createClient().get(
|
|
@@ -8971,12 +10166,12 @@ function register28(program2) {
|
|
|
8971
10166
|
// src/commands/tags.ts
|
|
8972
10167
|
var tags_exports2 = {};
|
|
8973
10168
|
__export(tags_exports2, {
|
|
8974
|
-
register: () =>
|
|
10169
|
+
register: () => register30
|
|
8975
10170
|
});
|
|
8976
10171
|
function tagsPath(org) {
|
|
8977
10172
|
return `/api/v1/orgs/${encodeURIComponent(org)}/tags`;
|
|
8978
10173
|
}
|
|
8979
|
-
function
|
|
10174
|
+
function register30(program2) {
|
|
8980
10175
|
const group = program2.command("tags").description("Manage org tags");
|
|
8981
10176
|
group.command("list").description("List tags").action(async (_opts, cmd) => {
|
|
8982
10177
|
const org = resolveOrg(cmd);
|
|
@@ -9019,9 +10214,9 @@ function register29(program2) {
|
|
|
9019
10214
|
// src/commands/usage.ts
|
|
9020
10215
|
var usage_exports = {};
|
|
9021
10216
|
__export(usage_exports, {
|
|
9022
|
-
register: () =>
|
|
10217
|
+
register: () => register31
|
|
9023
10218
|
});
|
|
9024
|
-
function
|
|
10219
|
+
function register31(program2) {
|
|
9025
10220
|
program2.command("usage").description("List credit ledger entries").option("--since <window>", 'e.g. "30d", "24h", or an ISO date').option("--limit <n>", "page size (1-200)", parseLimit, 50).option("--cursor <cursor>", "resume from a previous page's nextCursor").action(
|
|
9026
10221
|
async (opts, cmd) => {
|
|
9027
10222
|
const org = resolveOrg(cmd);
|
|
@@ -9068,9 +10263,10 @@ function register30(program2) {
|
|
|
9068
10263
|
// src/commands/visibility.ts
|
|
9069
10264
|
var visibility_exports2 = {};
|
|
9070
10265
|
__export(visibility_exports2, {
|
|
9071
|
-
register: () =>
|
|
10266
|
+
register: () => register32,
|
|
9072
10267
|
visibilityPath: () => visibilityPath
|
|
9073
10268
|
});
|
|
10269
|
+
import { Option as Option3 } from "commander";
|
|
9074
10270
|
function visibilityPath(org) {
|
|
9075
10271
|
return `/api/v1/orgs/${encodeURIComponent(org)}/visibility-prompts`;
|
|
9076
10272
|
}
|
|
@@ -9298,6 +10494,11 @@ function registerPreferenceGroup(group) {
|
|
|
9298
10494
|
"run only this scenario (repeatable; default: all)",
|
|
9299
10495
|
collect,
|
|
9300
10496
|
[]
|
|
10497
|
+
).addOption(
|
|
10498
|
+
new Option3(
|
|
10499
|
+
"--bill-to-org",
|
|
10500
|
+
"bill this run to the org's credits instead (superusers)"
|
|
10501
|
+
).hideHelp(!staffCommandsVisible())
|
|
9301
10502
|
).option("--yes", "skip the confirmation").action(
|
|
9302
10503
|
async (id, opts, cmd) => {
|
|
9303
10504
|
const org = resolveOrg(cmd);
|
|
@@ -9308,6 +10509,7 @@ function registerPreferenceGroup(group) {
|
|
|
9308
10509
|
const result = await createClient().post(
|
|
9309
10510
|
`${visibilityPath(org)}/${encodeURIComponent(id)}/run`,
|
|
9310
10511
|
{
|
|
10512
|
+
...opts.billToOrg ? { billToOrg: true } : {},
|
|
9311
10513
|
visibilityScenarioIds: opts.scenario.length > 0 ? opts.scenario : void 0
|
|
9312
10514
|
}
|
|
9313
10515
|
);
|
|
@@ -9417,18 +10619,13 @@ function registerPreferenceGroup(group) {
|
|
|
9417
10619
|
);
|
|
9418
10620
|
});
|
|
9419
10621
|
}
|
|
9420
|
-
function
|
|
10622
|
+
function register32(program2) {
|
|
9421
10623
|
const preference = program2.command("preference").description("Manage Agent Preference prompts and measured markets");
|
|
9422
10624
|
registerPreferenceGroup(preference);
|
|
9423
10625
|
const legacy = program2.command("visibility", { hidden: true });
|
|
9424
10626
|
registerPreferenceGroup(legacy);
|
|
9425
10627
|
}
|
|
9426
10628
|
|
|
9427
|
-
// src/lib/staff.ts
|
|
9428
|
-
function staffCommandsVisible(config = readConfig()) {
|
|
9429
|
-
return config.staff === true;
|
|
9430
|
-
}
|
|
9431
|
-
|
|
9432
10629
|
// src/index.ts
|
|
9433
10630
|
var program = new Command();
|
|
9434
10631
|
program.name("gauge").description(
|
|
@@ -9437,7 +10634,7 @@ program.name("gauge").description(
|
|
|
9437
10634
|
"--org <org>",
|
|
9438
10635
|
"organization slug or public id (default: `gauge config get defaultOrg`)"
|
|
9439
10636
|
).addOption(
|
|
9440
|
-
new
|
|
10637
|
+
new Option4("-o, --output <format>", "output format").choices([...OUTPUT_FORMATS]).default("table")
|
|
9441
10638
|
);
|
|
9442
10639
|
for (const group of [
|
|
9443
10640
|
instructions_exports,
|
|
@@ -9456,6 +10653,7 @@ for (const group of [
|
|
|
9456
10653
|
connections_exports2,
|
|
9457
10654
|
repos_exports2,
|
|
9458
10655
|
runs_exports2,
|
|
10656
|
+
runRequests_exports2,
|
|
9459
10657
|
optimizations_exports2,
|
|
9460
10658
|
experiments_exports2,
|
|
9461
10659
|
apply_exports,
|