@runloop/rl-cli 1.8.0 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +19 -5
  2. package/dist/cli.js +0 -0
  3. package/dist/commands/blueprint/delete.js +21 -0
  4. package/dist/commands/blueprint/list.js +226 -174
  5. package/dist/commands/blueprint/prune.js +13 -28
  6. package/dist/commands/devbox/create.js +41 -0
  7. package/dist/commands/devbox/list.js +125 -109
  8. package/dist/commands/devbox/tunnel.js +4 -19
  9. package/dist/commands/gateway-config/create.js +44 -0
  10. package/dist/commands/gateway-config/delete.js +21 -0
  11. package/dist/commands/gateway-config/get.js +15 -0
  12. package/dist/commands/gateway-config/list.js +493 -0
  13. package/dist/commands/gateway-config/update.js +60 -0
  14. package/dist/commands/snapshot/list.js +11 -2
  15. package/dist/commands/snapshot/prune.js +265 -0
  16. package/dist/components/BenchmarkMenu.js +23 -3
  17. package/dist/components/DetailedInfoView.js +20 -0
  18. package/dist/components/DevboxActionsMenu.js +9 -61
  19. package/dist/components/DevboxCreatePage.js +531 -14
  20. package/dist/components/DevboxDetailPage.js +27 -22
  21. package/dist/components/GatewayConfigCreatePage.js +265 -0
  22. package/dist/components/LogsViewer.js +6 -40
  23. package/dist/components/ResourceDetailPage.js +143 -160
  24. package/dist/components/ResourceListView.js +3 -33
  25. package/dist/components/ResourcePicker.js +220 -0
  26. package/dist/components/SecretCreatePage.js +2 -4
  27. package/dist/components/SettingsMenu.js +12 -2
  28. package/dist/components/StateHistory.js +1 -20
  29. package/dist/components/StatusBadge.js +9 -2
  30. package/dist/components/StreamingLogsViewer.js +8 -42
  31. package/dist/components/form/FormTextInput.js +4 -2
  32. package/dist/components/resourceDetailTypes.js +18 -0
  33. package/dist/hooks/useInputHandler.js +103 -0
  34. package/dist/router/Router.js +79 -2
  35. package/dist/screens/BenchmarkDetailScreen.js +163 -0
  36. package/dist/screens/BenchmarkJobCreateScreen.js +524 -0
  37. package/dist/screens/BenchmarkJobDetailScreen.js +614 -0
  38. package/dist/screens/BenchmarkJobListScreen.js +479 -0
  39. package/dist/screens/BenchmarkListScreen.js +266 -0
  40. package/dist/screens/BenchmarkMenuScreen.js +6 -0
  41. package/dist/screens/BenchmarkRunDetailScreen.js +258 -22
  42. package/dist/screens/BenchmarkRunListScreen.js +21 -1
  43. package/dist/screens/BlueprintDetailScreen.js +5 -1
  44. package/dist/screens/DevboxCreateScreen.js +2 -2
  45. package/dist/screens/GatewayConfigDetailScreen.js +236 -0
  46. package/dist/screens/GatewayConfigListScreen.js +7 -0
  47. package/dist/screens/ScenarioRunDetailScreen.js +6 -0
  48. package/dist/screens/SettingsMenuScreen.js +3 -0
  49. package/dist/screens/SnapshotDetailScreen.js +6 -0
  50. package/dist/services/agentService.js +42 -0
  51. package/dist/services/benchmarkJobService.js +122 -0
  52. package/dist/services/benchmarkService.js +47 -0
  53. package/dist/services/gatewayConfigService.js +114 -0
  54. package/dist/services/scenarioService.js +34 -0
  55. package/dist/store/benchmarkJobStore.js +66 -0
  56. package/dist/store/benchmarkStore.js +63 -0
  57. package/dist/store/gatewayConfigStore.js +83 -0
  58. package/dist/utils/browser.js +22 -0
  59. package/dist/utils/clipboard.js +41 -0
  60. package/dist/utils/commands.js +80 -0
  61. package/dist/utils/time.js +121 -0
  62. package/package.json +42 -43
@@ -82,6 +82,12 @@ export function BenchmarkRunListScreen() {
82
82
  color: colors.info,
83
83
  icon: figures.arrowRight,
84
84
  },
85
+ {
86
+ key: "create_job",
87
+ label: "Create Job",
88
+ color: colors.success,
89
+ icon: figures.play,
90
+ },
85
91
  ], []);
86
92
  // Build columns
87
93
  const columns = React.useMemo(() => [
@@ -150,6 +156,9 @@ export function BenchmarkRunListScreen() {
150
156
  benchmarkRunId: selectedRun.id,
151
157
  });
152
158
  }
159
+ else if (operationKey === "create_job") {
160
+ navigate("benchmark-job-create");
161
+ }
153
162
  }
154
163
  else if (input === "v" && selectedRun) {
155
164
  setShowPopup(false);
@@ -163,6 +172,10 @@ export function BenchmarkRunListScreen() {
163
172
  benchmarkRunId: selectedRun.id,
164
173
  });
165
174
  }
175
+ else if (input === "j") {
176
+ setShowPopup(false);
177
+ navigate("benchmark-job-create");
178
+ }
166
179
  else if (key.escape || input === "q") {
167
180
  setShowPopup(false);
168
181
  setSelectedOperation(0);
@@ -200,6 +213,10 @@ export function BenchmarkRunListScreen() {
200
213
  setShowPopup(true);
201
214
  setSelectedOperation(0);
202
215
  }
216
+ else if (input === "j") {
217
+ // Quick shortcut to create a new job
218
+ navigate("benchmark-job-create");
219
+ }
203
220
  else if (input === "/") {
204
221
  search.enterSearchMode();
205
222
  }
@@ -240,7 +257,9 @@ export function BenchmarkRunListScreen() {
240
257
  ? "v"
241
258
  : op.key === "view_scenarios"
242
259
  ? "s"
243
- : "",
260
+ : op.key === "create_job"
261
+ ? "j"
262
+ : "",
244
263
  })), selectedOperation: selectedOperation, onClose: () => setShowPopup(false) }) })), _jsx(NavigationTips, { showArrows: true, tips: [
245
264
  {
246
265
  icon: `${figures.arrowLeft}${figures.arrowRight}`,
@@ -248,6 +267,7 @@ export function BenchmarkRunListScreen() {
248
267
  condition: hasMore || hasPrev,
249
268
  },
250
269
  { key: "Enter", label: "Details" },
270
+ { key: "j", label: "New Job" },
251
271
  { key: "a", label: "Actions" },
252
272
  { key: "/", label: "Search" },
253
273
  { key: "Esc", label: "Back" },
@@ -351,5 +351,9 @@ export function BlueprintDetailScreen({ blueprintId, }) {
351
351
  });
352
352
  return lines;
353
353
  };
354
- return (_jsx(ResourceDetailPage, { resource: blueprint, resourceType: "Blueprints", getDisplayName: (bp) => bp.name || bp.id, getId: (bp) => bp.id, getStatus: (bp) => bp.status, detailSections: detailSections, operations: operations, onOperation: handleOperation, onBack: goBack, buildDetailLines: buildDetailLines, pollResource: blueprint.status === "building" ? pollBlueprint : undefined }));
354
+ return (_jsx(ResourceDetailPage, { resource: blueprint, resourceType: "Blueprints", getDisplayName: (bp) => bp.name || bp.id, getId: (bp) => bp.id, getStatus: (bp) => bp.status, detailSections: detailSections, operations: operations, onOperation: handleOperation, onBack: goBack, buildDetailLines: buildDetailLines, pollResource: blueprint.status === "queued" ||
355
+ blueprint.status === "provisioning" ||
356
+ blueprint.status === "building"
357
+ ? pollBlueprint
358
+ : undefined }));
355
359
  }
@@ -2,10 +2,10 @@ import { jsx as _jsx } from "react/jsx-runtime";
2
2
  import { useNavigation } from "../store/navigationStore.js";
3
3
  import { DevboxCreatePage } from "../components/DevboxCreatePage.js";
4
4
  export function DevboxCreateScreen() {
5
- const { goBack, navigate } = useNavigation();
5
+ const { goBack, navigate, params } = useNavigation();
6
6
  const handleCreate = (devbox) => {
7
7
  // After creation, navigate to the devbox detail page
8
8
  navigate("devbox-detail", { devboxId: devbox.id });
9
9
  };
10
- return _jsx(DevboxCreatePage, { onBack: goBack, onCreate: handleCreate });
10
+ return (_jsx(DevboxCreatePage, { onBack: goBack, onCreate: handleCreate, initialBlueprintId: params.blueprintId, initialSnapshotId: params.snapshotId }));
11
11
  }
@@ -0,0 +1,236 @@
1
+ import { jsx as _jsx, Fragment as _Fragment, jsxs as _jsxs } from "react/jsx-runtime";
2
+ /**
3
+ * GatewayConfigDetailScreen - Detail page for gateway configs
4
+ * Uses the generic ResourceDetailPage component
5
+ */
6
+ import React from "react";
7
+ import { Text } from "ink";
8
+ import figures from "figures";
9
+ import { useNavigation } from "../store/navigationStore.js";
10
+ import { useGatewayConfigStore, } from "../store/gatewayConfigStore.js";
11
+ import { ResourceDetailPage, formatTimestamp, } from "../components/ResourceDetailPage.js";
12
+ import { getGatewayConfig, deleteGatewayConfig, } from "../services/gatewayConfigService.js";
13
+ import { SpinnerComponent } from "../components/Spinner.js";
14
+ import { ErrorMessage } from "../components/ErrorMessage.js";
15
+ import { Breadcrumb } from "../components/Breadcrumb.js";
16
+ import { ConfirmationPrompt } from "../components/ConfirmationPrompt.js";
17
+ import { GatewayConfigCreatePage } from "../components/GatewayConfigCreatePage.js";
18
+ import { colors } from "../utils/theme.js";
19
+ /**
20
+ * Get a display label for the auth mechanism type
21
+ */
22
+ function getAuthTypeLabel(authMechanism) {
23
+ if (authMechanism.type === "bearer") {
24
+ return "Bearer Token";
25
+ }
26
+ if (authMechanism.type === "header") {
27
+ return authMechanism.key ? `Header: ${authMechanism.key}` : "Header";
28
+ }
29
+ return authMechanism.type;
30
+ }
31
+ export function GatewayConfigDetailScreen({ gatewayConfigId, }) {
32
+ const { goBack } = useNavigation();
33
+ const gatewayConfigs = useGatewayConfigStore((state) => state.gatewayConfigs);
34
+ const [loading, setLoading] = React.useState(false);
35
+ const [error, setError] = React.useState(null);
36
+ const [fetchedConfig, setFetchedConfig] = React.useState(null);
37
+ const [deleting, setDeleting] = React.useState(false);
38
+ const [showDeleteConfirm, setShowDeleteConfirm] = React.useState(false);
39
+ const [showEditForm, setShowEditForm] = React.useState(false);
40
+ // Find config in store first
41
+ const configFromStore = gatewayConfigs.find((c) => c.id === gatewayConfigId);
42
+ // Fetch config from API if not in store or missing full details
43
+ React.useEffect(() => {
44
+ if (gatewayConfigId && !loading && !fetchedConfig) {
45
+ // Always fetch full details since store may only have basic info
46
+ setLoading(true);
47
+ setError(null);
48
+ getGatewayConfig(gatewayConfigId)
49
+ .then((config) => {
50
+ setFetchedConfig(config);
51
+ setLoading(false);
52
+ })
53
+ .catch((err) => {
54
+ setError(err);
55
+ setLoading(false);
56
+ });
57
+ }
58
+ }, [gatewayConfigId, loading, fetchedConfig]);
59
+ // Use fetched config for full details, fall back to store for basic display
60
+ const config = fetchedConfig || configFromStore;
61
+ // Show loading state while fetching or before fetch starts
62
+ if (!config && gatewayConfigId && !error) {
63
+ return (_jsxs(_Fragment, { children: [_jsx(Breadcrumb, { items: [
64
+ { label: "Gateway Configs" },
65
+ { label: "Loading...", active: true },
66
+ ] }), _jsx(SpinnerComponent, { message: "Loading gateway config details..." })] }));
67
+ }
68
+ // Show error state if fetch failed
69
+ if (error && !config) {
70
+ return (_jsxs(_Fragment, { children: [_jsx(Breadcrumb, { items: [
71
+ { label: "Gateway Configs" },
72
+ { label: "Error", active: true },
73
+ ] }), _jsx(ErrorMessage, { message: "Failed to load gateway config details", error: error })] }));
74
+ }
75
+ // Show error if no config found
76
+ if (!config) {
77
+ return (_jsxs(_Fragment, { children: [_jsx(Breadcrumb, { items: [
78
+ { label: "Gateway Configs" },
79
+ { label: "Not Found", active: true },
80
+ ] }), _jsx(ErrorMessage, { message: `Gateway config ${gatewayConfigId || "unknown"} not found`, error: new Error("Gateway config not found") })] }));
81
+ }
82
+ // Build detail sections
83
+ const detailSections = [];
84
+ // Basic details section
85
+ const basicFields = [];
86
+ if (config.description) {
87
+ basicFields.push({
88
+ label: "Description",
89
+ value: config.description,
90
+ });
91
+ }
92
+ basicFields.push({
93
+ label: "Endpoint",
94
+ value: config.endpoint,
95
+ });
96
+ if (config.create_time_ms) {
97
+ basicFields.push({
98
+ label: "Created",
99
+ value: formatTimestamp(config.create_time_ms),
100
+ });
101
+ }
102
+ if (config.account_id) {
103
+ basicFields.push({
104
+ label: "Account ID",
105
+ value: config.account_id,
106
+ });
107
+ }
108
+ if (basicFields.length > 0) {
109
+ detailSections.push({
110
+ title: "Details",
111
+ icon: figures.squareSmallFilled,
112
+ color: colors.warning,
113
+ fields: basicFields,
114
+ });
115
+ }
116
+ // Auth mechanism section
117
+ const authFields = [];
118
+ authFields.push({
119
+ label: "Auth Type",
120
+ value: (_jsx(Text, { color: colors.info, bold: true, children: getAuthTypeLabel(config.auth_mechanism) })),
121
+ });
122
+ if (config.auth_mechanism.type === "header" && config.auth_mechanism.key) {
123
+ authFields.push({
124
+ label: "Header Key",
125
+ value: config.auth_mechanism.key,
126
+ });
127
+ }
128
+ detailSections.push({
129
+ title: "Authentication",
130
+ icon: figures.arrowRight,
131
+ color: colors.info,
132
+ fields: authFields,
133
+ });
134
+ // Operations available for gateway configs
135
+ const operations = [
136
+ {
137
+ key: "edit",
138
+ label: "Edit Gateway Config",
139
+ color: colors.warning,
140
+ icon: figures.pointer,
141
+ shortcut: "e",
142
+ },
143
+ {
144
+ key: "delete",
145
+ label: "Delete Gateway Config",
146
+ color: colors.error,
147
+ icon: figures.cross,
148
+ shortcut: "d",
149
+ },
150
+ ];
151
+ // Handle operation selection
152
+ const handleOperation = async (operation, _resource) => {
153
+ switch (operation) {
154
+ case "edit":
155
+ setShowEditForm(true);
156
+ break;
157
+ case "delete":
158
+ // Show confirmation dialog
159
+ setShowDeleteConfirm(true);
160
+ break;
161
+ }
162
+ };
163
+ // Execute delete after confirmation
164
+ const executeDelete = async () => {
165
+ if (!config)
166
+ return;
167
+ setShowDeleteConfirm(false);
168
+ setDeleting(true);
169
+ try {
170
+ await deleteGatewayConfig(config.id);
171
+ goBack();
172
+ }
173
+ catch (err) {
174
+ setError(err);
175
+ setDeleting(false);
176
+ }
177
+ };
178
+ // Build detailed info lines for full details view
179
+ const buildDetailLines = (gc) => {
180
+ const lines = [];
181
+ // Core Information
182
+ lines.push(_jsx(Text, { color: colors.warning, bold: true, children: "Gateway Config Details" }, "core-title"));
183
+ lines.push(_jsxs(Text, { color: colors.idColor, children: [" ", "ID: ", gc.id] }, "core-id"));
184
+ lines.push(_jsxs(Text, { dimColor: true, children: [" ", "Name: ", gc.name] }, "core-name"));
185
+ if (gc.description) {
186
+ lines.push(_jsxs(Text, { dimColor: true, children: [" ", "Description: ", gc.description] }, "core-desc"));
187
+ }
188
+ lines.push(_jsxs(Text, { dimColor: true, children: [" ", "Endpoint: ", gc.endpoint] }, "core-endpoint"));
189
+ if (gc.create_time_ms) {
190
+ lines.push(_jsxs(Text, { dimColor: true, children: [" ", "Created: ", new Date(gc.create_time_ms).toLocaleString()] }, "core-created"));
191
+ }
192
+ if (gc.account_id) {
193
+ lines.push(_jsxs(Text, { dimColor: true, children: [" ", "Account ID: ", gc.account_id] }, "core-account"));
194
+ }
195
+ lines.push(_jsx(Text, { children: " " }, "core-space"));
196
+ // Auth Mechanism
197
+ lines.push(_jsx(Text, { color: colors.warning, bold: true, children: "Authentication" }, "auth-title"));
198
+ lines.push(_jsxs(Text, { dimColor: true, children: [" ", "Type: ", getAuthTypeLabel(gc.auth_mechanism)] }, "auth-type"));
199
+ if (gc.auth_mechanism.type === "header" && gc.auth_mechanism.key) {
200
+ lines.push(_jsxs(Text, { dimColor: true, children: [" ", "Header Key: ", gc.auth_mechanism.key] }, "auth-key"));
201
+ }
202
+ lines.push(_jsx(Text, { children: " " }, "auth-space"));
203
+ // Raw JSON
204
+ lines.push(_jsx(Text, { color: colors.warning, bold: true, children: "Raw JSON" }, "json-title"));
205
+ const jsonLines = JSON.stringify(gc, null, 2).split("\n");
206
+ jsonLines.forEach((line, idx) => {
207
+ lines.push(_jsxs(Text, { dimColor: true, children: [" ", line] }, `json-${idx}`));
208
+ });
209
+ return lines;
210
+ };
211
+ // Show edit form
212
+ if (showEditForm && config) {
213
+ return (_jsx(GatewayConfigCreatePage, { onBack: () => setShowEditForm(false), onCreate: (updatedConfig) => {
214
+ // Update the fetched config with the new data
215
+ setFetchedConfig(updatedConfig);
216
+ setShowEditForm(false);
217
+ }, initialConfig: config }));
218
+ }
219
+ // Show delete confirmation
220
+ if (showDeleteConfirm && config) {
221
+ return (_jsx(ConfirmationPrompt, { title: "Delete Gateway Config", message: `Are you sure you want to delete "${config.name || config.id}"?`, details: "This action cannot be undone. Any devboxes using this gateway config will no longer have access to it.", breadcrumbItems: [
222
+ { label: "Gateway Configs" },
223
+ { label: config.name || config.id },
224
+ { label: "Delete", active: true },
225
+ ], onConfirm: executeDelete, onCancel: () => setShowDeleteConfirm(false) }));
226
+ }
227
+ // Show deleting state
228
+ if (deleting) {
229
+ return (_jsxs(_Fragment, { children: [_jsx(Breadcrumb, { items: [
230
+ { label: "Gateway Configs" },
231
+ { label: config.name || config.id },
232
+ { label: "Deleting...", active: true },
233
+ ] }), _jsx(SpinnerComponent, { message: "Deleting gateway config..." })] }));
234
+ }
235
+ return (_jsx(ResourceDetailPage, { resource: config, resourceType: "Gateway Configs", getDisplayName: (gc) => gc.name || gc.id, getId: (gc) => gc.id, getStatus: () => "active", detailSections: detailSections, operations: operations, onOperation: handleOperation, onBack: goBack, buildDetailLines: buildDetailLines }));
236
+ }
@@ -0,0 +1,7 @@
1
+ import { jsx as _jsx } from "react/jsx-runtime";
2
+ import { useNavigation } from "../store/navigationStore.js";
3
+ import { ListGatewayConfigsUI } from "../commands/gateway-config/list.js";
4
+ export function GatewayConfigListScreen() {
5
+ const { goBack } = useNavigation();
6
+ return _jsx(ListGatewayConfigsUI, { onBack: goBack });
7
+ }
@@ -100,6 +100,12 @@ export function ScenarioRunDetailScreen({ scenarioRunId, benchmarkRunId, }) {
100
100
  basicFields.push({
101
101
  label: "Benchmark Run ID",
102
102
  value: _jsx(Text, { color: colors.idColor, children: run.benchmark_run_id }),
103
+ action: {
104
+ type: "navigate",
105
+ screen: "benchmark-run-detail",
106
+ params: { benchmarkRunId: run.benchmark_run_id },
107
+ hint: "View Run",
108
+ },
103
109
  });
104
110
  }
105
111
  if (basicFields.length > 0) {
@@ -8,6 +8,9 @@ export function SettingsMenuScreen() {
8
8
  case "network-policies":
9
9
  navigate("network-policy-list");
10
10
  break;
11
+ case "gateway-configs":
12
+ navigate("gateway-config-list");
13
+ break;
11
14
  case "secrets":
12
15
  navigate("secret-list");
13
16
  break;
@@ -79,6 +79,12 @@ export function SnapshotDetailScreen({ snapshotId, }) {
79
79
  basicFields.push({
80
80
  label: "Source Devbox",
81
81
  value: _jsx(Text, { color: colors.idColor, children: snapshot.devbox_id }),
82
+ action: {
83
+ type: "navigate",
84
+ screen: "devbox-detail",
85
+ params: { devboxId: snapshot.devbox_id },
86
+ hint: "View Devbox",
87
+ },
82
88
  });
83
89
  }
84
90
  if (snapshot.disk_size_bytes) {
@@ -0,0 +1,42 @@
1
+ /**
2
+ * Agent Service - Handles API calls for agents
3
+ */
4
+ import { getClient } from "../utils/client.js";
5
+ /**
6
+ * List agents with pagination
7
+ * Can filter to only return public agents for benchmark jobs
8
+ */
9
+ export async function listAgents(options) {
10
+ const client = getClient();
11
+ const queryParams = {
12
+ limit: options.limit || 50,
13
+ };
14
+ if (options.startingAfter) {
15
+ queryParams.starting_after = options.startingAfter;
16
+ }
17
+ // Use API filter for public agents
18
+ if (options.publicOnly) {
19
+ queryParams.is_public = true;
20
+ }
21
+ const page = await client.agents.list(queryParams);
22
+ const agents = [];
23
+ // Collect agents from the cursor page
24
+ for await (const agent of page) {
25
+ agents.push(agent);
26
+ if (options.limit && agents.length >= options.limit) {
27
+ break;
28
+ }
29
+ }
30
+ return {
31
+ agents,
32
+ totalCount: agents.length,
33
+ hasMore: false, // Cursor pagination doesn't give us this directly
34
+ };
35
+ }
36
+ /**
37
+ * Get agent by ID
38
+ */
39
+ export async function getAgent(id) {
40
+ const client = getClient();
41
+ return client.agents.retrieve(id);
42
+ }
@@ -0,0 +1,122 @@
1
+ /**
2
+ * Benchmark Job Service - Handles API calls for benchmark jobs
3
+ */
4
+ import { getClient } from "../utils/client.js";
5
+ /**
6
+ * List benchmark jobs with pagination
7
+ */
8
+ export async function listBenchmarkJobs(options) {
9
+ const client = getClient();
10
+ const queryParams = {
11
+ limit: options.limit,
12
+ };
13
+ if (options.startingAfter) {
14
+ queryParams.starting_after = options.startingAfter;
15
+ }
16
+ if (options.name) {
17
+ queryParams.name = options.name;
18
+ }
19
+ const page = await client.benchmarkJobs.list(queryParams);
20
+ const jobs = page.jobs || [];
21
+ return {
22
+ jobs,
23
+ totalCount: page.total_count || jobs.length,
24
+ hasMore: page.has_more || false,
25
+ };
26
+ }
27
+ /**
28
+ * Get benchmark job by ID
29
+ */
30
+ export async function getBenchmarkJob(id) {
31
+ const client = getClient();
32
+ return client.benchmarkJobs.retrieve(id);
33
+ }
34
+ /**
35
+ * Create a benchmark job with either benchmark definition spec or scenario definition spec
36
+ */
37
+ export async function createBenchmarkJob(options) {
38
+ const client = getClient();
39
+ // Validate that either benchmarkId or scenarioIds is provided
40
+ if (!options.benchmarkId && !options.scenarioIds) {
41
+ throw new Error("Either benchmarkId or scenarioIds must be provided");
42
+ }
43
+ if (options.benchmarkId && options.scenarioIds) {
44
+ throw new Error("Cannot specify both benchmarkId and scenarioIds");
45
+ }
46
+ // Build agent configs in API format
47
+ // Use the same agent config type for both spec types
48
+ const agentConfigs = options.agentConfigs.map((agent) => {
49
+ const config = {
50
+ name: agent.name,
51
+ type: "job_agent",
52
+ };
53
+ if (agent.agentId) {
54
+ config.agent_id = agent.agentId;
55
+ }
56
+ if (agent.modelName) {
57
+ config.model_name = agent.modelName;
58
+ }
59
+ if (agent.timeoutSeconds) {
60
+ config.timeout_seconds = agent.timeoutSeconds;
61
+ }
62
+ if (agent.kwargs && Object.keys(agent.kwargs).length > 0) {
63
+ config.kwargs = agent.kwargs;
64
+ }
65
+ if ((agent.environmentVariables &&
66
+ Object.keys(agent.environmentVariables).length > 0) ||
67
+ (agent.secrets && Object.keys(agent.secrets).length > 0)) {
68
+ config.agent_environment = {};
69
+ if (agent.environmentVariables &&
70
+ Object.keys(agent.environmentVariables).length > 0) {
71
+ config.agent_environment.environment_variables =
72
+ agent.environmentVariables;
73
+ }
74
+ if (agent.secrets && Object.keys(agent.secrets).length > 0) {
75
+ config.agent_environment.secrets = agent.secrets;
76
+ }
77
+ }
78
+ return config;
79
+ });
80
+ // Build orchestrator config if provided
81
+ let orchestratorConfig;
82
+ if (options.orchestratorConfig) {
83
+ orchestratorConfig = {};
84
+ if (options.orchestratorConfig.nAttempts !== undefined) {
85
+ orchestratorConfig.n_attempts = options.orchestratorConfig.nAttempts;
86
+ }
87
+ if (options.orchestratorConfig.nConcurrentTrials !== undefined) {
88
+ orchestratorConfig.n_concurrent_trials =
89
+ options.orchestratorConfig.nConcurrentTrials;
90
+ }
91
+ if (options.orchestratorConfig.quiet !== undefined) {
92
+ orchestratorConfig.quiet = options.orchestratorConfig.quiet;
93
+ }
94
+ if (options.orchestratorConfig.timeoutMultiplier !== undefined) {
95
+ orchestratorConfig.timeout_multiplier =
96
+ options.orchestratorConfig.timeoutMultiplier;
97
+ }
98
+ }
99
+ // Build the appropriate spec based on what's provided
100
+ let spec;
101
+ if (options.benchmarkId) {
102
+ spec = {
103
+ type: "benchmark",
104
+ benchmark_id: options.benchmarkId,
105
+ agent_configs: agentConfigs,
106
+ orchestrator_config: orchestratorConfig,
107
+ };
108
+ }
109
+ else if (options.scenarioIds) {
110
+ spec = {
111
+ type: "scenarios",
112
+ scenario_ids: options.scenarioIds,
113
+ agent_configs: agentConfigs,
114
+ orchestrator_config: orchestratorConfig,
115
+ };
116
+ }
117
+ const createParams = {
118
+ name: options.name,
119
+ spec,
120
+ };
121
+ return client.benchmarkJobs.create(createParams);
122
+ }
@@ -71,3 +71,50 @@ export async function getScenarioRun(id) {
71
71
  const client = getClient();
72
72
  return client.scenarios.runs.retrieve(id);
73
73
  }
74
+ /**
75
+ * List benchmark definitions with pagination
76
+ */
77
+ export async function listBenchmarks(options) {
78
+ const client = getClient();
79
+ const queryParams = {
80
+ limit: options.limit,
81
+ };
82
+ if (options.startingAfter) {
83
+ queryParams.starting_after = options.startingAfter;
84
+ }
85
+ if (options.search) {
86
+ queryParams.search = options.search;
87
+ }
88
+ const page = await client.benchmarks.list(queryParams);
89
+ const benchmarks = page.benchmarks || [];
90
+ return {
91
+ benchmarks,
92
+ totalCount: page.total_count || benchmarks.length,
93
+ hasMore: page.has_more || false,
94
+ };
95
+ }
96
+ /**
97
+ * Get benchmark definition by ID
98
+ */
99
+ export async function getBenchmark(id) {
100
+ const client = getClient();
101
+ return client.benchmarks.retrieve(id);
102
+ }
103
+ /**
104
+ * Create/start a benchmark run with selected benchmarks
105
+ */
106
+ export async function createBenchmarkRun(benchmarkIds, options) {
107
+ const client = getClient();
108
+ const createParams = {
109
+ benchmark_ids: benchmarkIds,
110
+ };
111
+ if (options?.name) {
112
+ createParams.name = options.name;
113
+ }
114
+ if (options?.metadata) {
115
+ createParams.metadata = options.metadata;
116
+ }
117
+ // Use type assertion since the API client types may not be fully defined
118
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
119
+ return client.benchmarkRuns.create(createParams);
120
+ }