@kradle/cli 0.0.17 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/README.md +93 -65
  2. package/dist/commands/agent/list.d.ts +4 -0
  3. package/dist/commands/agent/list.js +6 -4
  4. package/dist/commands/challenge/build.d.ts +9 -1
  5. package/dist/commands/challenge/build.js +40 -12
  6. package/dist/commands/challenge/create.d.ts +5 -1
  7. package/dist/commands/challenge/create.js +17 -18
  8. package/dist/commands/challenge/delete.d.ts +4 -1
  9. package/dist/commands/challenge/delete.js +5 -5
  10. package/dist/commands/challenge/list.d.ts +5 -0
  11. package/dist/commands/challenge/list.js +11 -10
  12. package/dist/commands/challenge/run.d.ts +8 -1
  13. package/dist/commands/challenge/run.js +13 -8
  14. package/dist/commands/challenge/watch.d.ts +4 -1
  15. package/dist/commands/challenge/watch.js +8 -8
  16. package/dist/commands/{evaluation → experiment}/create.d.ts +4 -0
  17. package/dist/commands/{evaluation → experiment}/create.js +22 -21
  18. package/dist/commands/{evaluation → experiment}/list.js +17 -19
  19. package/dist/commands/experiment/recordings.d.ts +19 -0
  20. package/dist/commands/experiment/recordings.js +416 -0
  21. package/dist/commands/experiment/run.d.ts +17 -0
  22. package/dist/commands/experiment/run.js +67 -0
  23. package/dist/commands/init.js +2 -2
  24. package/dist/lib/api-client.d.ts +51 -10
  25. package/dist/lib/api-client.js +108 -39
  26. package/dist/lib/arguments.d.ts +3 -2
  27. package/dist/lib/arguments.js +5 -3
  28. package/dist/lib/challenge.d.ts +13 -18
  29. package/dist/lib/challenge.js +58 -62
  30. package/dist/lib/experiment/experimenter.d.ts +92 -0
  31. package/dist/lib/experiment/experimenter.js +368 -0
  32. package/dist/lib/{evaluation → experiment}/index.d.ts +1 -1
  33. package/dist/lib/{evaluation → experiment}/index.js +1 -1
  34. package/dist/lib/{evaluation → experiment}/runner.d.ts +2 -0
  35. package/dist/lib/{evaluation → experiment}/runner.js +21 -2
  36. package/dist/lib/{evaluation → experiment}/tui.d.ts +1 -1
  37. package/dist/lib/{evaluation → experiment}/tui.js +3 -3
  38. package/dist/lib/{evaluation → experiment}/types.d.ts +10 -4
  39. package/dist/lib/{evaluation → experiment}/types.js +5 -3
  40. package/dist/lib/flags.d.ts +47 -0
  41. package/dist/lib/flags.js +63 -0
  42. package/dist/lib/schemas.d.ts +63 -2
  43. package/dist/lib/schemas.js +27 -1
  44. package/dist/lib/utils.d.ts +9 -10
  45. package/dist/lib/utils.js +12 -12
  46. package/oclif.manifest.json +423 -64
  47. package/package.json +11 -8
  48. package/static/challenge.ts +12 -13
  49. package/static/experiment_template.ts +114 -0
  50. package/static/project_template/dev.env +5 -5
  51. package/static/project_template/prod.env +4 -4
  52. package/static/project_template/tsconfig.json +1 -1
  53. package/dist/commands/challenge/multi-upload.d.ts +0 -6
  54. package/dist/commands/challenge/multi-upload.js +0 -80
  55. package/dist/commands/evaluation/run.d.ts +0 -13
  56. package/dist/commands/evaluation/run.js +0 -61
  57. package/dist/lib/config.d.ts +0 -12
  58. package/dist/lib/config.js +0 -49
  59. package/dist/lib/evaluation/evaluator.d.ts +0 -88
  60. package/dist/lib/evaluation/evaluator.js +0 -268
  61. package/static/evaluation_template.ts +0 -69
  62. /package/dist/commands/{evaluation → experiment}/list.d.ts +0 -0
@@ -0,0 +1,63 @@
1
+ import { Flags } from "@oclif/core";
2
+ import { untildify } from "./utils.js";
3
+ /**
4
+ * All available config flags that can be used by commands.
5
+ * Each flag has an `env` property that allows it to be set via environment variable.
6
+ */
7
+ export const ALL_CONFIG_FLAGS = {
8
+ "api-url": Flags.string({
9
+ description: "Kradle Web API URL",
10
+ env: "KRADLE_API_URL",
11
+ required: true,
12
+ default: "https://api.kradle.ai/v0",
13
+ }),
14
+ "web-url": Flags.string({
15
+ description: "Kradle Web URL, used to display the run URL",
16
+ env: "KRADLE_WEB_URL",
17
+ required: true,
18
+ default: "https://kradle.ai",
19
+ }),
20
+ "studio-api-url": Flags.string({
21
+ description: "Kradle Studio API URL, used to communicate with the Studio API",
22
+ env: "KRADLE_STUDIO_API_URL",
23
+ required: true,
24
+ default: "http://localhost:2999/api/v0",
25
+ }),
26
+ "studio-url": Flags.string({
27
+ description: "Kradle Studio URL, used to display the run URL in Studio",
28
+ env: "KRADLE_STUDIO_URL",
29
+ required: true,
30
+ default: "kradle://open",
31
+ }),
32
+ "api-key": Flags.string({
33
+ description: "Kradle API key",
34
+ env: "KRADLE_API_KEY",
35
+ required: true,
36
+ }),
37
+ "challenges-path": Flags.string({
38
+ description: "Absolute path to the challenges directory",
39
+ env: "KRADLE_CHALLENGES_PATH",
40
+ default: "~/Documents/kradle-studio/challenges",
41
+ parse: async (input) => untildify(input),
42
+ }),
43
+ };
44
+ /**
45
+ * Returns a subset of config flags for use in a command's static flags definition.
46
+ *
47
+ * @example
48
+ * // In a command file:
49
+ * static override flags = {
50
+ * ...getConfigFlags("api-key", "web-api-url", "studio-api-url"),
51
+ * // other command-specific flags
52
+ * };
53
+ *
54
+ * @param keys - The config flag keys to include.
55
+ * @returns An object containing only the specified flags
56
+ */
57
+ export function getConfigFlags(...keys) {
58
+ const result = {};
59
+ for (const key of keys) {
60
+ result[key] = ALL_CONFIG_FLAGS[key];
61
+ }
62
+ return result;
63
+ }
@@ -40,6 +40,37 @@ export declare const ChallengeSchema: z.ZodObject<{
40
40
  updateTime: z.ZodOptional<z.ZodString>;
41
41
  creator: z.ZodOptional<z.ZodString>;
42
42
  }, z.core.$strip>;
43
+ export declare const ChallengeConfigSchema: z.ZodObject<{
44
+ name: z.ZodString;
45
+ description: z.ZodOptional<z.ZodString>;
46
+ domain: z.ZodString;
47
+ world: z.ZodString;
48
+ challengeConfig: z.ZodObject<{
49
+ cheat: z.ZodBoolean;
50
+ datapack: z.ZodBoolean;
51
+ gameMode: z.ZodEnum<{
52
+ survival: "survival";
53
+ creative: "creative";
54
+ adventure: "adventure";
55
+ spectator: "spectator";
56
+ }>;
57
+ }, z.core.$strip>;
58
+ task: z.ZodOptional<z.ZodString>;
59
+ roles: z.ZodRecord<z.ZodString, z.ZodObject<{
60
+ description: z.ZodOptional<z.ZodString>;
61
+ specificTask: z.ZodOptional<z.ZodString>;
62
+ minParticipants: z.ZodOptional<z.ZodNumber>;
63
+ maxParticipants: z.ZodOptional<z.ZodNumber>;
64
+ }, z.core.$strip>>;
65
+ objective: z.ZodObject<{
66
+ fieldName: z.ZodString;
67
+ direction: z.ZodEnum<{
68
+ maximize: "maximize";
69
+ minimize: "minimize";
70
+ }>;
71
+ }, z.core.$strip>;
72
+ endStates: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
73
+ }, z.core.$strip>;
43
74
  export declare const ChallengesResponseSchema: z.ZodObject<{
44
75
  challenges: z.ZodArray<z.ZodObject<{
45
76
  id: z.ZodOptional<z.ZodString>;
@@ -87,8 +118,19 @@ export declare const ChallengesResponseSchema: z.ZodObject<{
87
118
  export declare const HumanSchema: z.ZodObject<{
88
119
  username: z.ZodString;
89
120
  }, z.core.$strip>;
90
- export declare const RunResponseSchema: z.ZodObject<{
121
+ export declare const RunParticipantSchema: z.ZodObject<{
122
+ agent: z.ZodString;
123
+ role: z.ZodString;
124
+ inputOrder: z.ZodNumber;
125
+ }, z.core.$strip>;
126
+ export declare const JobResponseSchema: z.ZodObject<{
91
127
  runIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
128
+ participants: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
129
+ agent: z.ZodString;
130
+ role: z.ZodString;
131
+ inputOrder: z.ZodNumber;
132
+ }, z.core.$strip>>>;
133
+ id: z.ZodOptional<z.ZodString>;
92
134
  }, z.core.$strip>;
93
135
  export declare const RunStatusSchema: z.ZodObject<{
94
136
  id: z.ZodString;
@@ -134,10 +176,29 @@ export declare const AgentsResponseSchema: z.ZodObject<{
134
176
  }, z.core.$strip>>;
135
177
  nextPageToken: z.ZodOptional<z.ZodString>;
136
178
  }, z.core.$strip>;
179
+ export declare const RecordingMetadataSchema: z.ZodObject<{
180
+ timestamp: z.ZodString;
181
+ sizeBytes: z.ZodNumber;
182
+ }, z.core.$strip>;
183
+ export declare const RecordingsListResponseSchema: z.ZodObject<{
184
+ recordings: z.ZodArray<z.ZodObject<{
185
+ timestamp: z.ZodString;
186
+ sizeBytes: z.ZodNumber;
187
+ }, z.core.$strip>>;
188
+ }, z.core.$strip>;
189
+ export declare const RecordingDownloadUrlResponseSchema: z.ZodObject<{
190
+ downloadUrl: z.ZodString;
191
+ expiresAt: z.ZodString;
192
+ }, z.core.$strip>;
137
193
  export type ChallengeSchemaType = z.infer<typeof ChallengeSchema>;
194
+ export type ChallengeConfigSchemaType = z.infer<typeof ChallengeConfigSchema>;
138
195
  export type ChallengesResponseType = z.infer<typeof ChallengesResponseSchema>;
139
196
  export type HumanSchemaType = z.infer<typeof HumanSchema>;
140
- export type RunResponseType = z.infer<typeof RunResponseSchema>;
197
+ export type JobResponseType = z.infer<typeof JobResponseSchema>;
141
198
  export type RunStatusSchemaType = z.infer<typeof RunStatusSchema>;
142
199
  export type AgentSchemaType = z.infer<typeof AgentSchema>;
143
200
  export type AgentsResponseType = z.infer<typeof AgentsResponseSchema>;
201
+ export type RecordingMetadata = z.infer<typeof RecordingMetadataSchema>;
202
+ export type RecordingsListResponse = z.infer<typeof RecordingsListResponseSchema>;
203
+ export type RecordingDownloadUrlResponse = z.infer<typeof RecordingDownloadUrlResponseSchema>;
204
+ export type RunParticipant = z.infer<typeof RunParticipantSchema>;
@@ -28,6 +28,14 @@ export const ChallengeSchema = z.object({
28
28
  updateTime: z.string().optional(),
29
29
  creator: z.string().optional(),
30
30
  });
31
+ export const ChallengeConfigSchema = ChallengeSchema.omit({
32
+ slug: true,
33
+ id: true,
34
+ visibility: true,
35
+ creationTime: true,
36
+ updateTime: true,
37
+ creator: true,
38
+ });
31
39
  export const ChallengesResponseSchema = z.object({
32
40
  challenges: z.array(ChallengeSchema),
33
41
  nextPageToken: z.string().optional(),
@@ -35,8 +43,15 @@ export const ChallengesResponseSchema = z.object({
35
43
  export const HumanSchema = z.object({
36
44
  username: z.string(),
37
45
  });
38
- export const RunResponseSchema = z.object({
46
+ export const RunParticipantSchema = z.object({
47
+ agent: z.string(),
48
+ role: z.string(),
49
+ inputOrder: z.number(),
50
+ });
51
+ export const JobResponseSchema = z.object({
39
52
  runIds: z.array(z.string()).optional(),
53
+ participants: z.record(z.string(), RunParticipantSchema).optional(),
54
+ id: z.string().optional(),
40
55
  });
41
56
  export const RunStatusSchema = z.object({
42
57
  id: z.string(),
@@ -64,3 +79,14 @@ export const AgentsResponseSchema = z.object({
64
79
  agents: z.array(AgentSchema),
65
80
  nextPageToken: z.string().optional(),
66
81
  });
82
+ export const RecordingMetadataSchema = z.object({
83
+ timestamp: z.string(),
84
+ sizeBytes: z.number(),
85
+ });
86
+ export const RecordingsListResponseSchema = z.object({
87
+ recordings: z.array(RecordingMetadataSchema),
88
+ });
89
+ export const RecordingDownloadUrlResponseSchema = z.object({
90
+ downloadUrl: z.string(),
91
+ expiresAt: z.string(),
92
+ });
@@ -1,5 +1,4 @@
1
1
  import type { Dirent } from "node:fs";
2
- import type { Config } from "./config.js";
3
2
  export declare function loadTemplateRun(): Promise<Record<string, unknown>>;
4
3
  /**
5
4
  * Resolve tildes in a path to the home directory
@@ -33,34 +32,34 @@ export declare function getStaticResourcePath(name: string): string;
33
32
  */
34
33
  export declare function readDirSorted(dir: string): Promise<Dirent[]>;
35
34
  /**
36
- * Runs a NodeJS module. The module is executed in the current working directory, and the environment is set to the given config.
35
+ * Runs a NodeJS module. The module is executed in the current working directory, and the environment is set to the given env.
37
36
  * stdout & stdout are piped to the parent process.
38
37
  *
39
38
  * Returns a promise that resolves when the module completes.
40
39
  *
41
40
  * @param file The file to execute.
42
- * @param config The configuration to use.
41
+ * @param env The environment variables to set.
43
42
  * @param options The options to use.
44
43
  * @param options.silent Whether to suppress stdout.
45
44
  * @returns A promise that resolves when the command completes.
46
45
  */
47
- export declare function runNodeModule(file: string, config: Config, options?: {
46
+ export declare function runNodeModule(file: string, env: Record<string, string>, options?: {
48
47
  silent?: boolean;
49
48
  }): Promise<undefined>;
50
49
  /**
51
50
  * Execute a TypeScript file using NodeJS's built-in TS support.
52
51
  *
53
52
  * @param filePath The path to the TypeScript file.
54
- * @param config The configuration to use.
53
+ * @param env The environment variables to set.
55
54
  * @param options The options to use.
56
55
  * @param options.silent Whether to suppress stdout.
57
56
  * @returns The result of the execution.
58
57
  */
59
- export declare function executeTypescriptFile(filePath: string, config: Config, options?: {
58
+ export declare function executeTypescriptFile(filePath: string, env: Record<string, string>, options?: {
60
59
  silent?: boolean;
61
60
  }): Promise<unknown>;
62
61
  /**
63
- * Executes an arbitrary command. The command is spawned in the current working directory, and the environment is set to the given config.
62
+ * Executes an arbitrary command. The command is spawned in the current working directory, and the environment is set to the given env.
64
63
  *
65
64
  * stdout will be returned as a string.
66
65
  * if stderr is not empty, an error will be thrown.
@@ -76,17 +75,17 @@ export declare function executeCommand(command: string, args: string[], options?
76
75
  cwd?: string;
77
76
  }): Promise<string>;
78
77
  /**
79
- * Executes an arbitrary NodeJS command. The command is spawned in the current working directory, and the environment is set to the given config.
78
+ * Executes an arbitrary NodeJS command. The command is spawned in the current working directory, and the environment is set to the given env.
80
79
  * It will use the process.execPath of the current process to find the NodeJS executable.
81
80
  *
82
81
  * stdout will be returned as a string.
83
82
  * if stderr is not empty, an error will be thrown.
84
83
  *
85
84
  * @param args The arguments to pass to the command.
86
- * @param config The configuration to use.
85
+ * @param env The environment variables to set.
87
86
  * @returns A promise that resolves with the stdout of the command.
88
87
  */
89
- export declare function executeNodeCommand(args: string[], config: Config): Promise<string>;
88
+ export declare function executeNodeCommand(args: string[], env: Record<string, string>): Promise<string>;
90
89
  /**
91
90
  * Open a URL in the default browser.
92
91
  * This is fire-and-forget, so we don't wait for it to complete.
package/dist/lib/utils.js CHANGED
@@ -75,22 +75,22 @@ export async function readDirSorted(dir) {
75
75
  return files.sort((a, b) => a.name.localeCompare(b.name));
76
76
  }
77
77
  /**
78
- * Runs a NodeJS module. The module is executed in the current working directory, and the environment is set to the given config.
78
+ * Runs a NodeJS module. The module is executed in the current working directory, and the environment is set to the given env.
79
79
  * stdout & stdout are piped to the parent process.
80
80
  *
81
81
  * Returns a promise that resolves when the module completes.
82
82
  *
83
83
  * @param file The file to execute.
84
- * @param config The configuration to use.
84
+ * @param env The environment variables to set.
85
85
  * @param options The options to use.
86
86
  * @param options.silent Whether to suppress stdout.
87
87
  * @returns A promise that resolves when the command completes.
88
88
  */
89
- export function runNodeModule(file, config, options = {}) {
89
+ export function runNodeModule(file, env, options = {}) {
90
90
  return new Promise((resolve, reject) => {
91
91
  const child = fork(file, {
92
92
  cwd: path.resolve(process.cwd()),
93
- env: { ...process.env, ...config },
93
+ env: { ...process.env, ...env },
94
94
  silent: options.silent,
95
95
  });
96
96
  child.on("close", (code) => {
@@ -108,16 +108,16 @@ export function runNodeModule(file, config, options = {}) {
108
108
  * Execute a TypeScript file using NodeJS's built-in TS support.
109
109
  *
110
110
  * @param filePath The path to the TypeScript file.
111
- * @param config The configuration to use.
111
+ * @param env The environment variables to set.
112
112
  * @param options The options to use.
113
113
  * @param options.silent Whether to suppress stdout.
114
114
  * @returns The result of the execution.
115
115
  */
116
- export async function executeTypescriptFile(filePath, config, options = {}) {
117
- return await runNodeModule(filePath, config, options);
116
+ export async function executeTypescriptFile(filePath, env, options = {}) {
117
+ return await runNodeModule(filePath, env, options);
118
118
  }
119
119
  /**
120
- * Executes an arbitrary command. The command is spawned in the current working directory, and the environment is set to the given config.
120
+ * Executes an arbitrary command. The command is spawned in the current working directory, and the environment is set to the given env.
121
121
  *
122
122
  * stdout will be returned as a string.
123
123
  * if stderr is not empty, an error will be thrown.
@@ -155,18 +155,18 @@ export async function executeCommand(command, args, options) {
155
155
  });
156
156
  }
157
157
  /**
158
- * Executes an arbitrary NodeJS command. The command is spawned in the current working directory, and the environment is set to the given config.
158
+ * Executes an arbitrary NodeJS command. The command is spawned in the current working directory, and the environment is set to the given env.
159
159
  * It will use the process.execPath of the current process to find the NodeJS executable.
160
160
  *
161
161
  * stdout will be returned as a string.
162
162
  * if stderr is not empty, an error will be thrown.
163
163
  *
164
164
  * @param args The arguments to pass to the command.
165
- * @param config The configuration to use.
165
+ * @param env The environment variables to set.
166
166
  * @returns A promise that resolves with the stdout of the command.
167
167
  */
168
- export async function executeNodeCommand(args, config) {
169
- return executeCommand(process.execPath, args, { env: config });
168
+ export async function executeNodeCommand(args, env) {
169
+ return executeCommand(process.execPath, args, { env });
170
170
  }
171
171
  /**
172
172
  * Open a URL in the default browser.