@kradle/cli 0.0.16 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +62 -65
  2. package/dist/commands/agent/list.d.ts +4 -0
  3. package/dist/commands/agent/list.js +6 -4
  4. package/dist/commands/challenge/build.d.ts +9 -1
  5. package/dist/commands/challenge/build.js +40 -12
  6. package/dist/commands/challenge/create.d.ts +5 -1
  7. package/dist/commands/challenge/create.js +17 -18
  8. package/dist/commands/challenge/delete.d.ts +4 -1
  9. package/dist/commands/challenge/delete.js +5 -5
  10. package/dist/commands/challenge/list.d.ts +5 -0
  11. package/dist/commands/challenge/list.js +9 -10
  12. package/dist/commands/challenge/run.d.ts +8 -1
  13. package/dist/commands/challenge/run.js +13 -8
  14. package/dist/commands/challenge/watch.d.ts +4 -1
  15. package/dist/commands/challenge/watch.js +8 -8
  16. package/dist/commands/{evaluation → experiment}/create.d.ts +4 -0
  17. package/dist/commands/{evaluation → experiment}/create.js +22 -21
  18. package/dist/commands/{evaluation → experiment}/list.js +17 -19
  19. package/dist/commands/{evaluation → experiment}/run.d.ts +4 -1
  20. package/dist/commands/experiment/run.js +61 -0
  21. package/dist/commands/init.js +2 -2
  22. package/dist/lib/api-client.d.ts +29 -10
  23. package/dist/lib/api-client.js +81 -37
  24. package/dist/lib/arguments.d.ts +3 -2
  25. package/dist/lib/arguments.js +5 -3
  26. package/dist/lib/challenge.d.ts +13 -18
  27. package/dist/lib/challenge.js +60 -61
  28. package/dist/lib/experiment/experimenter.d.ts +87 -0
  29. package/dist/lib/{evaluation/evaluator.js → experiment/experimenter.js} +74 -72
  30. package/dist/lib/{evaluation → experiment}/index.d.ts +1 -1
  31. package/dist/lib/{evaluation → experiment}/index.js +1 -1
  32. package/dist/lib/{evaluation → experiment}/runner.js +2 -1
  33. package/dist/lib/{evaluation → experiment}/tui.d.ts +1 -1
  34. package/dist/lib/{evaluation → experiment}/tui.js +3 -3
  35. package/dist/lib/{evaluation → experiment}/types.d.ts +6 -4
  36. package/dist/lib/{evaluation → experiment}/types.js +4 -3
  37. package/dist/lib/flags.d.ts +47 -0
  38. package/dist/lib/flags.js +63 -0
  39. package/dist/lib/schemas.d.ts +32 -0
  40. package/dist/lib/schemas.js +8 -0
  41. package/dist/lib/utils.d.ts +9 -10
  42. package/dist/lib/utils.js +12 -12
  43. package/oclif.manifest.json +342 -64
  44. package/package.json +5 -6
  45. package/static/challenge.ts +12 -13
  46. package/static/experiment_template.ts +114 -0
  47. package/static/project_template/dev.env +5 -5
  48. package/static/project_template/prod.env +4 -4
  49. package/static/project_template/tsconfig.json +1 -1
  50. package/dist/commands/challenge/multi-upload.d.ts +0 -6
  51. package/dist/commands/challenge/multi-upload.js +0 -80
  52. package/dist/commands/evaluation/run.js +0 -61
  53. package/dist/lib/config.d.ts +0 -12
  54. package/dist/lib/config.js +0 -49
  55. package/dist/lib/evaluation/evaluator.d.ts +0 -88
  56. package/static/evaluation_template.ts +0 -69
  57. /package/dist/commands/{evaluation → experiment}/list.d.ts +0 -0
  58. /package/dist/lib/{evaluation → experiment}/runner.d.ts +0 -0
@@ -1,17 +1,10 @@
1
1
  import type { ApiClient } from "./api-client.js";
2
- import type { Config } from "./config.js";
3
- import { type ChallengeSchemaType } from "./schemas.js";
2
+ import { type ChallengeConfigSchemaType } from "./schemas.js";
4
3
  export declare const SOURCE_FOLDER = ".src";
5
4
  export declare class Challenge {
6
- readonly name: string;
7
- private readonly config;
8
- constructor(name: string, config: Config);
9
- private get kradleChallengesPath();
10
- /**
11
- * Get the short slug without username prefix
12
- * e.g., "florianernst:challenge" -> "challenge"
13
- */
14
- get shortSlug(): string;
5
+ readonly shortSlug: string;
6
+ private readonly kradleChallengesPath;
7
+ constructor(shortSlug: string, kradleChallengesPath: string);
15
8
  /**
16
9
  * Get the path to the challenge directory
17
10
  */
@@ -47,22 +40,24 @@ export declare class Challenge {
47
40
  /**
48
41
  * Load the challenge configuration from config.ts
49
42
  */
50
- loadConfig(): Promise<ChallengeSchemaType>;
43
+ loadConfig(): Promise<ChallengeConfigSchemaType>;
51
44
  /**
52
- * Upload the challenge datapack to Google Cloud Storage
45
+ * Build the challenge datapack and upload it to the cloud.
46
+ * @param api - The API client to use.
47
+ * @param asPublic - Whether the challenge should be uploaded as public.
48
+ * @returns The config and datapack hash.
53
49
  */
54
- upload(apiClient: ApiClient): Promise<void>;
55
- buildAndUpload(api: ApiClient): Promise<{
56
- config: ChallengeSchemaType;
50
+ buildAndUpload(api: ApiClient, asPublic: boolean): Promise<{
51
+ config: ChallengeConfigSchemaType;
57
52
  datapackHash: string;
58
53
  }>;
59
54
  /**
60
55
  * Get all local challenges from the challenges directory
61
56
  */
62
- static getLocalChallenges(): Promise<Record<string, boolean>>;
57
+ static getLocalChallenges(): Promise<string[]>;
63
58
  /**
64
59
  * Create a new local challenge folder with challenge.ts template
65
60
  * Note: config.ts is NOT created here - it should be generated later via `challenge config download`
66
61
  */
67
- static createLocal(slug: string, config: Config): Promise<void>;
62
+ static createLocal(slug: string, kradleChallengesPath: string): Promise<void>;
68
63
  }
@@ -4,25 +4,15 @@ import fs from "node:fs/promises";
4
4
  import path from "node:path";
5
5
  import pc from "picocolors";
6
6
  import * as tar from "tar";
7
- import { ChallengeSchema } from "./schemas.js";
7
+ import { ChallengeConfigSchema } from "./schemas.js";
8
8
  import { executeNodeCommand, executeTypescriptFile, getStaticResourcePath, readDirSorted } from "./utils.js";
9
9
  export const SOURCE_FOLDER = ".src";
10
10
  export class Challenge {
11
- name;
12
- config;
13
- constructor(name, config) {
14
- this.name = name;
15
- this.config = config;
16
- }
17
- get kradleChallengesPath() {
18
- return path.resolve(this.config.KRADLE_CHALLENGES_PATH);
19
- }
20
- /**
21
- * Get the short slug without username prefix
22
- * e.g., "florianernst:challenge" -> "challenge"
23
- */
24
- get shortSlug() {
25
- return this.name.includes(":") ? this.name.split(":")[1] : this.name;
11
+ shortSlug;
12
+ kradleChallengesPath;
13
+ constructor(shortSlug, kradleChallengesPath) {
14
+ this.shortSlug = shortSlug;
15
+ this.kradleChallengesPath = path.resolve(kradleChallengesPath);
26
16
  }
27
17
  /**
28
18
  * Get the path to the challenge directory
@@ -94,15 +84,27 @@ export class Challenge {
94
84
  throw new Error(`Challenge file not found at ${this.challengePath}`);
95
85
  }
96
86
  try {
97
- await executeTypescriptFile(this.challengePath, this.config, { silent });
87
+ await executeTypescriptFile(this.challengePath, {
88
+ KRADLE_CHALLENGES_PATH: this.kradleChallengesPath,
89
+ NAMESPACE: "kradle",
90
+ }, { silent });
98
91
  }
99
92
  catch (error) {
100
93
  throw new Error(`Failed to build datapack: ${error instanceof Error ? error.message : error}`);
101
94
  }
102
95
  // @TODO - re-enable once we have a proper build pipeline
103
96
  // Copy challenge.ts and config.ts to the src directory
104
- await fs.cp(path.join(this.challengeDir, "challenge.ts"), path.join(this.config.KRADLE_CHALLENGES_PATH, this.shortSlug, SOURCE_FOLDER, "challenge.ts"));
105
- await fs.cp(path.join(this.challengeDir, "config.ts"), path.join(this.config.KRADLE_CHALLENGES_PATH, this.shortSlug, SOURCE_FOLDER, "config.ts"));
97
+ await fs.cp(path.join(this.challengeDir, "challenge.ts"), path.join(this.kradleChallengesPath, this.shortSlug, SOURCE_FOLDER, "challenge.ts"));
98
+ await fs.cp(path.join(this.challengeDir, "config.ts"), path.join(this.kradleChallengesPath, this.shortSlug, SOURCE_FOLDER, "config.ts"));
99
+ // Create tarball
100
+ // We need to use the cwd option to make sure the structure of the tarball uses paths relative to the challenge directory
101
+ await tar.create({
102
+ "no-mtime": true,
103
+ file: this.tarballPath, // Output file name
104
+ gzip: true,
105
+ cwd: this.datapackPath, // Input directory
106
+ strict: true,
107
+ }, ["datapack", SOURCE_FOLDER]);
106
108
  }
107
109
  /**
108
110
  * Load the challenge configuration from config.ts
@@ -118,60 +120,54 @@ export class Challenge {
118
120
  "--no-warnings",
119
121
  "-e",
120
122
  `console.log(JSON.stringify(require("${this.configPath}").config));`,
121
- ], this.config);
122
- return ChallengeSchema.parse(JSON.parse(stdout));
123
+ ], {
124
+ KRADLE_CHALLENGES_PATH: this.kradleChallengesPath,
125
+ NAMESPACE: "kradle",
126
+ });
127
+ return ChallengeConfigSchema.parse(JSON.parse(stdout));
123
128
  }
124
129
  /**
125
- * Upload the challenge datapack to Google Cloud Storage
130
+ * Build the challenge datapack and upload it to the cloud.
131
+ * @param api - The API client to use.
132
+ * @param asPublic - Whether the challenge should be uploaded as public.
133
+ * @returns The config and datapack hash.
126
134
  */
127
- async upload(apiClient) {
128
- // Create tarball
129
- // We need to use the cwd option to make sure the structure of the tarball uses paths relative to the challenge directory
130
- await tar.create({
131
- "no-mtime": true,
132
- file: this.tarballPath, // Output file name
133
- gzip: true,
134
- cwd: this.datapackPath, // Input directory
135
- strict: true,
136
- }, ["datapack", SOURCE_FOLDER]);
137
- const uploadUrl = await apiClient.getChallengeUploadUrl(this);
138
- const fileBuffer = await fs.readFile(this.tarballPath);
139
- const response = await fetch(uploadUrl, {
140
- method: "PUT",
141
- headers: {
142
- "Content-Type": "application/gzip",
143
- "Content-Length": fileBuffer.length.toString(),
144
- },
145
- body: fileBuffer,
146
- });
147
- if (!response.ok) {
148
- throw new Error(`Failed to upload datapack: ${response.statusText}`);
149
- }
150
- }
151
- async buildAndUpload(api) {
135
+ async buildAndUpload(api, asPublic) {
152
136
  // Ensure challenge exists locally
153
137
  if (!this.exists()) {
154
- throw new Error(`Challenge "${this.name}" does not exist locally. Make sure both the challenge.ts file and the config.ts file exist.`);
138
+ throw new Error(`Challenge "${this.shortSlug}" does not exist locally. Make sure both the challenge.ts file and the config.ts file exist.`);
155
139
  }
156
140
  // Ensure challenge exists in the cloud
157
- if (!(await api.challengeExists(this.name))) {
158
- console.log(pc.yellow(`Challenge not found in cloud: ${this.name}`));
159
- console.log(pc.yellow(`Creating challenge: ${this.name}`));
160
- await api.createChallenge(this.name);
141
+ if (!(await api.challengeExists(this.shortSlug))) {
142
+ console.log(pc.yellow(`Challenge not found in cloud: ${this.shortSlug}`));
143
+ console.log(pc.yellow(`Creating challenge: ${this.shortSlug}`));
144
+ await api.createChallenge(this.shortSlug);
161
145
  console.log(pc.green(`✓ Challenge created in cloud`));
162
146
  }
163
- // Load config
164
147
  const config = await this.loadConfig();
148
+ // Ensure challenge's visibility is set to private - else, temporarily set it to private
149
+ const cloudChallengeVisibility = (await api.getChallenge(this.shortSlug)).visibility;
150
+ if (cloudChallengeVisibility === "public") {
151
+ console.log(pc.yellow(`⚠️ Cloud challenge is currently public. Setting its visibility to private...`));
152
+ await api.updateChallengeVisibility(this.shortSlug, "private");
153
+ console.log(pc.yellow(`✓ Challenge visibility set to private\n`));
154
+ }
165
155
  // Upload config
166
156
  console.log(pc.blue(`>> Uploading config: ${this.shortSlug}`));
167
- await api.updateChallenge(this, config);
168
- console.log(pc.green(`✓ Config uploaded`));
157
+ // We have to set it to private because we cannot update public challenges
158
+ await api.updateChallenge(this.shortSlug, config, "private");
159
+ console.log(pc.green(`✓ Config uploaded\n`));
169
160
  // Build and upload datapack
170
161
  console.log(pc.blue(`>> Building datapack: ${this.shortSlug}`));
171
162
  await this.build();
172
163
  console.log(pc.blue(`>> Uploading datapack: ${this.shortSlug}`));
173
- await this.upload(api);
174
- console.log(pc.green(`✓ Build & upload complete for ${this.shortSlug}`));
164
+ await api.uploadChallengeDatapack(this.shortSlug, this.tarballPath);
165
+ console.log(pc.green(`✓ Build & upload complete for ${this.shortSlug}\n`));
166
+ if (asPublic) {
167
+ console.log(pc.green(`👀 Setting challenge visibility to public...`));
168
+ await api.updateChallengeVisibility(this.shortSlug, "public");
169
+ console.log(pc.green(`✓ Challenge visibility set to public\n`));
170
+ }
175
171
  const datapackHash = await this.getDatapackHash();
176
172
  return { config, datapackHash };
177
173
  }
@@ -180,14 +176,17 @@ export class Challenge {
180
176
  */
181
177
  static async getLocalChallenges() {
182
178
  const challengesDir = path.resolve(process.cwd(), "challenges");
179
+ if (!existsSync(challengesDir)) {
180
+ return [];
181
+ }
183
182
  const entries = await readDirSorted(challengesDir);
184
- const challenges = {};
183
+ const challenges = [];
185
184
  for (const entry of entries) {
186
185
  if (entry.isDirectory()) {
187
186
  // Check if it has a challenge.ts file
188
187
  const challengePath = path.join(challengesDir, entry.name, "challenge.ts");
189
188
  if (existsSync(challengePath)) {
190
- challenges[entry.name] = true;
189
+ challenges.push(entry.name);
191
190
  }
192
191
  }
193
192
  }
@@ -197,8 +196,8 @@ export class Challenge {
197
196
  * Create a new local challenge folder with challenge.ts template
198
197
  * Note: config.ts is NOT created here - it should be generated later via `challenge config download`
199
198
  */
200
- static async createLocal(slug, config) {
201
- const challenge = new Challenge(slug, config);
199
+ static async createLocal(slug, kradleChallengesPath) {
200
+ const challenge = new Challenge(slug, kradleChallengesPath);
202
201
  const challengeTemplatePath = getStaticResourcePath("challenge.ts");
203
202
  // Create challenge directory
204
203
  await fs.mkdir(challenge.challengeDir, { recursive: true });
@@ -0,0 +1,87 @@
1
+ import type { ApiClient } from "../api-client.js";
2
+ import type { ExperimentMetadata, ExperimentOptions, Manifest, Progress } from "./types.js";
3
+ export declare class Experimenter {
4
+ private name;
5
+ private webUrl;
6
+ private api;
7
+ experimentDir: string;
8
+ metadataPath: string;
9
+ private runner?;
10
+ private tui?;
11
+ private currentVersion?;
12
+ constructor(name: string, webUrl: string, api: ApiClient);
13
+ /**
14
+ * Get paths for a specific version
15
+ */
16
+ private getVersionPaths;
17
+ get configPath(): string;
18
+ /**
19
+ * Get the current version directory path
20
+ */
21
+ getCurrentVersionDir(): string;
22
+ /**
23
+ * Check if experiment exists
24
+ */
25
+ exists(): Promise<boolean>;
26
+ /**
27
+ * Check if config.ts exists (master config)
28
+ */
29
+ configExists(): Promise<boolean>;
30
+ /**
31
+ * Load experiment metadata
32
+ */
33
+ loadMetadata(): Promise<ExperimentMetadata | null>;
34
+ /**
35
+ * Save experiment metadata
36
+ */
37
+ saveMetadata(metadata: ExperimentMetadata): Promise<void>;
38
+ /**
39
+ * Get the current version number, or -1 if none exists
40
+ */
41
+ getCurrentVersionNumber(): Promise<number>;
42
+ /**
43
+ * Create a new version
44
+ */
45
+ createNewVersion(): Promise<number>;
46
+ /**
47
+ * Get or create a version
48
+ * @param createNew - If true, always create a new version. Otherwise, use current version or create first one if none exists.
49
+ */
50
+ getOrCreateVersion(createNew: boolean): Promise<number>;
51
+ /**
52
+ * Load manifest from version
53
+ */
54
+ loadManifest(version: number): Promise<Manifest>;
55
+ /**
56
+ * Load progress from version
57
+ */
58
+ loadProgress(version: number): Promise<Progress | null>;
59
+ /**
60
+ * Save progress to current version
61
+ */
62
+ saveProgress(): Promise<void>;
63
+ /**
64
+ * Execute config.ts to generate manifest
65
+ */
66
+ generateManifest(configPath: string): Promise<Manifest>;
67
+ /**
68
+ * Execute config.ts file and return the manifest
69
+ */
70
+ private executeConfigFile;
71
+ /**
72
+ * Run the experiment
73
+ */
74
+ run(options: ExperimentOptions): Promise<void>;
75
+ /**
76
+ * Handle state change from runner
77
+ */
78
+ private onRunStateChange;
79
+ /**
80
+ * Handle quit request
81
+ */
82
+ private handleQuit;
83
+ /**
84
+ * Open run in browser
85
+ */
86
+ private openRun;
87
+ }
@@ -1,55 +1,56 @@
1
1
  import fs from "node:fs/promises";
2
2
  import path from "node:path";
3
+ import pc from "picocolors";
3
4
  import { executeNodeCommand, openInBrowser } from "../utils.js";
4
5
  import { Runner } from "./runner.js";
5
6
  import { TUI } from "./tui.js";
6
- import { EvaluationMetadataSchema, ManifestSchema, ProgressSchema } from "./types.js";
7
- export class Evaluator {
7
+ import { ExperimentMetadataSchema, ManifestSchema, ProgressSchema } from "./types.js";
8
+ export class Experimenter {
8
9
  name;
9
- config;
10
+ webUrl;
10
11
  api;
11
- evaluationDir;
12
+ experimentDir;
12
13
  metadataPath;
13
14
  runner;
14
15
  tui;
15
- currentIteration;
16
- constructor(name, config, api) {
16
+ currentVersion;
17
+ constructor(name, webUrl, api) {
17
18
  this.name = name;
18
- this.config = config;
19
+ this.webUrl = webUrl;
19
20
  this.api = api;
20
- this.evaluationDir = path.resolve(process.cwd(), "evaluations", name);
21
- this.metadataPath = path.join(this.evaluationDir, ".evaluation.json");
21
+ this.experimentDir = path.resolve(process.cwd(), "experiments", name);
22
+ this.metadataPath = path.join(this.experimentDir, ".experiment.json");
22
23
  }
23
24
  /**
24
- * Get paths for a specific iteration
25
+ * Get paths for a specific version
25
26
  */
26
- getIterationPaths(iteration) {
27
- const iterationDir = path.join(this.evaluationDir, "iterations", iteration.toString().padStart(3, "0"));
27
+ getVersionPaths(version) {
28
+ const versionDir = path.join(this.experimentDir, "versions", version.toString().padStart(3, "0"));
28
29
  return {
29
- iterationDir,
30
- configPath: path.join(iterationDir, "config.ts"),
31
- manifestPath: path.join(iterationDir, "manifest.json"),
32
- progressPath: path.join(iterationDir, "progress.json"),
30
+ versionDir,
31
+ configPath: path.join(versionDir, "config.ts"),
32
+ manifestPath: path.join(versionDir, "manifest.json"),
33
+ progressPath: path.join(versionDir, "progress.json"),
33
34
  };
34
35
  }
35
36
  get configPath() {
36
- return path.join(this.evaluationDir, "config.ts");
37
+ return path.join(this.experimentDir, "config.ts");
37
38
  }
38
39
  /**
39
- * Get the current iteration directory path
40
+ * Get the current version directory path
40
41
  */
41
- getCurrentIterationDir() {
42
- if (this.currentIteration === undefined) {
43
- throw new Error("No iteration set");
42
+ getCurrentVersionDir() {
43
+ if (this.currentVersion === undefined) {
44
+ throw new Error("No version set");
44
45
  }
45
- return this.getIterationPaths(this.currentIteration).iterationDir;
46
+ return this.getVersionPaths(this.currentVersion).versionDir;
46
47
  }
47
48
  /**
48
- * Check if evaluation exists
49
+ * Check if experiment exists
49
50
  */
50
51
  async exists() {
51
52
  try {
52
- await fs.access(this.evaluationDir);
53
+ await fs.access(this.experimentDir);
53
54
  return true;
54
55
  }
55
56
  catch {
@@ -69,81 +70,81 @@ export class Evaluator {
69
70
  }
70
71
  }
71
72
  /**
72
- * Load evaluation metadata
73
+ * Load experiment metadata
73
74
  */
74
75
  async loadMetadata() {
75
76
  try {
76
77
  const content = await fs.readFile(this.metadataPath, "utf-8");
77
78
  const data = JSON.parse(content);
78
- return EvaluationMetadataSchema.parse(data);
79
+ return ExperimentMetadataSchema.parse(data);
79
80
  }
80
81
  catch {
81
82
  return null;
82
83
  }
83
84
  }
84
85
  /**
85
- * Save evaluation metadata
86
+ * Save experiment metadata
86
87
  */
87
88
  async saveMetadata(metadata) {
88
89
  await fs.writeFile(this.metadataPath, JSON.stringify(metadata, null, 2));
89
90
  }
90
91
  /**
91
- * Get the current iteration number, or -1 if none exists
92
+ * Get the current version number, or -1 if none exists
92
93
  */
93
- async getCurrentIterationNumber() {
94
+ async getCurrentVersionNumber() {
94
95
  const metadata = await this.loadMetadata();
95
- return metadata?.currentIteration ?? -1;
96
+ return metadata?.currentVersion ?? -1;
96
97
  }
97
98
  /**
98
- * Create a new iteration
99
+ * Create a new version
99
100
  */
100
- async createNewIteration() {
101
- const currentIteration = await this.getCurrentIterationNumber();
102
- const newIteration = currentIteration + 1;
103
- const paths = this.getIterationPaths(newIteration);
104
- // Create iteration directory
105
- await fs.mkdir(paths.iterationDir, { recursive: true });
106
- // Copy master config to iteration
107
- const masterConfigPath = path.join(this.evaluationDir, "config.ts");
101
+ async createNewVersion() {
102
+ const currentVersion = await this.getCurrentVersionNumber();
103
+ const newVersion = currentVersion + 1;
104
+ const paths = this.getVersionPaths(newVersion);
105
+ // Create version directory
106
+ await fs.mkdir(paths.versionDir, { recursive: true });
107
+ // Copy master config to version
108
+ const masterConfigPath = path.join(this.experimentDir, "config.ts");
108
109
  await fs.copyFile(masterConfigPath, paths.configPath);
109
110
  // Generate manifest from config
110
111
  const manifest = await this.generateManifest(paths.configPath);
111
112
  await fs.writeFile(paths.manifestPath, JSON.stringify(manifest, null, 2));
112
113
  // Update metadata
113
- await this.saveMetadata({ currentIteration: newIteration });
114
- this.currentIteration = newIteration;
115
- return newIteration;
114
+ await this.saveMetadata({ currentVersion: newVersion });
115
+ this.currentVersion = newVersion;
116
+ return newVersion;
116
117
  }
117
118
  /**
118
- * Get or create an iteration
119
- * @param createNew - If true, always create a new iteration. Otherwise, use current iteration or create first one if none exists.
119
+ * Get or create a version
120
+ * @param createNew - If true, always create a new version. Otherwise, use current version or create first one if none exists.
120
121
  */
121
- async getOrCreateIteration(createNew) {
122
+ async getOrCreateVersion(createNew) {
122
123
  if (createNew) {
123
- return await this.createNewIteration();
124
+ return await this.createNewVersion();
124
125
  }
125
- const currentIteration = await this.getCurrentIterationNumber();
126
- if (currentIteration < 0) {
127
- return await this.createNewIteration();
126
+ const currentVersion = await this.getCurrentVersionNumber();
127
+ if (currentVersion < 0) {
128
+ return await this.createNewVersion();
128
129
  }
129
- this.currentIteration = currentIteration;
130
- return currentIteration;
130
+ this.currentVersion = currentVersion;
131
+ return currentVersion;
131
132
  }
132
133
  /**
133
- * Load manifest from iteration
134
+ * Load manifest from version
134
135
  */
135
- async loadManifest(iteration) {
136
- const paths = this.getIterationPaths(iteration);
136
+ async loadManifest(version) {
137
+ const paths = this.getVersionPaths(version);
137
138
  const content = await fs.readFile(paths.manifestPath, "utf-8");
138
139
  const data = JSON.parse(content);
139
140
  return ManifestSchema.parse(data);
140
141
  }
141
142
  /**
142
- * Load progress from iteration
143
+ * Load progress from version
143
144
  */
144
- async loadProgress(iteration) {
145
+ async loadProgress(version) {
145
146
  try {
146
- const paths = this.getIterationPaths(iteration);
147
+ const paths = this.getVersionPaths(version);
147
148
  const content = await fs.readFile(paths.progressPath, "utf-8");
148
149
  const data = JSON.parse(content);
149
150
  return ProgressSchema.parse(data);
@@ -153,12 +154,12 @@ export class Evaluator {
153
154
  }
154
155
  }
155
156
  /**
156
- * Save progress to current iteration
157
+ * Save progress to current version
157
158
  */
158
159
  async saveProgress() {
159
- if (!this.runner || this.currentIteration === undefined)
160
+ if (!this.runner || this.currentVersion === undefined)
160
161
  return;
161
- const paths = this.getIterationPaths(this.currentIteration);
162
+ const paths = this.getVersionPaths(this.currentVersion);
162
163
  const progress = {
163
164
  entries: this.runner.getProgressEntries(),
164
165
  lastUpdated: Date.now(),
@@ -183,34 +184,34 @@ export class Evaluator {
183
184
  "--no-warnings",
184
185
  "-e",
185
186
  `console.log(JSON.stringify(require("${configPath}").main()));`,
186
- ], this.config);
187
+ ], {});
187
188
  return JSON.parse(stdout.trim());
188
189
  }
189
190
  /**
190
- * Run the evaluation
191
+ * Run the experiment
191
192
  */
192
193
  async run(options) {
193
- const iteration = await this.getOrCreateIteration(options.new);
194
+ const version = await this.getOrCreateVersion(options.new);
194
195
  // Load manifest
195
- const manifest = await this.loadManifest(iteration);
196
- // We have 2 mandatory tags: "eval-<evaluation-name>" and "eval-<evaluation-name>-iteration-<iteration>"
197
- const evaluationTag = `eval-${this.name}`;
198
- const iterationTag = `${evaluationTag}-iteration-${iteration}`;
199
- const tags = [evaluationTag, iterationTag, ...(manifest.tags ?? [])];
196
+ const manifest = await this.loadManifest(version);
197
+ // We have 2 mandatory tags: "exp-<experiment-name>" and "exp-<experiment-name>-v<version>"
198
+ const experimentTag = `exp-${this.name}`;
199
+ const versionTag = `${experimentTag}-v${version}`;
200
+ const tags = [experimentTag, versionTag, ...(manifest.tags ?? [])];
200
201
  // Create runner
201
- this.runner = new Runner(manifest.runs, this.api, this.config.WEB_URL, {
202
+ this.runner = new Runner(manifest.runs, this.api, this.webUrl, {
202
203
  maxConcurrent: options.maxConcurrent,
203
204
  tags: tags,
204
205
  onStateChange: () => this.onRunStateChange(),
205
206
  });
206
207
  // Restore progress if applicable
207
- const progress = await this.loadProgress(iteration);
208
+ const progress = await this.loadProgress(version);
208
209
  if (progress) {
209
210
  this.runner.restoreProgress(progress.entries);
210
211
  }
211
212
  // Create TUI
212
213
  this.tui = new TUI({
213
- evaluationName: `${this.name} (iteration ${iteration})`,
214
+ experimentName: `${this.name} (v${version})`,
214
215
  onQuit: () => this.handleQuit(),
215
216
  onOpenRun: (index) => this.openRun(index),
216
217
  });
@@ -230,7 +231,7 @@ export class Evaluator {
230
231
  console.log("");
231
232
  }
232
233
  if (options.openMetabase ?? true) {
233
- openInBrowser(`https://daunt-fair.metabaseapp.com/dashboard/10-runs-analysis?tags=${iterationTag}`);
234
+ openInBrowser(`https://daunt-fair.metabaseapp.com/dashboard/10-runs-analysis?run_tags=${versionTag}`);
234
235
  }
235
236
  const errors = this.runner?.getAllStates().filter((state) => state.status === "error");
236
237
  if (errors?.length > 0) {
@@ -254,6 +255,7 @@ export class Evaluator {
254
255
  handleQuit() {
255
256
  this.runner?.stop();
256
257
  this.tui?.stop();
258
+ console.log(pc.yellow(`\nThe experiment has been interrupted. You can resume it later by running "kradle experiment run ${this.name}".`));
257
259
  process.exit(0);
258
260
  }
259
261
  /**
@@ -1,4 +1,4 @@
1
- export { Evaluator } from "./evaluator.js";
1
+ export { Experimenter } from "./experimenter.js";
2
2
  export { Runner } from "./runner.js";
3
3
  export { TUI } from "./tui.js";
4
4
  export * from "./types.js";
@@ -1,4 +1,4 @@
1
- export { Evaluator } from "./evaluator.js";
1
+ export { Experimenter } from "./experimenter.js";
2
2
  export { Runner } from "./runner.js";
3
3
  export { TUI } from "./tui.js";
4
4
  export * from "./types.js";
@@ -157,7 +157,8 @@ export class Runner {
157
157
  const runId = response.runIds[0];
158
158
  this.updateState(index, { runId, status: "running" });
159
159
  // Tag the run with all configured tags
160
- await Promise.all(this.tags.map((tag) => this.api.tagRun(runId, tag)));
160
+ const tags = [...this.tags, ...(state.config.tags ?? [])];
161
+ await Promise.all(tags.map((tag) => this.api.tagRun(runId, tag)));
161
162
  // Poll for completion
162
163
  await this.pollRunStatus(index, runId);
163
164
  }
@@ -1,6 +1,6 @@
1
1
  import type { RunState, StatusCounts } from "./types.js";
2
2
  export interface TUIOptions {
3
- evaluationName: string;
3
+ experimentName: string;
4
4
  onQuit: () => void;
5
5
  onOpenRun: (index: number) => void;
6
6
  }