@kradle/cli 0.0.16 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +62 -65
- package/dist/commands/agent/list.d.ts +4 -0
- package/dist/commands/agent/list.js +6 -4
- package/dist/commands/challenge/build.d.ts +9 -1
- package/dist/commands/challenge/build.js +40 -12
- package/dist/commands/challenge/create.d.ts +5 -1
- package/dist/commands/challenge/create.js +17 -18
- package/dist/commands/challenge/delete.d.ts +4 -1
- package/dist/commands/challenge/delete.js +5 -5
- package/dist/commands/challenge/list.d.ts +5 -0
- package/dist/commands/challenge/list.js +9 -10
- package/dist/commands/challenge/run.d.ts +8 -1
- package/dist/commands/challenge/run.js +13 -8
- package/dist/commands/challenge/watch.d.ts +4 -1
- package/dist/commands/challenge/watch.js +8 -8
- package/dist/commands/{evaluation → experiment}/create.d.ts +4 -0
- package/dist/commands/{evaluation → experiment}/create.js +22 -21
- package/dist/commands/{evaluation → experiment}/list.js +17 -19
- package/dist/commands/{evaluation → experiment}/run.d.ts +4 -1
- package/dist/commands/experiment/run.js +61 -0
- package/dist/commands/init.js +2 -2
- package/dist/lib/api-client.d.ts +29 -10
- package/dist/lib/api-client.js +81 -37
- package/dist/lib/arguments.d.ts +3 -2
- package/dist/lib/arguments.js +5 -3
- package/dist/lib/challenge.d.ts +13 -18
- package/dist/lib/challenge.js +60 -61
- package/dist/lib/experiment/experimenter.d.ts +87 -0
- package/dist/lib/{evaluation/evaluator.js → experiment/experimenter.js} +74 -72
- package/dist/lib/{evaluation → experiment}/index.d.ts +1 -1
- package/dist/lib/{evaluation → experiment}/index.js +1 -1
- package/dist/lib/{evaluation → experiment}/runner.js +2 -1
- package/dist/lib/{evaluation → experiment}/tui.d.ts +1 -1
- package/dist/lib/{evaluation → experiment}/tui.js +3 -3
- package/dist/lib/{evaluation → experiment}/types.d.ts +6 -4
- package/dist/lib/{evaluation → experiment}/types.js +4 -3
- package/dist/lib/flags.d.ts +47 -0
- package/dist/lib/flags.js +63 -0
- package/dist/lib/schemas.d.ts +32 -0
- package/dist/lib/schemas.js +8 -0
- package/dist/lib/utils.d.ts +9 -10
- package/dist/lib/utils.js +12 -12
- package/oclif.manifest.json +342 -64
- package/package.json +5 -6
- package/static/challenge.ts +12 -13
- package/static/experiment_template.ts +114 -0
- package/static/project_template/dev.env +5 -5
- package/static/project_template/prod.env +4 -4
- package/static/project_template/tsconfig.json +1 -1
- package/dist/commands/challenge/multi-upload.d.ts +0 -6
- package/dist/commands/challenge/multi-upload.js +0 -80
- package/dist/commands/evaluation/run.js +0 -61
- package/dist/lib/config.d.ts +0 -12
- package/dist/lib/config.js +0 -49
- package/dist/lib/evaluation/evaluator.d.ts +0 -88
- package/static/evaluation_template.ts +0 -69
- /package/dist/commands/{evaluation → experiment}/list.d.ts +0 -0
- /package/dist/lib/{evaluation → experiment}/runner.d.ts +0 -0
package/dist/lib/challenge.d.ts
CHANGED
|
@@ -1,17 +1,10 @@
|
|
|
1
1
|
import type { ApiClient } from "./api-client.js";
|
|
2
|
-
import type
|
|
3
|
-
import { type ChallengeSchemaType } from "./schemas.js";
|
|
2
|
+
import { type ChallengeConfigSchemaType } from "./schemas.js";
|
|
4
3
|
export declare const SOURCE_FOLDER = ".src";
|
|
5
4
|
export declare class Challenge {
|
|
6
|
-
readonly
|
|
7
|
-
private readonly
|
|
8
|
-
constructor(
|
|
9
|
-
private get kradleChallengesPath();
|
|
10
|
-
/**
|
|
11
|
-
* Get the short slug without username prefix
|
|
12
|
-
* e.g., "florianernst:challenge" -> "challenge"
|
|
13
|
-
*/
|
|
14
|
-
get shortSlug(): string;
|
|
5
|
+
readonly shortSlug: string;
|
|
6
|
+
private readonly kradleChallengesPath;
|
|
7
|
+
constructor(shortSlug: string, kradleChallengesPath: string);
|
|
15
8
|
/**
|
|
16
9
|
* Get the path to the challenge directory
|
|
17
10
|
*/
|
|
@@ -47,22 +40,24 @@ export declare class Challenge {
|
|
|
47
40
|
/**
|
|
48
41
|
* Load the challenge configuration from config.ts
|
|
49
42
|
*/
|
|
50
|
-
loadConfig(): Promise<
|
|
43
|
+
loadConfig(): Promise<ChallengeConfigSchemaType>;
|
|
51
44
|
/**
|
|
52
|
-
*
|
|
45
|
+
* Build the challenge datapack and upload it to the cloud.
|
|
46
|
+
* @param api - The API client to use.
|
|
47
|
+
* @param asPublic - Whether the challenge should be uploaded as public.
|
|
48
|
+
* @returns The config and datapack hash.
|
|
53
49
|
*/
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
config: ChallengeSchemaType;
|
|
50
|
+
buildAndUpload(api: ApiClient, asPublic: boolean): Promise<{
|
|
51
|
+
config: ChallengeConfigSchemaType;
|
|
57
52
|
datapackHash: string;
|
|
58
53
|
}>;
|
|
59
54
|
/**
|
|
60
55
|
* Get all local challenges from the challenges directory
|
|
61
56
|
*/
|
|
62
|
-
static getLocalChallenges(): Promise<
|
|
57
|
+
static getLocalChallenges(): Promise<string[]>;
|
|
63
58
|
/**
|
|
64
59
|
* Create a new local challenge folder with challenge.ts template
|
|
65
60
|
* Note: config.ts is NOT created here - it should be generated later via `challenge config download`
|
|
66
61
|
*/
|
|
67
|
-
static createLocal(slug: string,
|
|
62
|
+
static createLocal(slug: string, kradleChallengesPath: string): Promise<void>;
|
|
68
63
|
}
|
package/dist/lib/challenge.js
CHANGED
|
@@ -4,25 +4,15 @@ import fs from "node:fs/promises";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import pc from "picocolors";
|
|
6
6
|
import * as tar from "tar";
|
|
7
|
-
import {
|
|
7
|
+
import { ChallengeConfigSchema } from "./schemas.js";
|
|
8
8
|
import { executeNodeCommand, executeTypescriptFile, getStaticResourcePath, readDirSorted } from "./utils.js";
|
|
9
9
|
export const SOURCE_FOLDER = ".src";
|
|
10
10
|
export class Challenge {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
constructor(
|
|
14
|
-
this.
|
|
15
|
-
this.
|
|
16
|
-
}
|
|
17
|
-
get kradleChallengesPath() {
|
|
18
|
-
return path.resolve(this.config.KRADLE_CHALLENGES_PATH);
|
|
19
|
-
}
|
|
20
|
-
/**
|
|
21
|
-
* Get the short slug without username prefix
|
|
22
|
-
* e.g., "florianernst:challenge" -> "challenge"
|
|
23
|
-
*/
|
|
24
|
-
get shortSlug() {
|
|
25
|
-
return this.name.includes(":") ? this.name.split(":")[1] : this.name;
|
|
11
|
+
shortSlug;
|
|
12
|
+
kradleChallengesPath;
|
|
13
|
+
constructor(shortSlug, kradleChallengesPath) {
|
|
14
|
+
this.shortSlug = shortSlug;
|
|
15
|
+
this.kradleChallengesPath = path.resolve(kradleChallengesPath);
|
|
26
16
|
}
|
|
27
17
|
/**
|
|
28
18
|
* Get the path to the challenge directory
|
|
@@ -94,15 +84,27 @@ export class Challenge {
|
|
|
94
84
|
throw new Error(`Challenge file not found at ${this.challengePath}`);
|
|
95
85
|
}
|
|
96
86
|
try {
|
|
97
|
-
await executeTypescriptFile(this.challengePath,
|
|
87
|
+
await executeTypescriptFile(this.challengePath, {
|
|
88
|
+
KRADLE_CHALLENGES_PATH: this.kradleChallengesPath,
|
|
89
|
+
NAMESPACE: "kradle",
|
|
90
|
+
}, { silent });
|
|
98
91
|
}
|
|
99
92
|
catch (error) {
|
|
100
93
|
throw new Error(`Failed to build datapack: ${error instanceof Error ? error.message : error}`);
|
|
101
94
|
}
|
|
102
95
|
// @TODO - re-enable once we have a proper build pipeline
|
|
103
96
|
// Copy challenge.ts and config.ts to the src directory
|
|
104
|
-
await fs.cp(path.join(this.challengeDir, "challenge.ts"), path.join(this.
|
|
105
|
-
await fs.cp(path.join(this.challengeDir, "config.ts"), path.join(this.
|
|
97
|
+
await fs.cp(path.join(this.challengeDir, "challenge.ts"), path.join(this.kradleChallengesPath, this.shortSlug, SOURCE_FOLDER, "challenge.ts"));
|
|
98
|
+
await fs.cp(path.join(this.challengeDir, "config.ts"), path.join(this.kradleChallengesPath, this.shortSlug, SOURCE_FOLDER, "config.ts"));
|
|
99
|
+
// Create tarball
|
|
100
|
+
// We need to use the cwd option to make sure the structure of the tarball uses paths relative to the challenge directory
|
|
101
|
+
await tar.create({
|
|
102
|
+
"no-mtime": true,
|
|
103
|
+
file: this.tarballPath, // Output file name
|
|
104
|
+
gzip: true,
|
|
105
|
+
cwd: this.datapackPath, // Input directory
|
|
106
|
+
strict: true,
|
|
107
|
+
}, ["datapack", SOURCE_FOLDER]);
|
|
106
108
|
}
|
|
107
109
|
/**
|
|
108
110
|
* Load the challenge configuration from config.ts
|
|
@@ -118,60 +120,54 @@ export class Challenge {
|
|
|
118
120
|
"--no-warnings",
|
|
119
121
|
"-e",
|
|
120
122
|
`console.log(JSON.stringify(require("${this.configPath}").config));`,
|
|
121
|
-
],
|
|
122
|
-
|
|
123
|
+
], {
|
|
124
|
+
KRADLE_CHALLENGES_PATH: this.kradleChallengesPath,
|
|
125
|
+
NAMESPACE: "kradle",
|
|
126
|
+
});
|
|
127
|
+
return ChallengeConfigSchema.parse(JSON.parse(stdout));
|
|
123
128
|
}
|
|
124
129
|
/**
|
|
125
|
-
*
|
|
130
|
+
* Build the challenge datapack and upload it to the cloud.
|
|
131
|
+
* @param api - The API client to use.
|
|
132
|
+
* @param asPublic - Whether the challenge should be uploaded as public.
|
|
133
|
+
* @returns The config and datapack hash.
|
|
126
134
|
*/
|
|
127
|
-
async
|
|
128
|
-
// Create tarball
|
|
129
|
-
// We need to use the cwd option to make sure the structure of the tarball uses paths relative to the challenge directory
|
|
130
|
-
await tar.create({
|
|
131
|
-
"no-mtime": true,
|
|
132
|
-
file: this.tarballPath, // Output file name
|
|
133
|
-
gzip: true,
|
|
134
|
-
cwd: this.datapackPath, // Input directory
|
|
135
|
-
strict: true,
|
|
136
|
-
}, ["datapack", SOURCE_FOLDER]);
|
|
137
|
-
const uploadUrl = await apiClient.getChallengeUploadUrl(this);
|
|
138
|
-
const fileBuffer = await fs.readFile(this.tarballPath);
|
|
139
|
-
const response = await fetch(uploadUrl, {
|
|
140
|
-
method: "PUT",
|
|
141
|
-
headers: {
|
|
142
|
-
"Content-Type": "application/gzip",
|
|
143
|
-
"Content-Length": fileBuffer.length.toString(),
|
|
144
|
-
},
|
|
145
|
-
body: fileBuffer,
|
|
146
|
-
});
|
|
147
|
-
if (!response.ok) {
|
|
148
|
-
throw new Error(`Failed to upload datapack: ${response.statusText}`);
|
|
149
|
-
}
|
|
150
|
-
}
|
|
151
|
-
async buildAndUpload(api) {
|
|
135
|
+
async buildAndUpload(api, asPublic) {
|
|
152
136
|
// Ensure challenge exists locally
|
|
153
137
|
if (!this.exists()) {
|
|
154
|
-
throw new Error(`Challenge "${this.
|
|
138
|
+
throw new Error(`Challenge "${this.shortSlug}" does not exist locally. Make sure both the challenge.ts file and the config.ts file exist.`);
|
|
155
139
|
}
|
|
156
140
|
// Ensure challenge exists in the cloud
|
|
157
|
-
if (!(await api.challengeExists(this.
|
|
158
|
-
console.log(pc.yellow(`Challenge not found in cloud: ${this.
|
|
159
|
-
console.log(pc.yellow(`Creating challenge: ${this.
|
|
160
|
-
await api.createChallenge(this.
|
|
141
|
+
if (!(await api.challengeExists(this.shortSlug))) {
|
|
142
|
+
console.log(pc.yellow(`Challenge not found in cloud: ${this.shortSlug}`));
|
|
143
|
+
console.log(pc.yellow(`Creating challenge: ${this.shortSlug}`));
|
|
144
|
+
await api.createChallenge(this.shortSlug);
|
|
161
145
|
console.log(pc.green(`✓ Challenge created in cloud`));
|
|
162
146
|
}
|
|
163
|
-
// Load config
|
|
164
147
|
const config = await this.loadConfig();
|
|
148
|
+
// Ensure challenge's visibility is set to private - else, temporarily set it to private
|
|
149
|
+
const cloudChallengeVisibility = (await api.getChallenge(this.shortSlug)).visibility;
|
|
150
|
+
if (cloudChallengeVisibility === "public") {
|
|
151
|
+
console.log(pc.yellow(`⚠️ Cloud challenge is currently public. Setting its visibility to private...`));
|
|
152
|
+
await api.updateChallengeVisibility(this.shortSlug, "private");
|
|
153
|
+
console.log(pc.yellow(`✓ Challenge visibility set to private\n`));
|
|
154
|
+
}
|
|
165
155
|
// Upload config
|
|
166
156
|
console.log(pc.blue(`>> Uploading config: ${this.shortSlug}`));
|
|
167
|
-
|
|
168
|
-
|
|
157
|
+
// We have to set it to private because we cannot update public challenges
|
|
158
|
+
await api.updateChallenge(this.shortSlug, config, "private");
|
|
159
|
+
console.log(pc.green(`✓ Config uploaded\n`));
|
|
169
160
|
// Build and upload datapack
|
|
170
161
|
console.log(pc.blue(`>> Building datapack: ${this.shortSlug}`));
|
|
171
162
|
await this.build();
|
|
172
163
|
console.log(pc.blue(`>> Uploading datapack: ${this.shortSlug}`));
|
|
173
|
-
await this.
|
|
174
|
-
console.log(pc.green(`✓
|
|
164
|
+
await api.uploadChallengeDatapack(this.shortSlug, this.tarballPath);
|
|
165
|
+
console.log(pc.green(`✓ Build & upload complete for ${this.shortSlug}\n`));
|
|
166
|
+
if (asPublic) {
|
|
167
|
+
console.log(pc.green(`👀 Setting challenge visibility to public...`));
|
|
168
|
+
await api.updateChallengeVisibility(this.shortSlug, "public");
|
|
169
|
+
console.log(pc.green(`✓ Challenge visibility set to public\n`));
|
|
170
|
+
}
|
|
175
171
|
const datapackHash = await this.getDatapackHash();
|
|
176
172
|
return { config, datapackHash };
|
|
177
173
|
}
|
|
@@ -180,14 +176,17 @@ export class Challenge {
|
|
|
180
176
|
*/
|
|
181
177
|
static async getLocalChallenges() {
|
|
182
178
|
const challengesDir = path.resolve(process.cwd(), "challenges");
|
|
179
|
+
if (!existsSync(challengesDir)) {
|
|
180
|
+
return [];
|
|
181
|
+
}
|
|
183
182
|
const entries = await readDirSorted(challengesDir);
|
|
184
|
-
const challenges =
|
|
183
|
+
const challenges = [];
|
|
185
184
|
for (const entry of entries) {
|
|
186
185
|
if (entry.isDirectory()) {
|
|
187
186
|
// Check if it has a challenge.ts file
|
|
188
187
|
const challengePath = path.join(challengesDir, entry.name, "challenge.ts");
|
|
189
188
|
if (existsSync(challengePath)) {
|
|
190
|
-
challenges
|
|
189
|
+
challenges.push(entry.name);
|
|
191
190
|
}
|
|
192
191
|
}
|
|
193
192
|
}
|
|
@@ -197,8 +196,8 @@ export class Challenge {
|
|
|
197
196
|
* Create a new local challenge folder with challenge.ts template
|
|
198
197
|
* Note: config.ts is NOT created here - it should be generated later via `challenge config download`
|
|
199
198
|
*/
|
|
200
|
-
static async createLocal(slug,
|
|
201
|
-
const challenge = new Challenge(slug,
|
|
199
|
+
static async createLocal(slug, kradleChallengesPath) {
|
|
200
|
+
const challenge = new Challenge(slug, kradleChallengesPath);
|
|
202
201
|
const challengeTemplatePath = getStaticResourcePath("challenge.ts");
|
|
203
202
|
// Create challenge directory
|
|
204
203
|
await fs.mkdir(challenge.challengeDir, { recursive: true });
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import type { ApiClient } from "../api-client.js";
|
|
2
|
+
import type { ExperimentMetadata, ExperimentOptions, Manifest, Progress } from "./types.js";
|
|
3
|
+
export declare class Experimenter {
|
|
4
|
+
private name;
|
|
5
|
+
private webUrl;
|
|
6
|
+
private api;
|
|
7
|
+
experimentDir: string;
|
|
8
|
+
metadataPath: string;
|
|
9
|
+
private runner?;
|
|
10
|
+
private tui?;
|
|
11
|
+
private currentVersion?;
|
|
12
|
+
constructor(name: string, webUrl: string, api: ApiClient);
|
|
13
|
+
/**
|
|
14
|
+
* Get paths for a specific version
|
|
15
|
+
*/
|
|
16
|
+
private getVersionPaths;
|
|
17
|
+
get configPath(): string;
|
|
18
|
+
/**
|
|
19
|
+
* Get the current version directory path
|
|
20
|
+
*/
|
|
21
|
+
getCurrentVersionDir(): string;
|
|
22
|
+
/**
|
|
23
|
+
* Check if experiment exists
|
|
24
|
+
*/
|
|
25
|
+
exists(): Promise<boolean>;
|
|
26
|
+
/**
|
|
27
|
+
* Check if config.ts exists (master config)
|
|
28
|
+
*/
|
|
29
|
+
configExists(): Promise<boolean>;
|
|
30
|
+
/**
|
|
31
|
+
* Load experiment metadata
|
|
32
|
+
*/
|
|
33
|
+
loadMetadata(): Promise<ExperimentMetadata | null>;
|
|
34
|
+
/**
|
|
35
|
+
* Save experiment metadata
|
|
36
|
+
*/
|
|
37
|
+
saveMetadata(metadata: ExperimentMetadata): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Get the current version number, or -1 if none exists
|
|
40
|
+
*/
|
|
41
|
+
getCurrentVersionNumber(): Promise<number>;
|
|
42
|
+
/**
|
|
43
|
+
* Create a new version
|
|
44
|
+
*/
|
|
45
|
+
createNewVersion(): Promise<number>;
|
|
46
|
+
/**
|
|
47
|
+
* Get or create a version
|
|
48
|
+
* @param createNew - If true, always create a new version. Otherwise, use current version or create first one if none exists.
|
|
49
|
+
*/
|
|
50
|
+
getOrCreateVersion(createNew: boolean): Promise<number>;
|
|
51
|
+
/**
|
|
52
|
+
* Load manifest from version
|
|
53
|
+
*/
|
|
54
|
+
loadManifest(version: number): Promise<Manifest>;
|
|
55
|
+
/**
|
|
56
|
+
* Load progress from version
|
|
57
|
+
*/
|
|
58
|
+
loadProgress(version: number): Promise<Progress | null>;
|
|
59
|
+
/**
|
|
60
|
+
* Save progress to current version
|
|
61
|
+
*/
|
|
62
|
+
saveProgress(): Promise<void>;
|
|
63
|
+
/**
|
|
64
|
+
* Execute config.ts to generate manifest
|
|
65
|
+
*/
|
|
66
|
+
generateManifest(configPath: string): Promise<Manifest>;
|
|
67
|
+
/**
|
|
68
|
+
* Execute config.ts file and return the manifest
|
|
69
|
+
*/
|
|
70
|
+
private executeConfigFile;
|
|
71
|
+
/**
|
|
72
|
+
* Run the experiment
|
|
73
|
+
*/
|
|
74
|
+
run(options: ExperimentOptions): Promise<void>;
|
|
75
|
+
/**
|
|
76
|
+
* Handle state change from runner
|
|
77
|
+
*/
|
|
78
|
+
private onRunStateChange;
|
|
79
|
+
/**
|
|
80
|
+
* Handle quit request
|
|
81
|
+
*/
|
|
82
|
+
private handleQuit;
|
|
83
|
+
/**
|
|
84
|
+
* Open run in browser
|
|
85
|
+
*/
|
|
86
|
+
private openRun;
|
|
87
|
+
}
|
|
@@ -1,55 +1,56 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import pc from "picocolors";
|
|
3
4
|
import { executeNodeCommand, openInBrowser } from "../utils.js";
|
|
4
5
|
import { Runner } from "./runner.js";
|
|
5
6
|
import { TUI } from "./tui.js";
|
|
6
|
-
import {
|
|
7
|
-
export class
|
|
7
|
+
import { ExperimentMetadataSchema, ManifestSchema, ProgressSchema } from "./types.js";
|
|
8
|
+
export class Experimenter {
|
|
8
9
|
name;
|
|
9
|
-
|
|
10
|
+
webUrl;
|
|
10
11
|
api;
|
|
11
|
-
|
|
12
|
+
experimentDir;
|
|
12
13
|
metadataPath;
|
|
13
14
|
runner;
|
|
14
15
|
tui;
|
|
15
|
-
|
|
16
|
-
constructor(name,
|
|
16
|
+
currentVersion;
|
|
17
|
+
constructor(name, webUrl, api) {
|
|
17
18
|
this.name = name;
|
|
18
|
-
this.
|
|
19
|
+
this.webUrl = webUrl;
|
|
19
20
|
this.api = api;
|
|
20
|
-
this.
|
|
21
|
-
this.metadataPath = path.join(this.
|
|
21
|
+
this.experimentDir = path.resolve(process.cwd(), "experiments", name);
|
|
22
|
+
this.metadataPath = path.join(this.experimentDir, ".experiment.json");
|
|
22
23
|
}
|
|
23
24
|
/**
|
|
24
|
-
* Get paths for a specific
|
|
25
|
+
* Get paths for a specific version
|
|
25
26
|
*/
|
|
26
|
-
|
|
27
|
-
const
|
|
27
|
+
getVersionPaths(version) {
|
|
28
|
+
const versionDir = path.join(this.experimentDir, "versions", version.toString().padStart(3, "0"));
|
|
28
29
|
return {
|
|
29
|
-
|
|
30
|
-
configPath: path.join(
|
|
31
|
-
manifestPath: path.join(
|
|
32
|
-
progressPath: path.join(
|
|
30
|
+
versionDir,
|
|
31
|
+
configPath: path.join(versionDir, "config.ts"),
|
|
32
|
+
manifestPath: path.join(versionDir, "manifest.json"),
|
|
33
|
+
progressPath: path.join(versionDir, "progress.json"),
|
|
33
34
|
};
|
|
34
35
|
}
|
|
35
36
|
get configPath() {
|
|
36
|
-
return path.join(this.
|
|
37
|
+
return path.join(this.experimentDir, "config.ts");
|
|
37
38
|
}
|
|
38
39
|
/**
|
|
39
|
-
* Get the current
|
|
40
|
+
* Get the current version directory path
|
|
40
41
|
*/
|
|
41
|
-
|
|
42
|
-
if (this.
|
|
43
|
-
throw new Error("No
|
|
42
|
+
getCurrentVersionDir() {
|
|
43
|
+
if (this.currentVersion === undefined) {
|
|
44
|
+
throw new Error("No version set");
|
|
44
45
|
}
|
|
45
|
-
return this.
|
|
46
|
+
return this.getVersionPaths(this.currentVersion).versionDir;
|
|
46
47
|
}
|
|
47
48
|
/**
|
|
48
|
-
* Check if
|
|
49
|
+
* Check if experiment exists
|
|
49
50
|
*/
|
|
50
51
|
async exists() {
|
|
51
52
|
try {
|
|
52
|
-
await fs.access(this.
|
|
53
|
+
await fs.access(this.experimentDir);
|
|
53
54
|
return true;
|
|
54
55
|
}
|
|
55
56
|
catch {
|
|
@@ -69,81 +70,81 @@ export class Evaluator {
|
|
|
69
70
|
}
|
|
70
71
|
}
|
|
71
72
|
/**
|
|
72
|
-
* Load
|
|
73
|
+
* Load experiment metadata
|
|
73
74
|
*/
|
|
74
75
|
async loadMetadata() {
|
|
75
76
|
try {
|
|
76
77
|
const content = await fs.readFile(this.metadataPath, "utf-8");
|
|
77
78
|
const data = JSON.parse(content);
|
|
78
|
-
return
|
|
79
|
+
return ExperimentMetadataSchema.parse(data);
|
|
79
80
|
}
|
|
80
81
|
catch {
|
|
81
82
|
return null;
|
|
82
83
|
}
|
|
83
84
|
}
|
|
84
85
|
/**
|
|
85
|
-
* Save
|
|
86
|
+
* Save experiment metadata
|
|
86
87
|
*/
|
|
87
88
|
async saveMetadata(metadata) {
|
|
88
89
|
await fs.writeFile(this.metadataPath, JSON.stringify(metadata, null, 2));
|
|
89
90
|
}
|
|
90
91
|
/**
|
|
91
|
-
* Get the current
|
|
92
|
+
* Get the current version number, or -1 if none exists
|
|
92
93
|
*/
|
|
93
|
-
async
|
|
94
|
+
async getCurrentVersionNumber() {
|
|
94
95
|
const metadata = await this.loadMetadata();
|
|
95
|
-
return metadata?.
|
|
96
|
+
return metadata?.currentVersion ?? -1;
|
|
96
97
|
}
|
|
97
98
|
/**
|
|
98
|
-
* Create a new
|
|
99
|
+
* Create a new version
|
|
99
100
|
*/
|
|
100
|
-
async
|
|
101
|
-
const
|
|
102
|
-
const
|
|
103
|
-
const paths = this.
|
|
104
|
-
// Create
|
|
105
|
-
await fs.mkdir(paths.
|
|
106
|
-
// Copy master config to
|
|
107
|
-
const masterConfigPath = path.join(this.
|
|
101
|
+
async createNewVersion() {
|
|
102
|
+
const currentVersion = await this.getCurrentVersionNumber();
|
|
103
|
+
const newVersion = currentVersion + 1;
|
|
104
|
+
const paths = this.getVersionPaths(newVersion);
|
|
105
|
+
// Create version directory
|
|
106
|
+
await fs.mkdir(paths.versionDir, { recursive: true });
|
|
107
|
+
// Copy master config to version
|
|
108
|
+
const masterConfigPath = path.join(this.experimentDir, "config.ts");
|
|
108
109
|
await fs.copyFile(masterConfigPath, paths.configPath);
|
|
109
110
|
// Generate manifest from config
|
|
110
111
|
const manifest = await this.generateManifest(paths.configPath);
|
|
111
112
|
await fs.writeFile(paths.manifestPath, JSON.stringify(manifest, null, 2));
|
|
112
113
|
// Update metadata
|
|
113
|
-
await this.saveMetadata({
|
|
114
|
-
this.
|
|
115
|
-
return
|
|
114
|
+
await this.saveMetadata({ currentVersion: newVersion });
|
|
115
|
+
this.currentVersion = newVersion;
|
|
116
|
+
return newVersion;
|
|
116
117
|
}
|
|
117
118
|
/**
|
|
118
|
-
* Get or create
|
|
119
|
-
* @param createNew - If true, always create a new
|
|
119
|
+
* Get or create a version
|
|
120
|
+
* @param createNew - If true, always create a new version. Otherwise, use current version or create first one if none exists.
|
|
120
121
|
*/
|
|
121
|
-
async
|
|
122
|
+
async getOrCreateVersion(createNew) {
|
|
122
123
|
if (createNew) {
|
|
123
|
-
return await this.
|
|
124
|
+
return await this.createNewVersion();
|
|
124
125
|
}
|
|
125
|
-
const
|
|
126
|
-
if (
|
|
127
|
-
return await this.
|
|
126
|
+
const currentVersion = await this.getCurrentVersionNumber();
|
|
127
|
+
if (currentVersion < 0) {
|
|
128
|
+
return await this.createNewVersion();
|
|
128
129
|
}
|
|
129
|
-
this.
|
|
130
|
-
return
|
|
130
|
+
this.currentVersion = currentVersion;
|
|
131
|
+
return currentVersion;
|
|
131
132
|
}
|
|
132
133
|
/**
|
|
133
|
-
* Load manifest from
|
|
134
|
+
* Load manifest from version
|
|
134
135
|
*/
|
|
135
|
-
async loadManifest(
|
|
136
|
-
const paths = this.
|
|
136
|
+
async loadManifest(version) {
|
|
137
|
+
const paths = this.getVersionPaths(version);
|
|
137
138
|
const content = await fs.readFile(paths.manifestPath, "utf-8");
|
|
138
139
|
const data = JSON.parse(content);
|
|
139
140
|
return ManifestSchema.parse(data);
|
|
140
141
|
}
|
|
141
142
|
/**
|
|
142
|
-
* Load progress from
|
|
143
|
+
* Load progress from version
|
|
143
144
|
*/
|
|
144
|
-
async loadProgress(
|
|
145
|
+
async loadProgress(version) {
|
|
145
146
|
try {
|
|
146
|
-
const paths = this.
|
|
147
|
+
const paths = this.getVersionPaths(version);
|
|
147
148
|
const content = await fs.readFile(paths.progressPath, "utf-8");
|
|
148
149
|
const data = JSON.parse(content);
|
|
149
150
|
return ProgressSchema.parse(data);
|
|
@@ -153,12 +154,12 @@ export class Evaluator {
|
|
|
153
154
|
}
|
|
154
155
|
}
|
|
155
156
|
/**
|
|
156
|
-
* Save progress to current
|
|
157
|
+
* Save progress to current version
|
|
157
158
|
*/
|
|
158
159
|
async saveProgress() {
|
|
159
|
-
if (!this.runner || this.
|
|
160
|
+
if (!this.runner || this.currentVersion === undefined)
|
|
160
161
|
return;
|
|
161
|
-
const paths = this.
|
|
162
|
+
const paths = this.getVersionPaths(this.currentVersion);
|
|
162
163
|
const progress = {
|
|
163
164
|
entries: this.runner.getProgressEntries(),
|
|
164
165
|
lastUpdated: Date.now(),
|
|
@@ -183,34 +184,34 @@ export class Evaluator {
|
|
|
183
184
|
"--no-warnings",
|
|
184
185
|
"-e",
|
|
185
186
|
`console.log(JSON.stringify(require("${configPath}").main()));`,
|
|
186
|
-
],
|
|
187
|
+
], {});
|
|
187
188
|
return JSON.parse(stdout.trim());
|
|
188
189
|
}
|
|
189
190
|
/**
|
|
190
|
-
* Run the
|
|
191
|
+
* Run the experiment
|
|
191
192
|
*/
|
|
192
193
|
async run(options) {
|
|
193
|
-
const
|
|
194
|
+
const version = await this.getOrCreateVersion(options.new);
|
|
194
195
|
// Load manifest
|
|
195
|
-
const manifest = await this.loadManifest(
|
|
196
|
-
// We have 2 mandatory tags: "
|
|
197
|
-
const
|
|
198
|
-
const
|
|
199
|
-
const tags = [
|
|
196
|
+
const manifest = await this.loadManifest(version);
|
|
197
|
+
// We have 2 mandatory tags: "exp-<experiment-name>" and "exp-<experiment-name>-v<version>"
|
|
198
|
+
const experimentTag = `exp-${this.name}`;
|
|
199
|
+
const versionTag = `${experimentTag}-v${version}`;
|
|
200
|
+
const tags = [experimentTag, versionTag, ...(manifest.tags ?? [])];
|
|
200
201
|
// Create runner
|
|
201
|
-
this.runner = new Runner(manifest.runs, this.api, this.
|
|
202
|
+
this.runner = new Runner(manifest.runs, this.api, this.webUrl, {
|
|
202
203
|
maxConcurrent: options.maxConcurrent,
|
|
203
204
|
tags: tags,
|
|
204
205
|
onStateChange: () => this.onRunStateChange(),
|
|
205
206
|
});
|
|
206
207
|
// Restore progress if applicable
|
|
207
|
-
const progress = await this.loadProgress(
|
|
208
|
+
const progress = await this.loadProgress(version);
|
|
208
209
|
if (progress) {
|
|
209
210
|
this.runner.restoreProgress(progress.entries);
|
|
210
211
|
}
|
|
211
212
|
// Create TUI
|
|
212
213
|
this.tui = new TUI({
|
|
213
|
-
|
|
214
|
+
experimentName: `${this.name} (v${version})`,
|
|
214
215
|
onQuit: () => this.handleQuit(),
|
|
215
216
|
onOpenRun: (index) => this.openRun(index),
|
|
216
217
|
});
|
|
@@ -230,7 +231,7 @@ export class Evaluator {
|
|
|
230
231
|
console.log("");
|
|
231
232
|
}
|
|
232
233
|
if (options.openMetabase ?? true) {
|
|
233
|
-
openInBrowser(`https://daunt-fair.metabaseapp.com/dashboard/10-runs-analysis?
|
|
234
|
+
openInBrowser(`https://daunt-fair.metabaseapp.com/dashboard/10-runs-analysis?run_tags=${versionTag}`);
|
|
234
235
|
}
|
|
235
236
|
const errors = this.runner?.getAllStates().filter((state) => state.status === "error");
|
|
236
237
|
if (errors?.length > 0) {
|
|
@@ -254,6 +255,7 @@ export class Evaluator {
|
|
|
254
255
|
handleQuit() {
|
|
255
256
|
this.runner?.stop();
|
|
256
257
|
this.tui?.stop();
|
|
258
|
+
console.log(pc.yellow(`\nThe experiment has been interrupted. You can resume it later by running "kradle experiment run ${this.name}".`));
|
|
257
259
|
process.exit(0);
|
|
258
260
|
}
|
|
259
261
|
/**
|
|
@@ -157,7 +157,8 @@ export class Runner {
|
|
|
157
157
|
const runId = response.runIds[0];
|
|
158
158
|
this.updateState(index, { runId, status: "running" });
|
|
159
159
|
// Tag the run with all configured tags
|
|
160
|
-
|
|
160
|
+
const tags = [...this.tags, ...(state.config.tags ?? [])];
|
|
161
|
+
await Promise.all(tags.map((tag) => this.api.tagRun(runId, tag)));
|
|
161
162
|
// Poll for completion
|
|
162
163
|
await this.pollRunStatus(index, runId);
|
|
163
164
|
}
|