kradle 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -3
- package/dist/commands/challenge/list.js +4 -3
- package/dist/commands/challenge/pull.js +6 -60
- package/dist/commands/challenge/watch.js +6 -1
- package/dist/commands/experiment/logs.d.ts +18 -0
- package/dist/commands/experiment/logs.js +220 -0
- package/dist/commands/experiment/run.d.ts +1 -0
- package/dist/commands/experiment/run.js +6 -0
- package/dist/commands/world/list.js +3 -2
- package/dist/lib/api-client.d.ts +13 -1
- package/dist/lib/api-client.js +36 -2
- package/dist/lib/challenge.d.ts +8 -0
- package/dist/lib/challenge.js +46 -0
- package/dist/lib/experiment/experimenter.d.ts +9 -0
- package/dist/lib/experiment/experimenter.js +71 -5
- package/dist/lib/experiment/types.d.ts +1 -22
- package/dist/lib/experiment/types.js +0 -18
- package/dist/lib/schemas.d.ts +59 -0
- package/dist/lib/schemas.js +34 -0
- package/oclif.manifest.json +137 -57
- package/package.json +1 -1
- package/static/ai_docs/LLM_CLI_REFERENCE.md +78 -1
package/README.md
CHANGED
|
@@ -18,18 +18,19 @@ Kradle's CLI for managing Minecraft challenges, experiments, agents, and more!
|
|
|
18
18
|
|
|
19
19
|
## Installation
|
|
20
20
|
|
|
21
|
+
Make sure you have [NodeJS 22.18 or higher](https://nodejs.org/en/download/current) installed.
|
|
22
|
+
|
|
21
23
|
1. Install Kradle's CLI globally
|
|
22
24
|
```
|
|
23
25
|
npm i -g kradle
|
|
24
26
|
```
|
|
25
|
-
2. Initialize a new directory to store challenges and
|
|
27
|
+
2. Initialize a new directory to store challenges, and other Kradle resources:
|
|
26
28
|
```
|
|
27
29
|
kradle init
|
|
28
30
|
```
|
|
29
|
-
3. Congrats 🎉 You can now create a new challenge
|
|
31
|
+
3. Congrats 🎉 You can now create a new challenge:
|
|
30
32
|
```
|
|
31
33
|
kradle challenge create <challenge-name>
|
|
32
|
-
kradle experiment create <experiment-name>
|
|
33
34
|
```
|
|
34
35
|
|
|
35
36
|
In addition, you can enable [autocomplete](#Autocomplete).
|
|
@@ -184,6 +185,7 @@ kradle experiment run <name> # Resume current version or c
|
|
|
184
185
|
kradle experiment run <name> --new-version # Start a new version
|
|
185
186
|
kradle experiment run <name> --max-concurrent 10 # Control parallelism (default: 5)
|
|
186
187
|
kradle experiment run <name> --download-recordings # Auto-download recordings as runs complete
|
|
188
|
+
kradle experiment run <name> --download-logs # Auto-download logs as runs complete
|
|
187
189
|
```
|
|
188
190
|
|
|
189
191
|
The run command:
|
|
@@ -207,6 +209,21 @@ kradle experiment recordings <name> <run-id> --all # Download all participants
|
|
|
207
209
|
|
|
208
210
|
Recordings are saved to `experiments/<name>/versions/<version>/recordings/<run-id>/`.
|
|
209
211
|
|
|
212
|
+
### Download Logs
|
|
213
|
+
|
|
214
|
+
Download logs and run results from completed experiment runs:
|
|
215
|
+
|
|
216
|
+
```bash
|
|
217
|
+
kradle experiment logs <name> # Interactive selection of run
|
|
218
|
+
kradle experiment logs <name> <run-id> # Download specific run
|
|
219
|
+
kradle experiment logs <name> --all # Download all runs
|
|
220
|
+
kradle experiment logs <name> --version 2 # Download from specific version
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
Files are saved to `experiments/<name>/versions/<version>/logs/<run-id>/`:
|
|
224
|
+
- `run.json` - Run result with status, end_state, and participant results
|
|
225
|
+
- `logs.json` - Log entries from the run
|
|
226
|
+
|
|
210
227
|
### List Experiments
|
|
211
228
|
|
|
212
229
|
List all local experiments:
|
|
@@ -13,13 +13,14 @@ export default class List extends Command {
|
|
|
13
13
|
const { flags } = await this.parse(List);
|
|
14
14
|
const api = new ApiClient(flags["api-url"], flags["api-key"]);
|
|
15
15
|
this.log(pc.blue(">> Loading challenges..."));
|
|
16
|
-
const [cloudChallenges, localChallenges, human] = await Promise.all([
|
|
16
|
+
const [cloudChallenges, kradleChallenges, localChallenges, human] = await Promise.all([
|
|
17
17
|
api.listChallenges(),
|
|
18
|
+
api.listKradleChallenges(),
|
|
18
19
|
Challenge.getLocalChallenges(),
|
|
19
20
|
api.getHuman(),
|
|
20
21
|
]);
|
|
21
|
-
// Create a map for easy lookup
|
|
22
|
-
const cloudMap = new Map(cloudChallenges.map((c) => [c.slug, c]));
|
|
22
|
+
// Create a map for easy lookup (user's challenges + team-kradle challenges)
|
|
23
|
+
const cloudMap = new Map([...cloudChallenges, ...kradleChallenges].map((c) => [c.slug, c]));
|
|
23
24
|
const allSlugs = new Set([...cloudMap.keys(), ...localChallenges.map((id) => `${human.username}:${id}`)]);
|
|
24
25
|
this.log(pc.bold("\nChallenges:\n"));
|
|
25
26
|
this.log(`${"Status".padEnd(15)} ${"Slug".padEnd(40)} ${"Name".padEnd(30)}`);
|
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
|
-
import path from "node:path";
|
|
4
3
|
import { Command, Flags } from "@oclif/core";
|
|
5
4
|
import enquirer from "enquirer";
|
|
6
5
|
import { Listr } from "listr2";
|
|
7
6
|
import pc from "picocolors";
|
|
8
|
-
import * as tar from "tar";
|
|
9
7
|
import { ApiClient } from "../../lib/api-client.js";
|
|
10
8
|
import { extractShortSlug, getChallengeSlugArgument } from "../../lib/arguments.js";
|
|
11
|
-
import { Challenge
|
|
9
|
+
import { Challenge } from "../../lib/challenge.js";
|
|
12
10
|
import { getConfigFlags } from "../../lib/flags.js";
|
|
13
11
|
export default class Pull extends Command {
|
|
14
12
|
static description = "Pull a challenge from the cloud and extract source files locally";
|
|
@@ -97,67 +95,16 @@ export default class Pull extends Command {
|
|
|
97
95
|
return;
|
|
98
96
|
}
|
|
99
97
|
}
|
|
100
|
-
const tempTarballPath = path.join(flags["challenges-path"], `${challenge.shortSlug}-pull-temp.tar.gz`);
|
|
101
98
|
const tasks = new Listr([
|
|
102
99
|
{
|
|
103
|
-
title: "Downloading
|
|
104
|
-
task: async (_, task) => {
|
|
105
|
-
const { downloadUrl } = await api.getChallengeDownloadUrl(challengeSlug);
|
|
106
|
-
const response = await fetch(downloadUrl);
|
|
107
|
-
if (!response.ok) {
|
|
108
|
-
throw new Error(`Failed to download: ${response.status} ${response.statusText}`);
|
|
109
|
-
}
|
|
110
|
-
const buffer = await response.arrayBuffer();
|
|
111
|
-
await fs.mkdir(path.dirname(tempTarballPath), { recursive: true });
|
|
112
|
-
await fs.writeFile(tempTarballPath, Buffer.from(buffer));
|
|
113
|
-
task.title = "Downloaded challenge";
|
|
114
|
-
},
|
|
115
|
-
},
|
|
116
|
-
{
|
|
117
|
-
title: "Creating challenge directory",
|
|
100
|
+
title: "Downloading and extracting source files",
|
|
118
101
|
task: async (_, task) => {
|
|
119
102
|
await fs.mkdir(challenge.challengeDir, { recursive: true });
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
{
|
|
124
|
-
title: "Extracting source files",
|
|
125
|
-
task: async (_, task) => {
|
|
126
|
-
const filesToExtract = [`${SOURCE_FOLDER}/challenge.ts`, `${SOURCE_FOLDER}/config.ts`];
|
|
127
|
-
const tempExtractDir = path.join(flags["challenges-path"], `${challenge.shortSlug}-extract-temp`);
|
|
128
|
-
await fs.mkdir(tempExtractDir, { recursive: true });
|
|
129
|
-
try {
|
|
130
|
-
await tar.extract({
|
|
131
|
-
file: tempTarballPath,
|
|
132
|
-
cwd: tempExtractDir,
|
|
133
|
-
filter: (entryPath) => filesToExtract.some((f) => entryPath === f),
|
|
134
|
-
});
|
|
135
|
-
const srcChallengeTs = path.join(tempExtractDir, SOURCE_FOLDER, "challenge.ts");
|
|
136
|
-
const srcConfigTs = path.join(tempExtractDir, SOURCE_FOLDER, "config.ts");
|
|
137
|
-
let extractedCount = 0;
|
|
138
|
-
if (existsSync(srcChallengeTs)) {
|
|
139
|
-
await fs.copyFile(srcChallengeTs, challenge.challengePath);
|
|
140
|
-
extractedCount++;
|
|
141
|
-
}
|
|
142
|
-
if (existsSync(srcConfigTs)) {
|
|
143
|
-
await fs.copyFile(srcConfigTs, challenge.configPath);
|
|
144
|
-
extractedCount++;
|
|
145
|
-
}
|
|
146
|
-
if (extractedCount === 0) {
|
|
147
|
-
throw new Error(`No source files found in tarball. The challenge may not have been built with source files.`);
|
|
148
|
-
}
|
|
149
|
-
task.title = `Extracted ${extractedCount} source file(s)`;
|
|
103
|
+
const extractedCount = await Challenge.downloadSourceFiles(api, challengeSlug, challenge.challengeDir);
|
|
104
|
+
if (extractedCount === 0) {
|
|
105
|
+
throw new Error("No source files found in tarball. The challenge may not have been built with source files.");
|
|
150
106
|
}
|
|
151
|
-
|
|
152
|
-
await fs.rm(tempExtractDir, { recursive: true, force: true });
|
|
153
|
-
}
|
|
154
|
-
},
|
|
155
|
-
},
|
|
156
|
-
{
|
|
157
|
-
title: "Cleaning up",
|
|
158
|
-
task: async (_, task) => {
|
|
159
|
-
await fs.rm(tempTarballPath, { force: true });
|
|
160
|
-
task.title = "Cleaned up temporary files";
|
|
107
|
+
task.title = `Extracted ${extractedCount} source file(s)`;
|
|
161
108
|
},
|
|
162
109
|
},
|
|
163
110
|
{
|
|
@@ -175,7 +122,6 @@ export default class Pull extends Command {
|
|
|
175
122
|
this.log(pc.dim(` → config.ts: ${challenge.configPath}`));
|
|
176
123
|
}
|
|
177
124
|
catch (error) {
|
|
178
|
-
await fs.rm(tempTarballPath, { force: true }).catch(() => { });
|
|
179
125
|
this.error(pc.red(`Pull failed: ${error instanceof Error ? error.message : String(error)}`));
|
|
180
126
|
}
|
|
181
127
|
}
|
|
@@ -33,6 +33,9 @@ export default class Watch extends Command {
|
|
|
33
33
|
// Clear screen before subsequent rebuilds for clean output
|
|
34
34
|
clearScreen();
|
|
35
35
|
this.log(pc.cyan(`\n Rebuild started...\n`));
|
|
36
|
+
// Track if config changed so we can force tarball re-upload
|
|
37
|
+
// (tarball includes source files which contain config.ts)
|
|
38
|
+
let configChanged = false;
|
|
36
39
|
const tasks = new Listr([
|
|
37
40
|
{
|
|
38
41
|
title: "Checking configuration",
|
|
@@ -42,6 +45,7 @@ export default class Watch extends Command {
|
|
|
42
45
|
task.title = "Uploading configuration";
|
|
43
46
|
await api.updateChallenge(challenge.shortSlug, newConfig, "private");
|
|
44
47
|
lastConfig = newConfig;
|
|
48
|
+
configChanged = true;
|
|
45
49
|
task.title = "Configuration uploaded";
|
|
46
50
|
}
|
|
47
51
|
else {
|
|
@@ -61,7 +65,8 @@ export default class Watch extends Command {
|
|
|
61
65
|
title: "Checking datapack changes",
|
|
62
66
|
task: async (_ctx, task) => {
|
|
63
67
|
const newHash = await challenge.getDatapackHash();
|
|
64
|
-
if (
|
|
68
|
+
// Upload if datapack changed OR if config changed (since tarball includes source files)
|
|
69
|
+
if (newHash !== lastHash || configChanged) {
|
|
65
70
|
task.title = "Uploading datapack";
|
|
66
71
|
await api.uploadChallengeDatapack(challenge.shortSlug, challenge.tarballPath);
|
|
67
72
|
lastHash = newHash;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Command } from "@oclif/core";
|
|
2
|
+
export default class Logs extends Command {
|
|
3
|
+
static description: string;
|
|
4
|
+
static examples: string[];
|
|
5
|
+
static args: {
|
|
6
|
+
experimentName: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
|
|
7
|
+
runId: import("@oclif/core/interfaces").Arg<string | undefined, Record<string, unknown>>;
|
|
8
|
+
};
|
|
9
|
+
static flags: {
|
|
10
|
+
"api-key": import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
11
|
+
"api-url": import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
|
|
12
|
+
all: import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
13
|
+
version: import("@oclif/core/interfaces").OptionFlag<number | undefined, import("@oclif/core/interfaces").CustomOptions>;
|
|
14
|
+
};
|
|
15
|
+
run(): Promise<void>;
|
|
16
|
+
private downloadForExperiment;
|
|
17
|
+
private downloadLogs;
|
|
18
|
+
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { Args, Command, Flags } from "@oclif/core";
|
|
4
|
+
import enquirer from "enquirer";
|
|
5
|
+
import { Listr } from "listr2";
|
|
6
|
+
import pc from "picocolors";
|
|
7
|
+
import { ApiClient } from "../../lib/api-client.js";
|
|
8
|
+
import { Experimenter } from "../../lib/experiment/experimenter.js";
|
|
9
|
+
import { getConfigFlags } from "../../lib/flags.js";
|
|
10
|
+
// Get all versions for an experiment
|
|
11
|
+
async function getAllVersions(experimentDir) {
|
|
12
|
+
const versionsDir = path.join(experimentDir, "versions");
|
|
13
|
+
try {
|
|
14
|
+
const entries = await fs.readdir(versionsDir, { withFileTypes: true });
|
|
15
|
+
return entries
|
|
16
|
+
.filter((e) => e.isDirectory())
|
|
17
|
+
.map((e) => parseInt(e.name, 10))
|
|
18
|
+
.filter((n) => !Number.isNaN(n))
|
|
19
|
+
.sort((a, b) => a - b);
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return [];
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
export default class Logs extends Command {
|
|
26
|
+
static description = "Download logs from an experiment run";
|
|
27
|
+
static examples = [
|
|
28
|
+
"<%= config.bin %> <%= command.id %> my-experiment",
|
|
29
|
+
"<%= config.bin %> <%= command.id %> my-experiment <run-id>",
|
|
30
|
+
"<%= config.bin %> <%= command.id %> my-experiment --all",
|
|
31
|
+
"<%= config.bin %> <%= command.id %> my-experiment --version 2",
|
|
32
|
+
"<%= config.bin %> <%= command.id %> my-experiment --version 1 --all",
|
|
33
|
+
];
|
|
34
|
+
static args = {
|
|
35
|
+
experimentName: Args.string({
|
|
36
|
+
description: "Experiment name",
|
|
37
|
+
required: true,
|
|
38
|
+
}),
|
|
39
|
+
runId: Args.string({
|
|
40
|
+
description: "Specific run ID to download logs from (optional)",
|
|
41
|
+
required: false,
|
|
42
|
+
}),
|
|
43
|
+
};
|
|
44
|
+
static flags = {
|
|
45
|
+
all: Flags.boolean({
|
|
46
|
+
description: "Download logs for all runs",
|
|
47
|
+
default: false,
|
|
48
|
+
}),
|
|
49
|
+
version: Flags.integer({
|
|
50
|
+
description: "Specific experiment version to download logs from (e.g., 0, 1, 2)",
|
|
51
|
+
required: false,
|
|
52
|
+
}),
|
|
53
|
+
...getConfigFlags("api-key", "api-url"),
|
|
54
|
+
};
|
|
55
|
+
async run() {
|
|
56
|
+
const { args, flags } = await this.parse(Logs);
|
|
57
|
+
const api = new ApiClient(flags["api-url"], flags["api-key"]);
|
|
58
|
+
const { experimentName, runId } = args;
|
|
59
|
+
await this.downloadForExperiment(experimentName, runId, api, flags.all, flags.version);
|
|
60
|
+
}
|
|
61
|
+
async downloadForExperiment(experimentName, runId, api, all, version) {
|
|
62
|
+
const experimenter = new Experimenter(experimentName, "", api);
|
|
63
|
+
// Check if experiment exists
|
|
64
|
+
if (!(await experimenter.exists())) {
|
|
65
|
+
this.error(pc.red(`Experiment '${experimentName}' does not exist. Run 'kradle experiment list' to see available experiments.`));
|
|
66
|
+
}
|
|
67
|
+
const experimentDir = experimenter.experimentDir;
|
|
68
|
+
// Get all versions
|
|
69
|
+
const allVersions = await getAllVersions(experimentDir);
|
|
70
|
+
if (allVersions.length === 0) {
|
|
71
|
+
this.error(pc.red("No experiment versions found. Run the experiment first."));
|
|
72
|
+
}
|
|
73
|
+
// Default to latest version if not specified
|
|
74
|
+
let targetVersion;
|
|
75
|
+
if (version !== undefined) {
|
|
76
|
+
if (!allVersions.includes(version)) {
|
|
77
|
+
this.error(pc.red(`Version ${version} not found in experiment '${experimentName}'. ` +
|
|
78
|
+
`Available versions: ${allVersions.join(", ")}`));
|
|
79
|
+
}
|
|
80
|
+
targetVersion = version;
|
|
81
|
+
this.log(pc.blue(`>> Filtering to version ${version}`));
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
// Default to latest version
|
|
85
|
+
targetVersion = Math.max(...allVersions);
|
|
86
|
+
}
|
|
87
|
+
const allRunInfos = [];
|
|
88
|
+
const completedStatuses = new Set(["completed", "finished", "game_over"]);
|
|
89
|
+
const progressPath = path.join(experimentDir, "versions", targetVersion.toString().padStart(3, "0"), "progress.json");
|
|
90
|
+
try {
|
|
91
|
+
const progressData = await fs.readFile(progressPath, "utf-8");
|
|
92
|
+
const progress = JSON.parse(progressData);
|
|
93
|
+
for (const entry of progress.entries) {
|
|
94
|
+
// Only include runs that are completed (exclude in-progress, queued, or error runs)
|
|
95
|
+
if (entry.runId && completedStatuses.has(entry.status)) {
|
|
96
|
+
allRunInfos.push({
|
|
97
|
+
version: targetVersion,
|
|
98
|
+
runId: entry.runId,
|
|
99
|
+
index: entry.index,
|
|
100
|
+
status: entry.status,
|
|
101
|
+
participantIds: entry.participantIds,
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
catch { }
|
|
107
|
+
if (allRunInfos.length === 0) {
|
|
108
|
+
this.error(pc.yellow("No completed runs found. Wait for runs to finish or run the experiment first."));
|
|
109
|
+
}
|
|
110
|
+
let selectedRuns;
|
|
111
|
+
if (all && !runId) {
|
|
112
|
+
// Download all runs (--all without specific run)
|
|
113
|
+
selectedRuns = allRunInfos;
|
|
114
|
+
this.log(pc.blue(`>> Downloading logs for all ${selectedRuns.length} runs`));
|
|
115
|
+
}
|
|
116
|
+
else if (runId) {
|
|
117
|
+
// Find specific run by ID
|
|
118
|
+
const matchingRun = allRunInfos.find((r) => r.runId === runId);
|
|
119
|
+
if (!matchingRun) {
|
|
120
|
+
this.error(pc.red(`Run ID '${runId}' not found in experiment '${experimentName}'. ` +
|
|
121
|
+
`Run 'kradle experiment logs ${experimentName}' to see available runs.`));
|
|
122
|
+
}
|
|
123
|
+
selectedRuns = [matchingRun];
|
|
124
|
+
this.log(pc.blue(`>> Downloading logs for run: ${runId}`));
|
|
125
|
+
}
|
|
126
|
+
else {
|
|
127
|
+
// Interactive run selection
|
|
128
|
+
const choices = [
|
|
129
|
+
// Only show "All runs" option if there are multiple runs
|
|
130
|
+
...(allRunInfos.length > 1
|
|
131
|
+
? [
|
|
132
|
+
{
|
|
133
|
+
name: "all",
|
|
134
|
+
message: `All runs (${allRunInfos.length} total)`,
|
|
135
|
+
hint: "Download all",
|
|
136
|
+
},
|
|
137
|
+
]
|
|
138
|
+
: []),
|
|
139
|
+
...allRunInfos.map((run) => {
|
|
140
|
+
const participants = run.participantIds?.join(", ") || "No participants";
|
|
141
|
+
return {
|
|
142
|
+
name: run.runId,
|
|
143
|
+
message: `${participants} - ${run.runId}`,
|
|
144
|
+
hint: run.status,
|
|
145
|
+
};
|
|
146
|
+
}),
|
|
147
|
+
];
|
|
148
|
+
const { selectedRunId } = await enquirer.prompt({
|
|
149
|
+
type: "select",
|
|
150
|
+
name: "selectedRunId",
|
|
151
|
+
message: "Select a run to download logs from",
|
|
152
|
+
choices,
|
|
153
|
+
});
|
|
154
|
+
if (selectedRunId === "all") {
|
|
155
|
+
selectedRuns = allRunInfos;
|
|
156
|
+
this.log(pc.blue(`>> Downloading logs for all ${selectedRuns.length} runs`));
|
|
157
|
+
}
|
|
158
|
+
else {
|
|
159
|
+
const selectedRun = allRunInfos.find((r) => r.runId === selectedRunId);
|
|
160
|
+
if (!selectedRun) {
|
|
161
|
+
this.error(pc.red("Selected run not found."));
|
|
162
|
+
}
|
|
163
|
+
selectedRuns = [selectedRun];
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
// Build download targets
|
|
167
|
+
const downloadTargets = selectedRuns.map((run) => ({
|
|
168
|
+
version: run.version,
|
|
169
|
+
runId: run.runId,
|
|
170
|
+
experimentDir,
|
|
171
|
+
}));
|
|
172
|
+
this.log(pc.blue(`>> Fetching and downloading logs for ${downloadTargets.length} run(s)...`));
|
|
173
|
+
await this.downloadLogs(api, downloadTargets);
|
|
174
|
+
const logsDir = path.join(experimentDir, "versions", targetVersion.toString().padStart(3, "0"), "logs");
|
|
175
|
+
this.log(pc.green(`\n Downloaded logs for ${downloadTargets.length} run(s) to ${logsDir}`));
|
|
176
|
+
}
|
|
177
|
+
async downloadLogs(api, targets) {
|
|
178
|
+
const allTasks = [];
|
|
179
|
+
for (const target of targets) {
|
|
180
|
+
const { version, runId, experimentDir } = target;
|
|
181
|
+
allTasks.push({
|
|
182
|
+
title: `${runId}`,
|
|
183
|
+
task: async (_, task) => {
|
|
184
|
+
const runDir = path.join(experimentDir, "versions", version.toString().padStart(3, "0"), "logs", runId);
|
|
185
|
+
const runPath = path.join(runDir, "run.json");
|
|
186
|
+
const logsPath = path.join(runDir, "logs.json");
|
|
187
|
+
// Fetch run result
|
|
188
|
+
let runResult;
|
|
189
|
+
try {
|
|
190
|
+
runResult = await api.getRunResult(runId);
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
throw new Error(`Failed to fetch run result: ${error instanceof Error ? error.message : String(error)}`);
|
|
194
|
+
}
|
|
195
|
+
// Fetch logs
|
|
196
|
+
let logs = [];
|
|
197
|
+
try {
|
|
198
|
+
logs = await api.getRunLogs(runId);
|
|
199
|
+
}
|
|
200
|
+
catch (error) {
|
|
201
|
+
throw new Error(`Failed to fetch logs: ${error instanceof Error ? error.message : String(error)}`);
|
|
202
|
+
}
|
|
203
|
+
// Save run result and logs
|
|
204
|
+
await fs.mkdir(runDir, { recursive: true });
|
|
205
|
+
await fs.writeFile(runPath, JSON.stringify(runResult, null, 2));
|
|
206
|
+
await fs.writeFile(logsPath, JSON.stringify(logs, null, 2));
|
|
207
|
+
task.title = `${runId} (${logs.length} log entries)`;
|
|
208
|
+
},
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
if (allTasks.length === 0) {
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
const tasks = new Listr(allTasks, {
|
|
215
|
+
concurrent: 3,
|
|
216
|
+
exitOnError: false,
|
|
217
|
+
});
|
|
218
|
+
await tasks.run();
|
|
219
|
+
}
|
|
220
|
+
}
|
|
@@ -12,6 +12,7 @@ export default class Run extends Command {
|
|
|
12
12
|
"new-version": import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
13
13
|
"max-concurrent": import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
|
|
14
14
|
"download-recordings": import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
15
|
+
"download-logs": import("@oclif/core/interfaces").BooleanFlag<boolean>;
|
|
15
16
|
};
|
|
16
17
|
run(): Promise<void>;
|
|
17
18
|
}
|
|
@@ -33,6 +33,11 @@ export default class Run extends Command {
|
|
|
33
33
|
description: "Automatically download recordings after each run finishes",
|
|
34
34
|
default: false,
|
|
35
35
|
}),
|
|
36
|
+
"download-logs": Flags.boolean({
|
|
37
|
+
char: "l",
|
|
38
|
+
description: "Automatically download logs after each run finishes",
|
|
39
|
+
default: false,
|
|
40
|
+
}),
|
|
36
41
|
...getConfigFlags("api-key", "api-url", "web-url"),
|
|
37
42
|
};
|
|
38
43
|
async run() {
|
|
@@ -57,6 +62,7 @@ export default class Run extends Command {
|
|
|
57
62
|
maxConcurrent: flags["max-concurrent"],
|
|
58
63
|
openMetabase: true,
|
|
59
64
|
downloadRecordings: flags["download-recordings"],
|
|
65
|
+
downloadLogs: flags["download-logs"],
|
|
60
66
|
});
|
|
61
67
|
this.log(pc.green("\n✓ Experiment complete!"));
|
|
62
68
|
}
|
|
@@ -13,12 +13,13 @@ export default class List extends Command {
|
|
|
13
13
|
const { flags } = await this.parse(List);
|
|
14
14
|
const api = new ApiClient(flags["api-url"], flags["api-key"]);
|
|
15
15
|
this.log(pc.blue(">> Loading worlds..."));
|
|
16
|
-
const [cloudWorlds, localWorlds, human] = await Promise.all([
|
|
16
|
+
const [cloudWorlds, kradleWorlds, localWorlds, human] = await Promise.all([
|
|
17
17
|
api.listWorlds(),
|
|
18
|
+
api.listKradleWorlds(),
|
|
18
19
|
World.getLocalWorlds(),
|
|
19
20
|
api.getHuman(),
|
|
20
21
|
]);
|
|
21
|
-
const cloudMap = new Map(cloudWorlds.map((w) => [w.slug, w]));
|
|
22
|
+
const cloudMap = new Map([...cloudWorlds, ...kradleWorlds].map((w) => [w.slug, w]));
|
|
22
23
|
const allSlugs = new Set([...cloudMap.keys(), ...localWorlds.map((slug) => `${human.username}:${slug}`)]);
|
|
23
24
|
this.log(pc.bold("\nWorlds:\n"));
|
|
24
25
|
this.log(`${"Status".padEnd(15)} ${"Slug".padEnd(40)} ${"Name".padEnd(30)}`);
|
package/dist/lib/api-client.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type z from "zod";
|
|
2
|
-
import { type AgentSchemaType, type ChallengeConfigSchemaType, type ChallengeSchemaType, type DashboardUrlResponse, type DownloadUrlResponse, HumanSchema, type RecordingMetadata, type RunStatusSchemaType, type WorldConfigSchemaType, type WorldSchemaType } from "./schemas.js";
|
|
2
|
+
import { type AgentSchemaType, type ChallengeConfigSchemaType, type ChallengeSchemaType, type DashboardUrlResponse, type DownloadUrlResponse, HumanSchema, type ParsedLogEntry, type RecordingMetadata, type RunResultResponse, type RunStatusSchemaType, type WorldConfigSchemaType, type WorldSchemaType } from "./schemas.js";
|
|
3
3
|
export declare class ApiClient {
|
|
4
4
|
private apiUrl;
|
|
5
5
|
private kradleApiKey;
|
|
@@ -114,6 +114,18 @@ export declare class ApiClient {
|
|
|
114
114
|
* @returns Download URL and expiration time.
|
|
115
115
|
*/
|
|
116
116
|
getRecordingDownloadUrl(runId: string, participantId: string, timestamp: string): Promise<DownloadUrlResponse>;
|
|
117
|
+
/**
|
|
118
|
+
* Get logs for a run, and parse the message if it is a JSON object.
|
|
119
|
+
* @param runId - The ID of the run.
|
|
120
|
+
* @returns Array of log entries.
|
|
121
|
+
*/
|
|
122
|
+
getRunLogs(runId: string): Promise<ParsedLogEntry[]>;
|
|
123
|
+
/**
|
|
124
|
+
* Get the result of a run.
|
|
125
|
+
* @param runId - The ID of the run.
|
|
126
|
+
* @returns Run result with status, end_state, and participant results.
|
|
127
|
+
*/
|
|
128
|
+
getRunResult(runId: string): Promise<RunResultResponse>;
|
|
117
129
|
listWorlds(): Promise<WorldSchemaType[]>;
|
|
118
130
|
listKradleWorlds(): Promise<WorldSchemaType[]>;
|
|
119
131
|
getWorld(slug: string): Promise<WorldSchemaType>;
|
package/dist/lib/api-client.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
|
-
import { AgentsResponseSchema, ChallengeSchema, ChallengesResponseSchema, DashboardUrlResponseSchema, DownloadUrlResponseSchema, HumanSchema, JobResponseSchema, RecordingsListResponseSchema, RunStatusSchema, UploadUrlResponseSchema, WorldSchema, WorldsResponseSchema, } from "./schemas.js";
|
|
3
|
+
import { AgentsResponseSchema, ChallengeSchema, ChallengesResponseSchema, DashboardUrlResponseSchema, DownloadUrlResponseSchema, HumanSchema, JobResponseSchema, RecordingsListResponseSchema, RunLogsResponseSchema, RunResultResponseSchema, RunStatusSchema, UploadUrlResponseSchema, WorldSchema, WorldsResponseSchema, } from "./schemas.js";
|
|
4
4
|
const DEFAULT_PAGE_SIZE = 30;
|
|
5
5
|
const DEFAULT_CHALLENGE_SCHEMA = {
|
|
6
6
|
slug: "",
|
|
7
7
|
name: "",
|
|
8
8
|
visibility: "private",
|
|
9
9
|
domain: "minecraft",
|
|
10
|
-
world: "team-kradle:
|
|
10
|
+
world: "team-kradle:flat-world",
|
|
11
11
|
challengeConfig: { cheat: false, datapack: true, gameMode: "survival" },
|
|
12
12
|
task: ".",
|
|
13
13
|
roles: { "default-role": { description: "default-role", specificTask: "do your best!" } },
|
|
@@ -277,6 +277,40 @@ export class ApiClient {
|
|
|
277
277
|
const url = `runs/${runId}/recordings/${participantId}/downloadUrl?timestamp=${encodeURIComponent(timestamp)}`;
|
|
278
278
|
return this.get(url, {}, DownloadUrlResponseSchema);
|
|
279
279
|
}
|
|
280
|
+
/**
|
|
281
|
+
* Get logs for a run, and parse the message if it is a JSON object.
|
|
282
|
+
* @param runId - The ID of the run.
|
|
283
|
+
* @returns Array of log entries.
|
|
284
|
+
*/
|
|
285
|
+
async getRunLogs(runId) {
|
|
286
|
+
const url = `runs/${runId}/logs`;
|
|
287
|
+
const response = await this.get(url, {}, RunLogsResponseSchema);
|
|
288
|
+
return response.logs.map((log) => {
|
|
289
|
+
const { message, ...rest } = log;
|
|
290
|
+
try {
|
|
291
|
+
const parsedMessage = JSON.parse(message);
|
|
292
|
+
return {
|
|
293
|
+
...rest,
|
|
294
|
+
parsedMessage: parsedMessage,
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
catch {
|
|
298
|
+
return {
|
|
299
|
+
...rest,
|
|
300
|
+
message: message,
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* Get the result of a run.
|
|
307
|
+
* @param runId - The ID of the run.
|
|
308
|
+
* @returns Run result with status, end_state, and participant results.
|
|
309
|
+
*/
|
|
310
|
+
async getRunResult(runId) {
|
|
311
|
+
const url = `runs/${runId}`;
|
|
312
|
+
return this.get(url, {}, RunResultResponseSchema);
|
|
313
|
+
}
|
|
280
314
|
async listWorlds() {
|
|
281
315
|
return this.listResource("worlds", "worlds", WorldsResponseSchema);
|
|
282
316
|
}
|
package/dist/lib/challenge.d.ts
CHANGED
|
@@ -57,4 +57,12 @@ export declare class Challenge {
|
|
|
57
57
|
* Note: config.ts is NOT created here - it should be generated later via `challenge config download`
|
|
58
58
|
*/
|
|
59
59
|
static createLocal(slug: string, kradleChallengesPath: string): Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* Download challenge source files (challenge.ts and config.ts) from the cloud
|
|
62
|
+
* @param api - The API client to use
|
|
63
|
+
* @param slug - The challenge slug to download
|
|
64
|
+
* @param targetDir - The directory to save the source files to
|
|
65
|
+
* @returns The number of files extracted
|
|
66
|
+
*/
|
|
67
|
+
static downloadSourceFiles(api: ApiClient, slug: string, targetDir: string): Promise<number>;
|
|
60
68
|
}
|
package/dist/lib/challenge.js
CHANGED
|
@@ -194,4 +194,50 @@ export class Challenge {
|
|
|
194
194
|
// Copy challenge.ts template
|
|
195
195
|
await fs.copyFile(challengeTemplatePath, challenge.challengePath);
|
|
196
196
|
}
|
|
197
|
+
/**
|
|
198
|
+
* Download challenge source files (challenge.ts and config.ts) from the cloud
|
|
199
|
+
* @param api - The API client to use
|
|
200
|
+
* @param slug - The challenge slug to download
|
|
201
|
+
* @param targetDir - The directory to save the source files to
|
|
202
|
+
* @returns The number of files extracted
|
|
203
|
+
*/
|
|
204
|
+
static async downloadSourceFiles(api, slug, targetDir) {
|
|
205
|
+
const { downloadUrl } = await api.getChallengeDownloadUrl(slug);
|
|
206
|
+
const response = await fetch(downloadUrl);
|
|
207
|
+
if (!response.ok) {
|
|
208
|
+
throw new Error(`Failed to download challenge "${slug}": ${response.status} ${response.statusText}`);
|
|
209
|
+
}
|
|
210
|
+
// Save tarball to temp location
|
|
211
|
+
const tempTarballPath = path.join(targetDir, `${slug}-temp.tar.gz`);
|
|
212
|
+
const buffer = await response.arrayBuffer();
|
|
213
|
+
await fs.mkdir(targetDir, { recursive: true });
|
|
214
|
+
await fs.writeFile(tempTarballPath, Buffer.from(buffer));
|
|
215
|
+
const filesToExtract = [`${SOURCE_FOLDER}/challenge.ts`, `${SOURCE_FOLDER}/config.ts`];
|
|
216
|
+
const tempExtractDir = path.join(targetDir, `${slug}-extract-temp`);
|
|
217
|
+
try {
|
|
218
|
+
await fs.mkdir(tempExtractDir, { recursive: true });
|
|
219
|
+
await tar.extract({
|
|
220
|
+
file: tempTarballPath,
|
|
221
|
+
cwd: tempExtractDir,
|
|
222
|
+
filter: (entryPath) => filesToExtract.some((f) => entryPath === f),
|
|
223
|
+
});
|
|
224
|
+
const srcChallengeTs = path.join(tempExtractDir, SOURCE_FOLDER, "challenge.ts");
|
|
225
|
+
const srcConfigTs = path.join(tempExtractDir, SOURCE_FOLDER, "config.ts");
|
|
226
|
+
let extractedCount = 0;
|
|
227
|
+
if (existsSync(srcChallengeTs)) {
|
|
228
|
+
await fs.copyFile(srcChallengeTs, path.join(targetDir, "challenge.ts"));
|
|
229
|
+
extractedCount++;
|
|
230
|
+
}
|
|
231
|
+
if (existsSync(srcConfigTs)) {
|
|
232
|
+
await fs.copyFile(srcConfigTs, path.join(targetDir, "config.ts"));
|
|
233
|
+
extractedCount++;
|
|
234
|
+
}
|
|
235
|
+
return extractedCount;
|
|
236
|
+
}
|
|
237
|
+
finally {
|
|
238
|
+
// Clean up temp files
|
|
239
|
+
await fs.rm(tempExtractDir, { recursive: true, force: true });
|
|
240
|
+
await fs.rm(tempTarballPath, { force: true });
|
|
241
|
+
}
|
|
242
|
+
}
|
|
197
243
|
}
|
|
@@ -19,6 +19,11 @@ export declare class Experimenter {
|
|
|
19
19
|
* Get the current version directory path
|
|
20
20
|
*/
|
|
21
21
|
getCurrentVersionDir(): string;
|
|
22
|
+
/**
|
|
23
|
+
* Download challenge source files (config.ts and challenge.ts) from the cloud
|
|
24
|
+
* and store them in the version's challenge directory
|
|
25
|
+
*/
|
|
26
|
+
private downloadChallengeFiles;
|
|
22
27
|
/**
|
|
23
28
|
* Check if experiment exists
|
|
24
29
|
*/
|
|
@@ -84,6 +89,10 @@ export declare class Experimenter {
|
|
|
84
89
|
* Open run in browser
|
|
85
90
|
*/
|
|
86
91
|
private openRun;
|
|
92
|
+
/**
|
|
93
|
+
* Download logs and run result for a completed run
|
|
94
|
+
*/
|
|
95
|
+
private downloadLogsForRun;
|
|
87
96
|
/**
|
|
88
97
|
* Download recordings for a completed run with smart polling
|
|
89
98
|
* Polls for 90 seconds after run completion (matching pod grace period)
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
3
|
import pc from "picocolors";
|
|
4
|
+
import { Challenge } from "../challenge.js";
|
|
4
5
|
import { executeNodeCommand, openInBrowser } from "../utils.js";
|
|
5
6
|
import { Runner } from "./runner.js";
|
|
6
7
|
import { TUI } from "./tui.js";
|
|
@@ -39,6 +40,7 @@ export class Experimenter {
|
|
|
39
40
|
configPath: path.join(versionDir, "config.ts"),
|
|
40
41
|
manifestPath: path.join(versionDir, "manifest.json"),
|
|
41
42
|
progressPath: path.join(versionDir, "progress.json"),
|
|
43
|
+
challengeDir: path.join(versionDir, "challenge"),
|
|
42
44
|
};
|
|
43
45
|
}
|
|
44
46
|
get configPath() {
|
|
@@ -53,6 +55,22 @@ export class Experimenter {
|
|
|
53
55
|
}
|
|
54
56
|
return this.getVersionPaths(this.currentVersion).versionDir;
|
|
55
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Download challenge source files (config.ts and challenge.ts) from the cloud
|
|
60
|
+
* and store them in the version's challenge directory
|
|
61
|
+
*/
|
|
62
|
+
async downloadChallengeFiles(manifest, challengeDir) {
|
|
63
|
+
// Extract unique challenge slugs from manifest (typically just one per experiment)
|
|
64
|
+
const challengeSlugs = [...new Set(manifest.runs.map((run) => run.challenge_slug))];
|
|
65
|
+
for (const slug of challengeSlugs) {
|
|
66
|
+
try {
|
|
67
|
+
await Challenge.downloadSourceFiles(this.api, slug, challengeDir);
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
console.error(pc.yellow(`Warning: Failed to download challenge files for "${slug}": ${error instanceof Error ? error.message : String(error)}`));
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
56
74
|
/**
|
|
57
75
|
* Check if experiment exists
|
|
58
76
|
*/
|
|
@@ -118,6 +136,8 @@ export class Experimenter {
|
|
|
118
136
|
// Generate manifest from config
|
|
119
137
|
const manifest = await this.generateManifest(paths.configPath);
|
|
120
138
|
await fs.writeFile(paths.manifestPath, JSON.stringify(manifest, null, 2));
|
|
139
|
+
// Download challenge files from cloud to preserve exact state
|
|
140
|
+
await this.downloadChallengeFiles(manifest, paths.challengeDir);
|
|
121
141
|
// Update metadata
|
|
122
142
|
await this.saveMetadata({ currentVersion: newVersion });
|
|
123
143
|
this.currentVersion = newVersion;
|
|
@@ -231,14 +251,20 @@ export class Experimenter {
|
|
|
231
251
|
maxConcurrent: options.maxConcurrent,
|
|
232
252
|
tags: tags,
|
|
233
253
|
onStateChange: () => this.onRunStateChange(),
|
|
234
|
-
onRunComplete: options.downloadRecordings
|
|
254
|
+
onRunComplete: options.downloadRecordings || options.downloadLogs
|
|
235
255
|
? async (index, runId) => {
|
|
236
256
|
const state = this.runner?.getRunState(index);
|
|
237
|
-
if (
|
|
238
|
-
|
|
239
|
-
|
|
257
|
+
if (options.downloadRecordings) {
|
|
258
|
+
if (!state?.participantIds) {
|
|
259
|
+
console.error(pc.yellow(`Warning: Participant IDs not available for run ${runId}, skipping recording download.`));
|
|
260
|
+
}
|
|
261
|
+
else {
|
|
262
|
+
await this.downloadRecordingsForRun(runId, state.participantIds, version);
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
if (options.downloadLogs) {
|
|
266
|
+
await this.downloadLogsForRun(runId, version);
|
|
240
267
|
}
|
|
241
|
-
await this.downloadRecordingsForRun(runId, state.participantIds, version);
|
|
242
268
|
}
|
|
243
269
|
: undefined,
|
|
244
270
|
});
|
|
@@ -312,6 +338,46 @@ export class Experimenter {
|
|
|
312
338
|
openInBrowser(url);
|
|
313
339
|
}
|
|
314
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Download logs and run result for a completed run
|
|
343
|
+
*/
|
|
344
|
+
async downloadLogsForRun(runId, version) {
|
|
345
|
+
const runDir = path.join(this.experimentDir, "versions", version.toString().padStart(3, "0"), "logs", runId);
|
|
346
|
+
const runPath = path.join(runDir, "run.json");
|
|
347
|
+
const logsPath = path.join(runDir, "logs.json");
|
|
348
|
+
// Download run result
|
|
349
|
+
try {
|
|
350
|
+
// Check if file already exists
|
|
351
|
+
try {
|
|
352
|
+
await fs.access(runPath);
|
|
353
|
+
}
|
|
354
|
+
catch {
|
|
355
|
+
const runResult = await this.api.getRunResult(runId);
|
|
356
|
+
await fs.mkdir(runDir, { recursive: true });
|
|
357
|
+
await fs.writeFile(runPath, JSON.stringify(runResult, null, 2));
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
catch (error) {
|
|
361
|
+
console.error(pc.yellow(`Warning: Failed to download run result for ${runId}: ${error instanceof Error ? error.message : String(error)}`));
|
|
362
|
+
}
|
|
363
|
+
// Download logs
|
|
364
|
+
try {
|
|
365
|
+
// Check if file already exists
|
|
366
|
+
try {
|
|
367
|
+
await fs.access(logsPath);
|
|
368
|
+
}
|
|
369
|
+
catch {
|
|
370
|
+
const logs = await this.api.getRunLogs(runId);
|
|
371
|
+
if (logs.length > 0) {
|
|
372
|
+
await fs.mkdir(runDir, { recursive: true });
|
|
373
|
+
await fs.writeFile(logsPath, JSON.stringify(logs, null, 2));
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
catch (error) {
|
|
378
|
+
console.error(`Failed to download logs for run ${runId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
315
381
|
/**
|
|
316
382
|
* Download recordings for a completed run with smart polling
|
|
317
383
|
* Polls for 90 seconds after run completion (matching pod grace period)
|
|
@@ -73,24 +73,6 @@ export declare const ProgressSchema: z.ZodObject<{
|
|
|
73
73
|
lastUpdated: z.ZodNumber;
|
|
74
74
|
}, z.core.$strip>;
|
|
75
75
|
export type Progress = z.infer<typeof ProgressSchema>;
|
|
76
|
-
export declare const RunResultSchema: z.ZodObject<{
|
|
77
|
-
index: z.ZodNumber;
|
|
78
|
-
runId: z.ZodString;
|
|
79
|
-
challenge_slug: z.ZodString;
|
|
80
|
-
participants: z.ZodArray<z.ZodObject<{
|
|
81
|
-
agent: z.ZodString;
|
|
82
|
-
role: z.ZodOptional<z.ZodString>;
|
|
83
|
-
}, z.core.$strip>>;
|
|
84
|
-
status: z.ZodString;
|
|
85
|
-
startTime: z.ZodNumber;
|
|
86
|
-
endTime: z.ZodNumber;
|
|
87
|
-
duration: z.ZodNumber;
|
|
88
|
-
logs: z.ZodOptional<z.ZodArray<z.ZodUnknown>>;
|
|
89
|
-
summary: z.ZodOptional<z.ZodString>;
|
|
90
|
-
error: z.ZodOptional<z.ZodString>;
|
|
91
|
-
outcome: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
92
|
-
}, z.core.$strip>;
|
|
93
|
-
export type RunResult = z.infer<typeof RunResultSchema>;
|
|
94
76
|
export interface RunState {
|
|
95
77
|
index: number;
|
|
96
78
|
config: RunConfig;
|
|
@@ -114,10 +96,6 @@ export declare const RunStatusResponseSchema: z.ZodObject<{
|
|
|
114
96
|
outcome: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
115
97
|
}, z.core.$strip>;
|
|
116
98
|
export type RunStatusResponse = z.infer<typeof RunStatusResponseSchema>;
|
|
117
|
-
export declare const RunLogsResponseSchema: z.ZodObject<{
|
|
118
|
-
logs: z.ZodArray<z.ZodUnknown>;
|
|
119
|
-
}, z.core.$strip>;
|
|
120
|
-
export type RunLogsResponse = z.infer<typeof RunLogsResponseSchema>;
|
|
121
99
|
export declare const ExperimentMetadataSchema: z.ZodObject<{
|
|
122
100
|
currentVersion: z.ZodNumber;
|
|
123
101
|
}, z.core.$strip>;
|
|
@@ -127,6 +105,7 @@ export interface ExperimentOptions {
|
|
|
127
105
|
maxConcurrent: number;
|
|
128
106
|
openMetabase?: boolean;
|
|
129
107
|
downloadRecordings?: boolean;
|
|
108
|
+
downloadLogs?: boolean;
|
|
130
109
|
}
|
|
131
110
|
export declare const STATUS_ICONS: Record<RunStatus, {
|
|
132
111
|
icon: string;
|
|
@@ -42,21 +42,6 @@ export const ProgressSchema = z.object({
|
|
|
42
42
|
entries: z.array(ProgressEntrySchema),
|
|
43
43
|
lastUpdated: z.number(),
|
|
44
44
|
});
|
|
45
|
-
// Run result with logs and summary
|
|
46
|
-
export const RunResultSchema = z.object({
|
|
47
|
-
index: z.number(),
|
|
48
|
-
runId: z.string(),
|
|
49
|
-
challenge_slug: z.string(),
|
|
50
|
-
participants: z.array(ParticipantSchema),
|
|
51
|
-
status: z.string(),
|
|
52
|
-
startTime: z.number(),
|
|
53
|
-
endTime: z.number(),
|
|
54
|
-
duration: z.number(),
|
|
55
|
-
logs: z.array(z.unknown()).optional(),
|
|
56
|
-
summary: z.string().optional(),
|
|
57
|
-
error: z.string().optional(),
|
|
58
|
-
outcome: z.record(z.string(), z.unknown()).optional(),
|
|
59
|
-
});
|
|
60
45
|
// API response schemas for run status
|
|
61
46
|
export const RunStatusResponseSchema = z.object({
|
|
62
47
|
id: z.string(),
|
|
@@ -65,9 +50,6 @@ export const RunStatusResponseSchema = z.object({
|
|
|
65
50
|
updatedAt: z.string().optional(),
|
|
66
51
|
outcome: z.record(z.string(), z.unknown()).optional(),
|
|
67
52
|
});
|
|
68
|
-
export const RunLogsResponseSchema = z.object({
|
|
69
|
-
logs: z.array(z.unknown()),
|
|
70
|
-
});
|
|
71
53
|
// Experiment metadata stored in .experiment.json
|
|
72
54
|
export const ExperimentMetadataSchema = z.object({
|
|
73
55
|
currentVersion: z.number(),
|
package/dist/lib/schemas.d.ts
CHANGED
|
@@ -190,6 +190,57 @@ export declare const RecordingsListResponseSchema: z.ZodObject<{
|
|
|
190
190
|
sizeBytes: z.ZodNumber;
|
|
191
191
|
}, z.core.$strip>>;
|
|
192
192
|
}, z.core.$strip>;
|
|
193
|
+
export declare const LogEntrySchema: z.ZodObject<{
|
|
194
|
+
participantId: z.ZodString;
|
|
195
|
+
message: z.ZodString;
|
|
196
|
+
level: z.ZodString;
|
|
197
|
+
creationTime: z.ZodString;
|
|
198
|
+
}, z.core.$strip>;
|
|
199
|
+
export declare const RunLogsResponseSchema: z.ZodObject<{
|
|
200
|
+
logs: z.ZodArray<z.ZodObject<{
|
|
201
|
+
participantId: z.ZodString;
|
|
202
|
+
message: z.ZodString;
|
|
203
|
+
level: z.ZodString;
|
|
204
|
+
creationTime: z.ZodString;
|
|
205
|
+
}, z.core.$strip>>;
|
|
206
|
+
}, z.core.$strip>;
|
|
207
|
+
export declare const RunParticipantResultSchema: z.ZodObject<{
|
|
208
|
+
agent: z.ZodString;
|
|
209
|
+
winner: z.ZodBoolean;
|
|
210
|
+
score: z.ZodOptional<z.ZodNumber>;
|
|
211
|
+
timeToSuccess: z.ZodOptional<z.ZodNumber>;
|
|
212
|
+
}, z.core.$strip>;
|
|
213
|
+
export declare const RunAggregatedResultsSchema: z.ZodObject<{
|
|
214
|
+
participantCount: z.ZodNumber;
|
|
215
|
+
successfulParticipantCount: z.ZodNumber;
|
|
216
|
+
successfulParticipantIds: z.ZodArray<z.ZodString>;
|
|
217
|
+
unsuccessfulParticipantCount: z.ZodNumber;
|
|
218
|
+
unsuccessfulParticipantIds: z.ZodArray<z.ZodString>;
|
|
219
|
+
totalTime: z.ZodNumber;
|
|
220
|
+
}, z.core.$strip>;
|
|
221
|
+
export declare const RunResultResponseSchema: z.ZodObject<{
|
|
222
|
+
id: z.ZodOptional<z.ZodString>;
|
|
223
|
+
challenge: z.ZodOptional<z.ZodString>;
|
|
224
|
+
status: z.ZodString;
|
|
225
|
+
endState: z.ZodOptional<z.ZodString>;
|
|
226
|
+
finishedStatus: z.ZodOptional<z.ZodString>;
|
|
227
|
+
totalTime: z.ZodOptional<z.ZodNumber>;
|
|
228
|
+
endTime: z.ZodOptional<z.ZodString>;
|
|
229
|
+
aggregatedResults: z.ZodOptional<z.ZodObject<{
|
|
230
|
+
participantCount: z.ZodNumber;
|
|
231
|
+
successfulParticipantCount: z.ZodNumber;
|
|
232
|
+
successfulParticipantIds: z.ZodArray<z.ZodString>;
|
|
233
|
+
unsuccessfulParticipantCount: z.ZodNumber;
|
|
234
|
+
unsuccessfulParticipantIds: z.ZodArray<z.ZodString>;
|
|
235
|
+
totalTime: z.ZodNumber;
|
|
236
|
+
}, z.core.$strip>>;
|
|
237
|
+
participantResults: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
238
|
+
agent: z.ZodString;
|
|
239
|
+
winner: z.ZodBoolean;
|
|
240
|
+
score: z.ZodOptional<z.ZodNumber>;
|
|
241
|
+
timeToSuccess: z.ZodOptional<z.ZodNumber>;
|
|
242
|
+
}, z.core.$strip>>>;
|
|
243
|
+
}, z.core.$strip>;
|
|
193
244
|
export declare const WorldSchema: z.ZodObject<{
|
|
194
245
|
id: z.ZodString;
|
|
195
246
|
slug: z.ZodString;
|
|
@@ -239,6 +290,14 @@ export type AgentsResponseType = z.infer<typeof AgentsResponseSchema>;
|
|
|
239
290
|
export type RecordingMetadata = z.infer<typeof RecordingMetadataSchema>;
|
|
240
291
|
export type RecordingsListResponse = z.infer<typeof RecordingsListResponseSchema>;
|
|
241
292
|
export type RunParticipant = z.infer<typeof RunParticipantSchema>;
|
|
293
|
+
export type LogEntry = z.infer<typeof LogEntrySchema>;
|
|
294
|
+
export type ParsedLogEntry = LogEntry | (Omit<LogEntry, "message"> & {
|
|
295
|
+
parsedMessage: Record<string, unknown>;
|
|
296
|
+
});
|
|
297
|
+
export type RunLogsResponse = z.infer<typeof RunLogsResponseSchema>;
|
|
298
|
+
export type RunParticipantResult = z.infer<typeof RunParticipantResultSchema>;
|
|
299
|
+
export type RunAggregatedResults = z.infer<typeof RunAggregatedResultsSchema>;
|
|
300
|
+
export type RunResultResponse = z.infer<typeof RunResultResponseSchema>;
|
|
242
301
|
export type DownloadUrlResponse = z.infer<typeof DownloadUrlResponseSchema>;
|
|
243
302
|
export type WorldSchemaType = z.infer<typeof WorldSchema>;
|
|
244
303
|
export type WorldConfigSchemaType = z.infer<typeof WorldConfigSchema>;
|
package/dist/lib/schemas.js
CHANGED
|
@@ -90,6 +90,40 @@ export const RecordingMetadataSchema = z.object({
|
|
|
90
90
|
export const RecordingsListResponseSchema = z.object({
|
|
91
91
|
recordings: z.array(RecordingMetadataSchema),
|
|
92
92
|
});
|
|
93
|
+
export const LogEntrySchema = z.object({
|
|
94
|
+
participantId: z.string(),
|
|
95
|
+
message: z.string(),
|
|
96
|
+
level: z.string(),
|
|
97
|
+
creationTime: z.string(),
|
|
98
|
+
});
|
|
99
|
+
export const RunLogsResponseSchema = z.object({
|
|
100
|
+
logs: z.array(LogEntrySchema),
|
|
101
|
+
});
|
|
102
|
+
export const RunParticipantResultSchema = z.object({
|
|
103
|
+
agent: z.string(),
|
|
104
|
+
winner: z.boolean(),
|
|
105
|
+
score: z.number().optional(),
|
|
106
|
+
timeToSuccess: z.number().optional(),
|
|
107
|
+
});
|
|
108
|
+
export const RunAggregatedResultsSchema = z.object({
|
|
109
|
+
participantCount: z.number(),
|
|
110
|
+
successfulParticipantCount: z.number(),
|
|
111
|
+
successfulParticipantIds: z.array(z.string()),
|
|
112
|
+
unsuccessfulParticipantCount: z.number(),
|
|
113
|
+
unsuccessfulParticipantIds: z.array(z.string()),
|
|
114
|
+
totalTime: z.number(),
|
|
115
|
+
});
|
|
116
|
+
export const RunResultResponseSchema = z.object({
|
|
117
|
+
id: z.string().optional(),
|
|
118
|
+
challenge: z.string().optional(),
|
|
119
|
+
status: z.string(),
|
|
120
|
+
endState: z.string().optional(),
|
|
121
|
+
finishedStatus: z.string().optional(),
|
|
122
|
+
totalTime: z.number().optional(),
|
|
123
|
+
endTime: z.string().optional(),
|
|
124
|
+
aggregatedResults: RunAggregatedResultsSchema.optional(),
|
|
125
|
+
participantResults: z.record(z.string(), RunParticipantResultSchema).optional(),
|
|
126
|
+
});
|
|
93
127
|
export const WorldSchema = z.object({
|
|
94
128
|
id: z.string(),
|
|
95
129
|
slug: z.string(),
|
package/oclif.manifest.json
CHANGED
|
@@ -86,6 +86,62 @@
|
|
|
86
86
|
"list.js"
|
|
87
87
|
]
|
|
88
88
|
},
|
|
89
|
+
"ai-docs:challenges-sdk": {
|
|
90
|
+
"aliases": [],
|
|
91
|
+
"args": {
|
|
92
|
+
"version": {
|
|
93
|
+
"description": "SDK version to fetch docs for. If not specified, the locally installed version will be used if available, otherwise the latest version will be used.",
|
|
94
|
+
"name": "version",
|
|
95
|
+
"required": false
|
|
96
|
+
}
|
|
97
|
+
},
|
|
98
|
+
"description": "Output the @kradle/challenges-sdk API reference documentation for LLMs",
|
|
99
|
+
"examples": [
|
|
100
|
+
"<%= config.bin %> <%= command.id %>",
|
|
101
|
+
"<%= config.bin %> <%= command.id %> 0.2.1",
|
|
102
|
+
"<%= config.bin %> <%= command.id %> latest"
|
|
103
|
+
],
|
|
104
|
+
"flags": {},
|
|
105
|
+
"hasDynamicHelp": false,
|
|
106
|
+
"hiddenAliases": [],
|
|
107
|
+
"id": "ai-docs:challenges-sdk",
|
|
108
|
+
"pluginAlias": "kradle",
|
|
109
|
+
"pluginName": "kradle",
|
|
110
|
+
"pluginType": "core",
|
|
111
|
+
"strict": true,
|
|
112
|
+
"enableJsonFlag": false,
|
|
113
|
+
"isESM": true,
|
|
114
|
+
"relativePath": [
|
|
115
|
+
"dist",
|
|
116
|
+
"commands",
|
|
117
|
+
"ai-docs",
|
|
118
|
+
"challenges-sdk.js"
|
|
119
|
+
]
|
|
120
|
+
},
|
|
121
|
+
"ai-docs:cli": {
|
|
122
|
+
"aliases": [],
|
|
123
|
+
"args": {},
|
|
124
|
+
"description": "Output the Kradle CLI reference documentation for LLMs",
|
|
125
|
+
"examples": [
|
|
126
|
+
"<%= config.bin %> <%= command.id %>"
|
|
127
|
+
],
|
|
128
|
+
"flags": {},
|
|
129
|
+
"hasDynamicHelp": false,
|
|
130
|
+
"hiddenAliases": [],
|
|
131
|
+
"id": "ai-docs:cli",
|
|
132
|
+
"pluginAlias": "kradle",
|
|
133
|
+
"pluginName": "kradle",
|
|
134
|
+
"pluginType": "core",
|
|
135
|
+
"strict": true,
|
|
136
|
+
"enableJsonFlag": false,
|
|
137
|
+
"isESM": true,
|
|
138
|
+
"relativePath": [
|
|
139
|
+
"dist",
|
|
140
|
+
"commands",
|
|
141
|
+
"ai-docs",
|
|
142
|
+
"cli.js"
|
|
143
|
+
]
|
|
144
|
+
},
|
|
89
145
|
"challenge:build": {
|
|
90
146
|
"aliases": [],
|
|
91
147
|
"args": {
|
|
@@ -661,6 +717,79 @@
|
|
|
661
717
|
"list.js"
|
|
662
718
|
]
|
|
663
719
|
},
|
|
720
|
+
"experiment:logs": {
|
|
721
|
+
"aliases": [],
|
|
722
|
+
"args": {
|
|
723
|
+
"experimentName": {
|
|
724
|
+
"description": "Experiment name",
|
|
725
|
+
"name": "experimentName",
|
|
726
|
+
"required": true
|
|
727
|
+
},
|
|
728
|
+
"runId": {
|
|
729
|
+
"description": "Specific run ID to download logs from (optional)",
|
|
730
|
+
"name": "runId",
|
|
731
|
+
"required": false
|
|
732
|
+
}
|
|
733
|
+
},
|
|
734
|
+
"description": "Download logs from an experiment run",
|
|
735
|
+
"examples": [
|
|
736
|
+
"<%= config.bin %> <%= command.id %> my-experiment",
|
|
737
|
+
"<%= config.bin %> <%= command.id %> my-experiment <run-id>",
|
|
738
|
+
"<%= config.bin %> <%= command.id %> my-experiment --all",
|
|
739
|
+
"<%= config.bin %> <%= command.id %> my-experiment --version 2",
|
|
740
|
+
"<%= config.bin %> <%= command.id %> my-experiment --version 1 --all"
|
|
741
|
+
],
|
|
742
|
+
"flags": {
|
|
743
|
+
"all": {
|
|
744
|
+
"description": "Download logs for all runs",
|
|
745
|
+
"name": "all",
|
|
746
|
+
"allowNo": false,
|
|
747
|
+
"type": "boolean"
|
|
748
|
+
},
|
|
749
|
+
"version": {
|
|
750
|
+
"description": "Specific experiment version to download logs from (e.g., 0, 1, 2)",
|
|
751
|
+
"name": "version",
|
|
752
|
+
"required": false,
|
|
753
|
+
"hasDynamicHelp": false,
|
|
754
|
+
"multiple": false,
|
|
755
|
+
"type": "option"
|
|
756
|
+
},
|
|
757
|
+
"api-key": {
|
|
758
|
+
"description": "Kradle API key",
|
|
759
|
+
"env": "KRADLE_API_KEY",
|
|
760
|
+
"name": "api-key",
|
|
761
|
+
"required": true,
|
|
762
|
+
"hasDynamicHelp": false,
|
|
763
|
+
"multiple": false,
|
|
764
|
+
"type": "option"
|
|
765
|
+
},
|
|
766
|
+
"api-url": {
|
|
767
|
+
"description": "Kradle Web API URL",
|
|
768
|
+
"env": "KRADLE_API_URL",
|
|
769
|
+
"name": "api-url",
|
|
770
|
+
"required": true,
|
|
771
|
+
"default": "https://api.kradle.ai/v0",
|
|
772
|
+
"hasDynamicHelp": false,
|
|
773
|
+
"multiple": false,
|
|
774
|
+
"type": "option"
|
|
775
|
+
}
|
|
776
|
+
},
|
|
777
|
+
"hasDynamicHelp": false,
|
|
778
|
+
"hiddenAliases": [],
|
|
779
|
+
"id": "experiment:logs",
|
|
780
|
+
"pluginAlias": "kradle",
|
|
781
|
+
"pluginName": "kradle",
|
|
782
|
+
"pluginType": "core",
|
|
783
|
+
"strict": true,
|
|
784
|
+
"enableJsonFlag": false,
|
|
785
|
+
"isESM": true,
|
|
786
|
+
"relativePath": [
|
|
787
|
+
"dist",
|
|
788
|
+
"commands",
|
|
789
|
+
"experiment",
|
|
790
|
+
"logs.js"
|
|
791
|
+
]
|
|
792
|
+
},
|
|
664
793
|
"experiment:recordings": {
|
|
665
794
|
"aliases": [],
|
|
666
795
|
"args": {
|
|
@@ -774,6 +903,13 @@
|
|
|
774
903
|
"allowNo": false,
|
|
775
904
|
"type": "boolean"
|
|
776
905
|
},
|
|
906
|
+
"download-logs": {
|
|
907
|
+
"char": "l",
|
|
908
|
+
"description": "Automatically download logs after each run finishes",
|
|
909
|
+
"name": "download-logs",
|
|
910
|
+
"allowNo": false,
|
|
911
|
+
"type": "boolean"
|
|
912
|
+
},
|
|
777
913
|
"api-key": {
|
|
778
914
|
"description": "Kradle API key",
|
|
779
915
|
"env": "KRADLE_API_KEY",
|
|
@@ -1105,63 +1241,7 @@
|
|
|
1105
1241
|
"world",
|
|
1106
1242
|
"push.js"
|
|
1107
1243
|
]
|
|
1108
|
-
},
|
|
1109
|
-
"ai-docs:challenges-sdk": {
|
|
1110
|
-
"aliases": [],
|
|
1111
|
-
"args": {
|
|
1112
|
-
"version": {
|
|
1113
|
-
"description": "SDK version to fetch docs for. If not specified, the locally installed version will be used if available, otherwise the latest version will be used.",
|
|
1114
|
-
"name": "version",
|
|
1115
|
-
"required": false
|
|
1116
|
-
}
|
|
1117
|
-
},
|
|
1118
|
-
"description": "Output the @kradle/challenges-sdk API reference documentation for LLMs",
|
|
1119
|
-
"examples": [
|
|
1120
|
-
"<%= config.bin %> <%= command.id %>",
|
|
1121
|
-
"<%= config.bin %> <%= command.id %> 0.2.1",
|
|
1122
|
-
"<%= config.bin %> <%= command.id %> latest"
|
|
1123
|
-
],
|
|
1124
|
-
"flags": {},
|
|
1125
|
-
"hasDynamicHelp": false,
|
|
1126
|
-
"hiddenAliases": [],
|
|
1127
|
-
"id": "ai-docs:challenges-sdk",
|
|
1128
|
-
"pluginAlias": "kradle",
|
|
1129
|
-
"pluginName": "kradle",
|
|
1130
|
-
"pluginType": "core",
|
|
1131
|
-
"strict": true,
|
|
1132
|
-
"enableJsonFlag": false,
|
|
1133
|
-
"isESM": true,
|
|
1134
|
-
"relativePath": [
|
|
1135
|
-
"dist",
|
|
1136
|
-
"commands",
|
|
1137
|
-
"ai-docs",
|
|
1138
|
-
"challenges-sdk.js"
|
|
1139
|
-
]
|
|
1140
|
-
},
|
|
1141
|
-
"ai-docs:cli": {
|
|
1142
|
-
"aliases": [],
|
|
1143
|
-
"args": {},
|
|
1144
|
-
"description": "Output the Kradle CLI reference documentation for LLMs",
|
|
1145
|
-
"examples": [
|
|
1146
|
-
"<%= config.bin %> <%= command.id %>"
|
|
1147
|
-
],
|
|
1148
|
-
"flags": {},
|
|
1149
|
-
"hasDynamicHelp": false,
|
|
1150
|
-
"hiddenAliases": [],
|
|
1151
|
-
"id": "ai-docs:cli",
|
|
1152
|
-
"pluginAlias": "kradle",
|
|
1153
|
-
"pluginName": "kradle",
|
|
1154
|
-
"pluginType": "core",
|
|
1155
|
-
"strict": true,
|
|
1156
|
-
"enableJsonFlag": false,
|
|
1157
|
-
"isESM": true,
|
|
1158
|
-
"relativePath": [
|
|
1159
|
-
"dist",
|
|
1160
|
-
"commands",
|
|
1161
|
-
"ai-docs",
|
|
1162
|
-
"cli.js"
|
|
1163
|
-
]
|
|
1164
1244
|
}
|
|
1165
1245
|
},
|
|
1166
|
-
"version": "0.
|
|
1246
|
+
"version": "0.4.0"
|
|
1167
1247
|
}
|
package/package.json
CHANGED
|
@@ -436,6 +436,7 @@ kradle experiment run <name>
|
|
|
436
436
|
kradle experiment run <name> --new-version
|
|
437
437
|
kradle experiment run <name> --max-concurrent 10
|
|
438
438
|
kradle experiment run <name> --download-recordings
|
|
439
|
+
kradle experiment run <name> --download-logs
|
|
439
440
|
```
|
|
440
441
|
|
|
441
442
|
**Arguments:**
|
|
@@ -449,6 +450,7 @@ kradle experiment run <name> --download-recordings
|
|
|
449
450
|
| `--new-version` | `-n` | Start a fresh version instead of resuming | false |
|
|
450
451
|
| `--max-concurrent` | `-m` | Maximum parallel runs | 5 |
|
|
451
452
|
| `--download-recordings` | `-d` | Auto-download recordings as runs complete | false |
|
|
453
|
+
| `--download-logs` | `-l` | Auto-download logs as runs complete | false |
|
|
452
454
|
|
|
453
455
|
**Behavior:**
|
|
454
456
|
1. Creates new version or resumes existing incomplete version
|
|
@@ -471,8 +473,11 @@ kradle experiment run benchmark-v1 --max-concurrent 10
|
|
|
471
473
|
# Auto-download recordings
|
|
472
474
|
kradle experiment run benchmark-v1 --download-recordings
|
|
473
475
|
|
|
476
|
+
# Auto-download logs
|
|
477
|
+
kradle experiment run benchmark-v1 --download-logs
|
|
478
|
+
|
|
474
479
|
# Combine flags
|
|
475
|
-
kradle experiment run benchmark-v1 -n -m 10 -d
|
|
480
|
+
kradle experiment run benchmark-v1 -n -m 10 -d -l
|
|
476
481
|
```
|
|
477
482
|
|
|
478
483
|
---
|
|
@@ -536,6 +541,77 @@ kradle experiment recordings benchmark-v1 --version 2 --all
|
|
|
536
541
|
|
|
537
542
|
---
|
|
538
543
|
|
|
544
|
+
### `kradle experiment logs <name>`
|
|
545
|
+
|
|
546
|
+
Downloads logs and run results from completed experiment runs.
|
|
547
|
+
|
|
548
|
+
**Usage:**
|
|
549
|
+
```bash
|
|
550
|
+
kradle experiment logs <name>
|
|
551
|
+
kradle experiment logs <name> <run-id>
|
|
552
|
+
kradle experiment logs <name> --all
|
|
553
|
+
kradle experiment logs <name> --version 2
|
|
554
|
+
```
|
|
555
|
+
|
|
556
|
+
**Arguments:**
|
|
557
|
+
| Argument | Description | Required |
|
|
558
|
+
|----------|-------------|----------|
|
|
559
|
+
| `name` | Experiment name | Yes |
|
|
560
|
+
| `run-id` | Specific run ID to download | No |
|
|
561
|
+
|
|
562
|
+
**Flags:**
|
|
563
|
+
| Flag | Description | Default |
|
|
564
|
+
|------|-------------|---------|
|
|
565
|
+
| `--all` | Download logs for all runs (skips interactive selection) | false |
|
|
566
|
+
| `--version` | Specific experiment version number | Latest version |
|
|
567
|
+
|
|
568
|
+
**Interactive mode (no flags):**
|
|
569
|
+
1. Select a run from completed runs list
|
|
570
|
+
|
|
571
|
+
**Output location:**
|
|
572
|
+
```
|
|
573
|
+
experiments/<name>/versions/<version>/logs/<run-id>/
|
|
574
|
+
```
|
|
575
|
+
|
|
576
|
+
**Downloaded files:**
|
|
577
|
+
|
|
578
|
+
1. `run.json` - Run result containing:
|
|
579
|
+
- `id`: Run ID
|
|
580
|
+
- `challenge`: Challenge slug
|
|
581
|
+
- `status`: Run status
|
|
582
|
+
- `end_state`: Final game state
|
|
583
|
+
- `finished_status`: How the run finished
|
|
584
|
+
- `total_time`: Total run duration
|
|
585
|
+
- `end_time`: When the run ended
|
|
586
|
+
- `aggregated_results`: Summary with participant counts, successful/unsuccessful IDs
|
|
587
|
+
- `participant_results`: Per-participant results (agent, winner, score, time_to_success)
|
|
588
|
+
|
|
589
|
+
2. `logs.json` - Array of log entries containing:
|
|
590
|
+
- `participant_id`: The participant that generated the log
|
|
591
|
+
- `message`: The log message content
|
|
592
|
+
- `level`: Log level (debug, info, warning, error)
|
|
593
|
+
- `creation_time`: Timestamp when the log was created
|
|
594
|
+
|
|
595
|
+
**Examples:**
|
|
596
|
+
```bash
|
|
597
|
+
# Interactive selection
|
|
598
|
+
kradle experiment logs benchmark-v1
|
|
599
|
+
|
|
600
|
+
# Download specific run
|
|
601
|
+
kradle experiment logs benchmark-v1 abc123-run-id
|
|
602
|
+
|
|
603
|
+
# Download all logs from all runs
|
|
604
|
+
kradle experiment logs benchmark-v1 --all
|
|
605
|
+
|
|
606
|
+
# Download from specific version
|
|
607
|
+
kradle experiment logs benchmark-v1 --version 1
|
|
608
|
+
|
|
609
|
+
# Combine version and all flags
|
|
610
|
+
kradle experiment logs benchmark-v1 --version 2 --all
|
|
611
|
+
```
|
|
612
|
+
|
|
613
|
+
---
|
|
614
|
+
|
|
539
615
|
### `kradle experiment list`
|
|
540
616
|
|
|
541
617
|
Lists all local experiments.
|
|
@@ -952,6 +1028,7 @@ kradle challenge build --all --visibility public
|
|
|
952
1028
|
| `kradle experiment create <name>` | Create new experiment |
|
|
953
1029
|
| `kradle experiment run <name>` | Run/resume experiment |
|
|
954
1030
|
| `kradle experiment recordings <name>` | Download recordings |
|
|
1031
|
+
| `kradle experiment logs <name>` | Download logs |
|
|
955
1032
|
| `kradle experiment list` | List experiments |
|
|
956
1033
|
| `kradle world import <path>` | Import Minecraft world folder |
|
|
957
1034
|
| `kradle world push <slug...>` | Push world(s) to cloud |
|