@kradle/cli 0.0.17 → 0.2.0

package/dist/commands/experiment/recordings.js

@@ -0,0 +1,416 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import { Args, Command, Flags } from "@oclif/core";
+import enquirer from "enquirer";
+import { Listr } from "listr2";
+import pc from "picocolors";
+import { ApiClient } from "../../lib/api-client.js";
+import { Experimenter } from "../../lib/experiment/experimenter.js";
+import { getConfigFlags } from "../../lib/flags.js";
+// Check if recordings exist locally for a run
+async function checkRecordingsExist(experimentDir, version, runId) {
+    const recordingsPath = path.join(experimentDir, "versions", version.toString().padStart(3, "0"), "recordings", runId);
+    try {
+        await fs.access(recordingsPath);
+        const files = await fs.readdir(recordingsPath, { recursive: true });
+        return files.some((f) => String(f).endsWith(".mcpr"));
+    }
+    catch {
+        return false;
+    }
+}
+// Format bytes for display
+function formatBytes(bytes) {
+    if (bytes === 0)
+        return "0 Bytes";
+    const k = 1024;
+    const sizes = ["Bytes", "KB", "MB", "GB"];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return `${Math.round((bytes / k ** i) * 100) / 100} ${sizes[i]}`;
+}
+// Sanitize timestamp string for use in filenames
+function sanitizeTimestamp(timestamp) {
+    // Replace colons, spaces, and other problematic characters
+    return timestamp
+        .replace(/:/g, "-")
+        .replace(/\s+/g, "_")
+        .replace(/[<>:"|?*]/g, "_");
+}
+// Download file with retry logic
+async function downloadFile(url, outputPath, maxRetries = 3) {
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+        try {
+            const response = await fetch(url);
+            if (!response.ok) {
+                throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+            }
+            const buffer = await response.arrayBuffer();
+            await fs.mkdir(path.dirname(outputPath), { recursive: true });
+            await fs.writeFile(outputPath, Buffer.from(buffer));
+            return;
+        }
+        catch (error) {
+            if (attempt === maxRetries)
+                throw error;
+            await new Promise((resolve) => setTimeout(resolve, 1000 * attempt)); // linear backoff: 1s, 2s, 3s
+        }
+    }
+}
+// Get all versions for an experiment
+async function getAllVersions(experimentDir) {
+    const versionsDir = path.join(experimentDir, "versions");
+    try {
+        const entries = await fs.readdir(versionsDir, { withFileTypes: true });
+        return entries
+            .filter((e) => e.isDirectory())
+            .map((e) => parseInt(e.name, 10))
+            .filter((n) => !Number.isNaN(n))
+            .sort((a, b) => a - b);
+    }
+    catch {
+        return [];
+    }
+}
+export default class Recordings extends Command {
+    static description = "Download recordings from an experiment run";
+    static examples = [
+        "<%= config.bin %> <%= command.id %> my-experiment",
+        "<%= config.bin %> <%= command.id %> my-experiment <run-id>",
+        "<%= config.bin %> <%= command.id %> my-experiment --all",
+        "<%= config.bin %> <%= command.id %> my-experiment <run-id> --all",
+        "<%= config.bin %> <%= command.id %> my-experiment --version 2",
+        "<%= config.bin %> <%= command.id %> my-experiment --version 1 --all",
+    ];
+    static args = {
+        experimentName: Args.string({
+            description: "Experiment name",
+            required: true,
+        }),
+        runId: Args.string({
+            description: "Specific run ID to download recordings from (optional)",
+            required: false,
+        }),
+    };
+    static flags = {
+        all: Flags.boolean({
+            description: "Download all runs and participants (if no run specified), or all participants (if run specified)",
+            default: false,
+        }),
+        version: Flags.integer({
+            description: "Specific experiment version to download recordings from (e.g., 0, 1, 2)",
+            required: false,
+        }),
+        ...getConfigFlags("api-key", "api-url"),
+    };
+    async run() {
+        const { args, flags } = await this.parse(Recordings);
+        const api = new ApiClient(flags["api-url"], flags["api-key"]);
+        const { experimentName, runId } = args;
+        await this.downloadForExperiment(experimentName, runId, api, flags.all, flags.version);
+    }
+    async downloadForExperiment(experimentName, runId, api, all, version) {
+        const experimenter = new Experimenter(experimentName, "", api);
+        // Check if experiment exists
+        if (!(await experimenter.exists())) {
+            this.error(pc.red(`Experiment '${experimentName}' does not exist. Run 'kradle experiment list' to see available experiments.`));
+        }
+        const experimentDir = experimenter.experimentDir;
+        // Get all versions
+        const allVersions = await getAllVersions(experimentDir);
+        if (allVersions.length === 0) {
+            this.error(pc.red("No experiment versions found. Run the experiment first."));
+        }
+        // Default to latest version if not specified
+        let targetVersion;
+        if (version !== undefined) {
+            if (!allVersions.includes(version)) {
+                this.error(pc.red(`Version ${version} not found in experiment '${experimentName}'. ` +
+                    `Available versions: ${allVersions.join(", ")}`));
+            }
+            targetVersion = version;
+            this.log(pc.blue(`>> Filtering to version ${version}`));
+        }
+        else {
+            // Default to latest version
+            targetVersion = Math.max(...allVersions);
+        }
+        const allRunInfos = [];
+        const completedStatuses = new Set(["completed", "finished", "game_over"]);
+        const progressPath = path.join(experimentDir, "versions", targetVersion.toString().padStart(3, "0"), "progress.json");
+        try {
+            const progressData = await fs.readFile(progressPath, "utf-8");
+            const progress = JSON.parse(progressData);
+            for (const entry of progress.entries) {
+                // Only include runs that are completed (exclude in-progress, queued, or error runs)
+                if (entry.runId && completedStatuses.has(entry.status)) {
+                    const hasRecordings = await checkRecordingsExist(experimentDir, targetVersion, entry.runId);
+                    allRunInfos.push({
+                        version: targetVersion,
+                        runId: entry.runId,
+                        index: entry.index,
+                        status: entry.status,
+                        hasRecordings,
+                        participantIds: entry.participantIds,
+                        endTime: entry.endTime,
+                    });
+                }
+            }
+        }
+        catch { }
+        if (allRunInfos.length === 0) {
+            this.error(pc.yellow("No completed runs found. Wait for runs to finish or run the experiment first."));
+        }
+        let selectedRuns;
+        if (all && !runId) {
+            // Download all runs and all participants (--all without specific run)
+            selectedRuns = allRunInfos;
+            this.log(pc.blue(`>> Downloading recordings for all ${selectedRuns.length} runs`));
+        }
+        else if (runId) {
+            // Find specific run by ID
+            const matchingRun = allRunInfos.find((r) => r.runId === runId);
+            if (!matchingRun) {
+                this.error(pc.red(`Run ID '${runId}' not found in experiment '${experimentName}'. ` +
+                    `Run 'kradle experiment recordings ${experimentName}' to see available runs.`));
+            }
+            selectedRuns = [matchingRun];
+            this.log(pc.blue(`>> Downloading recordings for run: ${runId}`));
+        }
+        else {
+            // Interactive run selection
+            const choices = [
+                // Only show "All runs" option if there are multiple runs
+                ...(allRunInfos.length > 1
+                    ? [
+                        {
+                            name: "all",
+                            message: `All runs (${allRunInfos.length} total)`,
+                            hint: "Download all",
+                        },
+                    ]
+                    : []),
+                ...allRunInfos.map((run) => {
+                    const indicator = run.hasRecordings ? pc.blue("·") : "☐";
+                    const participants = run.participantIds?.join(", ") || "No participants";
+                    return {
+                        name: run.runId,
+                        message: `${indicator} ${participants} - ${run.runId}`,
+                        hint: run.status,
+                    };
+                }),
+            ];
+            const { selectedRunId } = await enquirer.prompt({
+                type: "select",
+                name: "selectedRunId",
+                message: "Select a run to download recordings from",
+                choices,
+            });
+            if (selectedRunId === "all") {
+                selectedRuns = allRunInfos;
+                this.log(pc.blue(`>> Downloading recordings for all ${selectedRuns.length} runs`));
+            }
+            else {
+                const selectedRun = allRunInfos.find((r) => r.runId === selectedRunId);
+                if (!selectedRun) {
+                    this.error(pc.red("Selected run not found."));
+                }
+                selectedRuns = [selectedRun];
+            }
+        }
+        // Build download targets
+        const downloadTargets = [];
+        // Prompt for participant selection only if: single run selected AND not --all flag
+        if (selectedRuns.length === 1 && !all) {
+            const firstRunInfo = allRunInfos.find((r) => r.runId === selectedRuns[0].runId);
+            if (!firstRunInfo?.participantIds || firstRunInfo.participantIds.length === 0) {
+                this.error(pc.red("Participant IDs not available for this run. This may be an old run created before recording support was added."));
+            }
+            // Interactive participant selection for single run
+            const participantChoices = [
+                { name: "all", message: "All participants", value: "all" },
+                ...firstRunInfo.participantIds.map((id, idx) => ({
+                    name: id,
+                    message: `Participant ${idx}: ${id}`,
+                    value: id,
+                })),
+            ];
+            const { participantSelection } = await enquirer.prompt({
+                type: "select",
+                name: "participantSelection",
+                message: "Select participant(s)",
+                choices: participantChoices,
+            });
+            const run = selectedRuns[0];
+            if (participantSelection === "all") {
+                // Download all participants
+                for (const participantId of firstRunInfo.participantIds) {
+                    downloadTargets.push({
+                        version: run.version,
+                        runId: run.runId,
+                        participantId,
+                        experimentDir,
+                    });
+                }
+            }
+            else {
+                // Download single participant
+                downloadTargets.push({
+                    version: run.version,
+                    runId: run.runId,
+                    participantId: participantSelection,
+                    experimentDir,
+                });
+            }
+        }
+        else {
+            // Multiple runs OR --all flag: automatically download all participants
+            for (const run of selectedRuns) {
+                const runInfo = allRunInfos.find((r) => r.runId === run.runId);
+                if (!runInfo?.participantIds || runInfo.participantIds.length === 0) {
+                    this.log(pc.yellow(`Warning: Skipping run ${run.runId} - no participant IDs available`));
+                    continue;
+                }
+                // Download all participants for this run
+                for (const participantId of runInfo.participantIds) {
+                    downloadTargets.push({
+                        version: run.version,
+                        runId: run.runId,
+                        participantId,
+                        experimentDir,
+                    });
+                }
+            }
+        }
+        // Ensure we wait at least 90 seconds after run completion before downloading
+        // to avoid missing recordings that take time to upload
+        const MIN_WAIT_AFTER_COMPLETION_MS = 90000; // 90 seconds
+        const now = Date.now();
+        // Group targets by run and check wait times
+        const runEndTimes = new Map();
+        for (const target of downloadTargets) {
+            const runInfo = allRunInfos.find((r) => r.runId === target.runId);
+            if (runInfo?.endTime) {
+                runEndTimes.set(target.runId, runInfo.endTime);
+            }
+        }
+        // Separate into ready and deferred targets
+        const readyTargets = [];
+        const deferredTargets = [];
+        for (const target of downloadTargets) {
+            const endTime = runEndTimes.get(target.runId);
+            if (endTime) {
+                const timeSinceCompletion = now - endTime;
+                if (timeSinceCompletion < MIN_WAIT_AFTER_COMPLETION_MS) {
+                    deferredTargets.push(target);
+                }
+                else {
+                    readyTargets.push(target);
+                }
+            }
+            else {
+                // No end time available, process immediately
+                readyTargets.push(target);
+            }
+        }
+        // Process ready targets first
+        if (readyTargets.length > 0) {
+            this.log(pc.blue(`>> Fetching and downloading recordings for ${readyTargets.length} ready target(s)...`));
+            await this.fetchAndDownloadTargets(api, readyTargets);
+        }
+        // Wait for deferred targets, then process them
+        if (deferredTargets.length > 0) {
+            const uniqueDeferredRuns = new Set(deferredTargets.map((t) => t.runId));
+            const maxWaitNeeded = Math.max(...Array.from(uniqueDeferredRuns)
+                .map((runId) => {
+                const endTime = runEndTimes.get(runId);
+                if (!endTime)
+                    return 0;
+                return MIN_WAIT_AFTER_COMPLETION_MS - (now - endTime);
+            })
+                .filter((wait) => wait > 0));
+            this.log(pc.yellow(`>> Waiting ${Math.ceil(maxWaitNeeded / 1000)}s for ${uniqueDeferredRuns.size} recent run(s) to ensure all recordings are uploaded...`));
+            // Wait the necessary time
+            await new Promise((resolve) => setTimeout(resolve, maxWaitNeeded));
+            this.log(pc.blue(`>> Fetching and downloading recordings for ${deferredTargets.length} deferred target(s)...`));
+            await this.fetchAndDownloadTargets(api, deferredTargets);
+        }
+        if (readyTargets.length === 0 && deferredTargets.length === 0) {
+            this.log(pc.yellow("No recordings to download."));
+            return;
+        }
+        const totalTargets = readyTargets.length + deferredTargets.length;
+        const uniqueRuns = new Set([...readyTargets, ...deferredTargets].map((t) => t.runId));
+        // All targets are from the same version
+        const recordingsDir = path.join(experimentDir, "versions", targetVersion.toString().padStart(3, "0"), "recordings");
+        this.log(pc.green(`\n✓ Downloaded recordings for ${totalTargets} participant(s) across ${uniqueRuns.size} run(s) to ${recordingsDir}`));
+    }
+    async fetchAndDownloadTargets(api, targets) {
+        // Fetch all recordings to show summary
+        let totalCount = 0;
+        let totalSize = 0;
+        for (const target of targets) {
+            try {
+                const recordings = await api.getRunRecordings(target.runId, target.participantId);
+                totalCount += recordings.length;
+                totalSize += recordings.reduce((sum, r) => sum + r.sizeBytes, 0);
+            }
+            catch {
+                // Skip targets with no recordings
+            }
+        }
+        if (totalCount === 0) {
+            this.log(pc.yellow("   No recordings found for these targets."));
+            return;
+        }
+        this.log(pc.blue(`   Found ${totalCount} recordings (Total: ${formatBytes(totalSize)})`));
+        // Download all recordings
+        await this.downloadRecordings(api, targets);
+    }
+    async downloadRecordings(api, targets) {
+        const allTasks = [];
+        for (const target of targets) {
+            const { version, runId, participantId, experimentDir } = target;
+            // Fetch recordings for this target
+            let recordings;
+            try {
+                recordings = await api.getRunRecordings(runId, participantId);
+            }
+            catch {
+                // Skip targets with no recordings
+                continue;
+            }
+            if (recordings.length === 0)
+                continue;
+            // Create tasks for each recording
+            for (const recording of recordings) {
+                const sanitizedFilename = `${sanitizeTimestamp(recording.timestamp)}.mcpr`;
+                allTasks.push({
+                    title: `${participantId} - ${runId} - ${sanitizedFilename}`,
+                    task: async (_, task) => {
+                        const outputPath = path.join(experimentDir, "versions", version.toString().padStart(3, "0"), "recordings", runId, participantId, sanitizedFilename);
+                        // Skip if already exists
+                        try {
+                            await fs.access(outputPath);
+                            task.skip("Already downloaded");
+                            return;
+                        }
+                        catch { }
+                        // Get download URL
+                        const { downloadUrl } = await api.getRecordingDownloadUrl(runId, participantId, recording.timestamp);
+                        // Download file
+                        await downloadFile(downloadUrl, outputPath);
+                        task.title = `${participantId} - ${runId} - ${sanitizedFilename} (${formatBytes(recording.sizeBytes)})`;
+                    },
+                });
+            }
+        }
+        if (allTasks.length === 0) {
+            return;
+        }
+        const tasks = new Listr(allTasks, {
+            concurrent: 3,
+            exitOnError: false,
+        });
+        await tasks.run();
+    }
+}

package/dist/commands/experiment/run.d.ts

@@ -0,0 +1,17 @@
+import { Command } from "@oclif/core";
+export default class Run extends Command {
+    static description: string;
+    static examples: string[];
+    static args: {
+        name: import("@oclif/core/interfaces").Arg<string, Record<string, unknown>>;
+    };
+    static flags: {
+        "api-key": import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        "api-url": import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        "web-url": import("@oclif/core/interfaces").OptionFlag<string, import("@oclif/core/interfaces").CustomOptions>;
+        "new-version": import("@oclif/core/interfaces").BooleanFlag<boolean>;
+        "max-concurrent": import("@oclif/core/interfaces").OptionFlag<number, import("@oclif/core/interfaces").CustomOptions>;
+        "download-recordings": import("@oclif/core/interfaces").BooleanFlag<boolean>;
+    };
+    run(): Promise<void>;
+}

package/dist/commands/experiment/run.js

@@ -0,0 +1,67 @@
+import { Args, Command, Flags } from "@oclif/core";
+import pc from "picocolors";
+import { ApiClient } from "../../lib/api-client.js";
+import { Experimenter } from "../../lib/experiment/experimenter.js";
+import { getConfigFlags } from "../../lib/flags.js";
+const DEFAULT_MAX_CONCURRENT = 5;
+export default class Run extends Command {
+    static description = "Run an experiment. If the experiment had an ongoing version, it will resume from the last state.";
+    static examples = [
+        "<%= config.bin %> <%= command.id %> my-experiment",
+        "<%= config.bin %> <%= command.id %> my-experiment --new-version",
+        "<%= config.bin %> <%= command.id %> my-experiment --max-concurrent 10",
+    ];
+    static args = {
+        name: Args.string({
+            description: "Name of the experiment to run",
+            required: true,
+        }),
+    };
+    static flags = {
+        "new-version": Flags.boolean({
+            char: "n",
+            description: "Start a new version of the experiment",
+            default: false,
+        }),
+        "max-concurrent": Flags.integer({
+            char: "m",
+            description: "Maximum concurrent runs",
+            default: DEFAULT_MAX_CONCURRENT,
+        }),
+        "download-recordings": Flags.boolean({
+            char: "d",
+            description: "Automatically download recordings after each run finishes",
+            default: false,
+        }),
+        ...getConfigFlags("api-key", "api-url", "web-url"),
+    };
+    async run() {
+        const { args, flags } = await this.parse(Run);
+        const api = new ApiClient(flags["api-url"], flags["api-key"]);
+        const experimenter = new Experimenter(args.name, flags["web-url"], api);
+        // Check if experiment exists
+        if (!(await experimenter.exists())) {
+            this.error(pc.red(`Experiment '${args.name}' does not exist. Run 'kradle experiment create ${args.name}' first.`));
+        }
+        // Check if config.ts exists
+        if (!(await experimenter.configExists())) {
+            this.error(pc.red(`Config file not found at ${experimenter.configPath}`));
+        }
+        this.log(pc.blue(`>> Starting experiment: ${args.name}`));
+        if (flags["new-version"]) {
+            this.log(pc.yellow("   Starting a new version of the experiment."));
+        }
+        try {
+            await experimenter.run({
+                new: flags["new-version"],
+                maxConcurrent: flags["max-concurrent"],
+                openMetabase: true,
+                downloadRecordings: flags["download-recordings"],
+            });
+            this.log(pc.green("\n✓ Experiment complete!"));
+        }
+        catch (error) {
+            this.error(pc.red(`Experiment failed: ${error instanceof Error ? error.message : String(error)}`));
+        }
+    }
+}

package/dist/commands/init.js

@@ -34,10 +34,10 @@ export default class Init extends Command {
             const nonHiddenFiles = files.filter((f) => !f.startsWith("."));
             const useCurrentDir = nonHiddenFiles.length === 0;
             if (useCurrentDir) {
-                this.log(pc.yellow("Current directory is empty, it will be used to store challenges and evaluations."));
+                this.log(pc.yellow("Current directory is empty, it will be used to store challenges and experiments."));
             }
             else {
-                this.log(pc.yellow("Current directory is not empty, a subdirectory will be created to store challenges and evaluations."));
+                this.log(pc.yellow("Current directory is not empty, a subdirectory will be created to store challenges and experiments."));
             }
             let projectName;
             if (flags.name) {

package/dist/lib/api-client.d.ts

@@ -1,10 +1,10 @@
 import type z from "zod";
-import type { Challenge } from "./challenge.js";
-import type { Config } from "./config.js";
-import { type AgentSchemaType, type ChallengeSchemaType, HumanSchema, type RunStatusSchemaType } from "./schemas.js";
+import { type AgentSchemaType, type ChallengeConfigSchemaType, type ChallengeSchemaType, HumanSchema, type RecordingDownloadUrlResponse, type RecordingMetadata, type RunStatusSchemaType } from "./schemas.js";
 export declare class ApiClient {
-    private config;
-    constructor(config: Config);
+    private apiUrl;
+    private kradleApiKey;
+    private isStudio;
+    constructor(apiUrl: string, kradleApiKey: string, isStudio?: boolean);
     private request;
     private get;
     private post;
@@ -40,17 +40,43 @@ export declare class ApiClient {
     createChallenge(slug: string): Promise<unknown>;
     /**
      * Update a challenge definition in the cloud.
-     * @param challenge - The challenge to update.
-     * @param challengeConfig - The challenge config to upload. If not provided, the config will be loaded from the challenge.
+     * @param challengeSlug - The slug of the challenge.
+     * @param challengeConfig - The challenge config to upload.
+     * @param visibility - The visibility to set.
      * @returns The updated challenge.
      */
-    updateChallenge(challenge: Challenge, challengeConfig?: ChallengeSchemaType): Promise<void>;
-    getChallengeUploadUrl(challenge: Challenge): Promise<string>;
+    updateChallenge(challengeSlug: string, challengeConfig: ChallengeConfigSchemaType, visibility: "private" | "public"): Promise<void>;
+    /**
+     * Update the visibility of a challenge.
+     * @param challengeSlug - The slug of the challenge.
+     * @param visibility - The visibility to set.
+     * @returns The updated challenge.
+     */
+    updateChallengeVisibility(challengeSlug: string, visibility: "private" | "public"): Promise<void>;
+    /**
+     * Upload a challenge datapack to Google Cloud Storage.
+     * @param slug - The slug of the challenge.
+     * @param tarballPath - The path to the tarball file.
+     * @returns The upload URL.
+     */
+    uploadChallengeDatapack(slug: string, tarballPath: string): Promise<void>;
+    /**
+     * Get the upload URL for a challenge datapack.
+     * @param slug - The slug of the challenge.
+     * @returns The upload URL.
+     */
+    getChallengeUploadUrl(slug: string): Promise<string>;
     runChallenge(runData: {
         challenge: string;
         participants: unknown[];
-    }, studio?: boolean): Promise<{
+    }): Promise<{
         runIds?: string[] | undefined;
+        participants?: Record<string, {
+            agent: string;
+            role: string;
+            inputOrder: number;
+        }> | undefined;
+        id?: string | undefined;
     }>;
     deleteChallenge(challengeId: string): Promise<void>;
     /**
@@ -66,4 +92,19 @@ export declare class ApiClient {
      * @throws an error if the tag fails to be added.
      */
     tagRun(runId: string, tag: string): Promise<void>;
+    /**
+     * Get recordings for a specific participant in a run.
+     * @param runId - The ID of the run.
+     * @param participantId - The ID of the participant.
+     * @returns Array of recording metadata.
+     */
+    getRunRecordings(runId: string, participantId: string): Promise<RecordingMetadata[]>;
+    /**
+     * Get a signed download URL for a specific recording.
+     * @param runId - The ID of the run.
+     * @param participantId - The ID of the participant.
+     * @param timestamp - The timestamp of the recording.
+     * @returns Download URL and expiration time.
+     */
+    getRecordingDownloadUrl(runId: string, participantId: string, timestamp: string): Promise<RecordingDownloadUrlResponse>;
 }