@dailephd/my-dev-kit-lab 0.2.0

package/dist/src/experiments/runner.js

@@ -0,0 +1,181 @@
+import path from "node:path";
+import { createDefaultExperimentPluginRegistry } from "./defaultRegistry.js";
+import { buildDefaultExperimentOutputRoot, isPathInside } from "./outputPaths.js";
+import { resolveExperimentTarget } from "./target.js";
+export async function runExperiment(options) {
+    const toolRoot = path.resolve(options.toolRoot ?? process.cwd());
+    const registry = options.registry ?? createDefaultExperimentPluginRegistry();
+    const plugin = registry.get(options.pluginId);
+    const runId = options.runId ?? defaultRunId(options.pluginId);
+    const startedAt = options.startedAt ?? new Date();
+    const target = resolveExperimentTarget(options.targetPath, toolRoot);
+    const validation = plugin.validateConfig(options.config ?? {});
+    if (!validation.valid || validation.config === undefined) {
+        throw new Error(`Invalid experiment config for ${plugin.metadata.id}: ${validation.errors.join("; ")}`);
+    }
+    const outputRoot = resolveExperimentOutputRoot({
+        toolRoot,
+        pluginId: plugin.metadata.id,
+        runId,
+        target,
+        optionsOutputRoot: options.outputRoot,
+        inputConfig: options.config,
+    });
+    assertOutputRootAllowed({ outputRoot, target });
+    const context = buildExperimentContext({
+        runId,
+        startedAt,
+        pluginId: plugin.metadata.id,
+        pluginSchemaVersion: plugin.metadata.schemaVersion,
+        toolRoot,
+        target,
+        outputRoot,
+        config: withOutputRoot(validation.config, outputRoot),
+        inputs: options.inputs,
+        logger: options.logger,
+    });
+    try {
+        await plugin.prepare?.(context);
+        const result = await plugin.run(context);
+        const summary = result.summary ?? plugin.summarize?.(result, context);
+        return normalizeResult({
+            result,
+            summary,
+            validation,
+            outputRoot,
+            pluginSchemaVersion: plugin.metadata.schemaVersion,
+        });
+    }
+    catch (error) {
+        return buildFailedRun({
+            error,
+            runId,
+            startedAt,
+            pluginId: plugin.metadata.id,
+            pluginSchemaVersion: plugin.metadata.schemaVersion,
+            target,
+            outputRoot,
+            validationWarnings: validation.warnings,
+        });
+    }
+    finally {
+        await plugin.cleanup?.(context);
+    }
+}
+function buildExperimentContext(args) {
+    return {
+        runId: args.runId,
+        startedAt: args.startedAt,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        toolRoot: args.toolRoot,
+        targetRoot: args.target.targetRoot,
+        isSelfTarget: args.target.isSelf,
+        target: args.target,
+        config: args.config,
+        outputRoot: args.outputRoot,
+        environment: {
+            platform: process.platform,
+            nodeVersion: process.version,
+        },
+        inputs: args.inputs,
+        logger: args.logger,
+    };
+}
+function resolveExperimentOutputRoot(args) {
+    const explicitOutDir = readStringField(args.inputConfig, "outDir");
+    const outputRoot = args.optionsOutputRoot ??
+        explicitOutDir ??
+        buildDefaultExperimentOutputRoot({
+            toolRoot: args.toolRoot,
+            pluginId: args.pluginId,
+            runId: args.runId,
+            target: args.target,
+        });
+    return path.isAbsolute(outputRoot) ? path.resolve(outputRoot) : path.resolve(args.toolRoot, outputRoot);
+}
+function assertOutputRootAllowed(args) {
+    if (!args.target.isSelf && isPathInside(args.target.targetRoot, args.outputRoot)) {
+        throw new Error(`Experiment output root must not be inside the external target project: ${args.outputRoot}`);
+    }
+}
+function withOutputRoot(config, outputRoot) {
+    if (!config || typeof config !== "object" || Array.isArray(config)) {
+        return config;
+    }
+    return {
+        ...config,
+        outDir: outputRoot,
+    };
+}
+function normalizeResult(args) {
+    const validationWarnings = args.validation.warnings.map((message) => ({
+        code: "config-validation-warning",
+        message,
+    }));
+    const warnings = [...args.result.warnings, ...validationWarnings];
+    return {
+        ...args.result,
+        summary: args.summary ?? args.result.summary,
+        warnings,
+        metadata: {
+            ...args.result.metadata,
+            outputRoot: args.outputRoot,
+            pluginSchemaVersion: args.pluginSchemaVersion,
+        },
+    };
+}
+function buildFailedRun(args) {
+    const completedAt = new Date().toISOString();
+    const failure = {
+        code: "experiment-run-failed",
+        message: args.error instanceof Error ? args.error.message : String(args.error),
+        recoverable: false,
+    };
+    const warnings = args.validationWarnings.map((message) => ({
+        code: "config-validation-warning",
+        message,
+    }));
+    const artifacts = [];
+    const summary = {
+        status: "failed",
+        totalCases: 0,
+        completedCases: 0,
+        partialCases: 0,
+        failedCases: 0,
+        skippedCases: 0,
+        metrics: [],
+        warnings,
+        failures: [failure],
+    };
+    return {
+        runId: args.runId,
+        pluginId: args.pluginId,
+        startedAt: args.startedAt.toISOString(),
+        completedAt,
+        status: "failed",
+        target: args.target,
+        variants: [],
+        cases: [],
+        metrics: [],
+        artifacts,
+        warnings,
+        failures: [failure],
+        summary,
+        metadata: {
+            outputRoot: args.outputRoot,
+            pluginSchemaVersion: args.pluginSchemaVersion,
+        },
+    };
+}
+function readStringField(value, key) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+        return undefined;
+    }
+    const field = value[key];
+    return typeof field === "string" && field.trim() ? field : undefined;
+}
+function defaultRunId(pluginId) {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    return `${pluginId}-${timestamp}`;
+}

package/dist/src/experiments/target.js

@@ -0,0 +1,8 @@
+import { resolveLocalProjectTarget } from "../core/localProjectTarget.js";
+export function resolveExperimentTarget(targetPathArg, toolRoot) {
+    const target = resolveLocalProjectTarget(targetPathArg, toolRoot);
+    return {
+        ...target,
+        kind: target.isSelf ? "self" : "external-local",
+    };
+}

package/dist/src/experiments/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/gallery/index.js

@@ -0,0 +1,2 @@
+export * from "./types.js";
+export * from "./writeGalleryManifest.js";

package/dist/src/gallery/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/gallery/writeGalleryManifest.js

@@ -0,0 +1,214 @@
+import { existsSync } from "node:fs";
+import { mkdir, readdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+function toRelativePath(outDir, targetPath) {
+    if (!targetPath) {
+        return undefined;
+    }
+    return path.relative(outDir, targetPath).replace(/\\/g, "/");
+}
+function inferItemStatus(summary, screenshot, warnings) {
+    if (summary.completedCaseCount === 0) {
+        return "skipped";
+    }
+    if (screenshot.status === "failed" || warnings.length > 0 || summary.skippedCaseCount > 0) {
+        return "warning";
+    }
+    return "pass";
+}
+export async function writeGalleryManifest(options) {
+    const outDir = path.resolve(options.outDir);
+    const manifestPath = path.join(outDir, "gallery-manifest.json");
+    const requiredArtifactEntries = [
+        ["summaryPath", options.artifactPaths.summaryPath],
+        ["runsPath", options.artifactPaths.runsPath],
+        ["htmlPath", options.artifactPaths.htmlPath]
+    ];
+    for (const [label, artifactPath] of requiredArtifactEntries) {
+        if (!artifactPath) {
+            throw new Error(`Missing required artifact path: ${label}`);
+        }
+        if (!existsSync(artifactPath)) {
+            throw new Error(`Required artifact does not exist: ${artifactPath}`);
+        }
+    }
+    const warnings = [...options.warnings];
+    if (options.screenshot.warning) {
+        warnings.push(options.screenshot.warning);
+    }
+    if (options.screenshot.status === "failed" && options.screenshot.error) {
+        warnings.push(`PNG screenshot capture failed: ${options.screenshot.error}`);
+    }
+    const itemStatus = inferItemStatus(options.summary, options.screenshot, warnings);
+    const manifest = {
+        generatedAt: options.generatedAt ?? new Date().toISOString(),
+        projectName: "my-dev-kit-lab",
+        title: "my-dev-kit-lab demo gallery",
+        description: "Deterministic Milestone 1 MVP artifacts combining benchmark validation, token-savings evaluation, reports, and optional screenshots.",
+        outputDirectory: ".",
+        items: [
+            {
+                id: "token-savings-demo",
+                title: "Token savings demo",
+                description: "Static context comparison between raw full-file reading and external my-dev-kit retrieval.",
+                kind: "token-savings-report",
+                status: itemStatus,
+                htmlPath: toRelativePath(outDir, options.artifactPaths.htmlPath) ?? "token-savings-report.html",
+                screenshotPath: options.screenshot.status === "captured" && existsSync(options.artifactPaths.pngPath)
+                    ? toRelativePath(outDir, options.artifactPaths.pngPath)
+                    : undefined,
+                summaryPath: toRelativePath(outDir, options.artifactPaths.summaryPath),
+                runsPath: toRelativePath(outDir, options.artifactPaths.runsPath),
+                metrics: [
+                    { id: "case-count", label: "Case count", value: options.summary.caseCount },
+                    { id: "completed-case-count", label: "Completed case count", value: options.summary.completedCaseCount },
+                    { id: "skipped-case-count", label: "Skipped case count", value: options.summary.skippedCaseCount },
+                    { id: "average-raw-tokens", label: "Average raw tokens", value: options.summary.averageRawTokens.toFixed(2) },
+                    { id: "average-my-dev-kit-tokens", label: "Average my-dev-kit tokens", value: options.summary.averageMyDevKitTokens.toFixed(2) },
+                    { id: "average-tokens-saved", label: "Average tokens saved", value: options.summary.averageTokensSaved.toFixed(2) },
+                    { id: "average-percent-saved", label: "Average percent saved", value: options.summary.averagePercentSaved.toFixed(2), unit: "%" },
+                    { id: "token-count-method", label: "Token count method", value: options.summary.tokenCountMethod }
+                ],
+                warnings
+            }
+        ],
+        warnings
+    };
+    await mkdir(outDir, { recursive: true });
+    await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
+    return { manifest, manifestPath };
+}
+export async function writeExperimentGalleryManifest(options) {
+    const outDir = path.resolve(options.outDir);
+    await mkdir(outDir, { recursive: true });
+    const warnings = [...(options.warnings ?? [])];
+    const items = [];
+    if (options.experimentDir) {
+        items.push({
+            id: "controlled-experiment",
+            title: "Controlled experiment artifacts",
+            description: "Raw controlled experiment JSON artifacts and per-run records.",
+            kind: "controlled-experiment",
+            status: existsSync(path.join(options.experimentDir, "experiment-summary.json")) ? "pass" : "warning",
+            htmlPath: "",
+            summaryPath: toRelativePath(outDir, path.join(options.experimentDir, "experiment-summary.json")),
+            runsPath: toRelativePath(outDir, path.join(options.experimentDir, "experiment-runs.json")),
+            artifactPaths: [path.join(options.experimentDir, "experiment-comparisons.json"), path.join(options.experimentDir, "experiment-config.json")]
+                .filter(existsSync)
+                .map((artifactPath) => toRelativePath(outDir, artifactPath) ?? artifactPath),
+            tags: ["experiment"],
+            metrics: [],
+            warnings: []
+        });
+    }
+    if (options.reportDir) {
+        const pngPath = path.join(options.reportDir, "experiment-report.png");
+        items.push({
+            id: "experiment-report",
+            title: "Experiment report",
+            description: "Final controlled experiment HTML report.",
+            kind: "experiment-report",
+            status: existsSync(path.join(options.reportDir, "experiment-report.html")) ? "pass" : "warning",
+            htmlPath: toRelativePath(outDir, path.join(options.reportDir, "experiment-report.html")) ?? "experiment-report.html",
+            screenshotPath: existsSync(pngPath) ? toRelativePath(outDir, pngPath) : undefined,
+            summaryPath: toRelativePath(outDir, path.join(options.reportDir, "experiment-report.json")),
+            artifactPaths: [path.join(options.reportDir, "experiment-report-artifacts.json")]
+                .filter(existsSync)
+                .map((artifactPath) => toRelativePath(outDir, artifactPath) ?? artifactPath),
+            tags: ["report"],
+            metrics: [],
+            warnings: []
+        });
+    }
+    if (options.plotsDir) {
+        const chartPaths = await listSvgCharts(path.join(options.plotsDir, "charts"));
+        items.push({
+            id: "experiment-plots",
+            title: "Experiment plots",
+            description: "Static SVG charts and plot-ready data from controlled experiment comparisons.",
+            kind: "experiment-plots",
+            status: chartPaths.length > 0 ? "pass" : "warning",
+            htmlPath: "",
+            summaryPath: toRelativePath(outDir, path.join(options.plotsDir, "plots-summary.json")),
+            runsPath: toRelativePath(outDir, path.join(options.plotsDir, "plot-data.json")),
+            artifactPaths: chartPaths.map((artifactPath) => toRelativePath(outDir, artifactPath) ?? artifactPath),
+            tags: ["plots", "charts"],
+            metrics: [{ id: "chart-count", label: "Chart count", value: chartPaths.length }],
+            warnings: chartPaths.length === 0 ? ["No chart SVG files found."] : []
+        });
+    }
+    if (options.visualizationsDir) {
+        const artifacts = await listFiles(path.join(options.visualizationsDir, "artifacts"));
+        items.push({
+            id: "visualization-demo",
+            title: "my-dev-kit visualization demos",
+            description: "Bounded my-dev-kit visualization command smoke artifacts.",
+            kind: "visualization-demo",
+            status: existsSync(path.join(options.visualizationsDir, "visualization-demo-summary.json")) ? "pass" : "warning",
+            htmlPath: "",
+            summaryPath: toRelativePath(outDir, path.join(options.visualizationsDir, "visualization-demo-summary.json")),
+            runsPath: toRelativePath(outDir, path.join(options.visualizationsDir, "visualization-demo-runs.json")),
+            artifactPaths: artifacts.map((artifactPath) => toRelativePath(outDir, artifactPath) ?? artifactPath),
+            tags: ["visualization"],
+            metrics: [{ id: "artifact-count", label: "Artifact count", value: artifacts.length }],
+            warnings: []
+        });
+    }
+    items.push({
+        id: "final-demo",
+        title: "Final demo artifact index",
+        description: "Combined artifact set for the final my-dev-kit-lab batch demo.",
+        kind: "final-demo",
+        status: warnings.length > 0 ? "warning" : "pass",
+        htmlPath: "gallery-index.html",
+        tags: ["final-demo"],
+        metrics: [{ id: "item-count", label: "Gallery item count", value: items.length }],
+        warnings
+    });
+    const manifest = {
+        generatedAt: options.generatedAt ?? new Date().toISOString(),
+        projectName: "my-dev-kit-lab",
+        title: options.title ?? "my-dev-kit-lab final demo gallery",
+        description: options.description ?? "Controlled experiment report, plots, visualization demos, and final artifact index.",
+        outputDirectory: ".",
+        items,
+        warnings
+    };
+    const manifestPath = path.join(outDir, "gallery-manifest.json");
+    const indexPath = path.join(outDir, "gallery-index.html");
+    await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
+    await writeFile(indexPath, renderGalleryIndex(manifest), "utf8");
+    return { manifest, manifestPath, indexPath };
+}
+async function listSvgCharts(chartsDir) {
+    return (await listFiles(chartsDir)).filter((filePath) => filePath.endsWith(".svg"));
+}
+async function listFiles(dir) {
+    try {
+        const entries = await readdir(dir, { withFileTypes: true });
+        const nested = await Promise.all(entries.map((entry) => {
+            const entryPath = path.join(dir, entry.name);
+            return entry.isDirectory() ? listFiles(entryPath) : Promise.resolve([entryPath]);
+        }));
+        return nested.flat().sort();
+    }
+    catch {
+        return [];
+    }
+}
+function renderGalleryIndex(manifest) {
+    return `<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="utf-8" /><title>${escapeHtml(manifest.title)}</title><style>body{font-family:Arial,Helvetica,sans-serif;margin:32px;color:#17212b}section{border:1px solid #d9e1ea;border-radius:8px;padding:16px;margin:12px 0}code{background:#f2f4f7;padding:2px 4px}</style></head>
+<body><h1>${escapeHtml(manifest.title)}</h1><p>${escapeHtml(manifest.description)}</p>${manifest.items
+        .map((item) => `<section><h2>${escapeHtml(item.title)}</h2><p>${escapeHtml(item.description)}</p><p>Status: <strong>${escapeHtml(item.status)}</strong></p><p>Kind: ${escapeHtml(item.kind)}</p>${item.htmlPath ? `<p>HTML: <code>${escapeHtml(item.htmlPath)}</code></p>` : ""}${item.summaryPath ? `<p>Summary: <code>${escapeHtml(item.summaryPath)}</code></p>` : ""}${item.screenshotPath ? `<p>Screenshot: <code>${escapeHtml(item.screenshotPath)}</code></p>` : ""}${item.artifactPaths?.length ? `<ul>${item.artifactPaths.map((artifact) => `<li><code>${escapeHtml(artifact)}</code></li>`).join("")}</ul>` : ""}</section>`)
+        .join("")}</body></html>`;
+}
+function escapeHtml(value) {
+    return String(value ?? "")
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}

package/dist/src/index.js

@@ -0,0 +1,12 @@
+export const projectName = "my-dev-kit-lab";
+export function describeLab() {
+    return `${projectName} hosts deterministic benchmark projects and validation workflows for my-dev-kit.`;
+}
+export * from "./report/index.js";
+export * from "./screenshot/index.js";
+export * from "./evaluation/index.js";
+export * from "./prompts/index.js";
+export * from "./agents/index.js";
+export * from "./gallery/index.js";
+export * from "./plots/index.js";
+export * from "./visualizationDemos/index.js";

package/dist/src/plots/buildExperimentPlotData.js

@@ -0,0 +1,137 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+export async function buildExperimentPlotData(options) {
+    const repoRoot = path.resolve(options.repoRoot ?? process.cwd());
+    const experimentDir = path.resolve(repoRoot, options.experimentDir);
+    const runsPayload = JSON.parse(await readFile(path.join(experimentDir, "experiment-runs.json"), "utf8"));
+    const comparisonsPayload = JSON.parse(await readFile(path.join(experimentDir, "experiment-comparisons.json"), "utf8"));
+    return buildExperimentPlotDataFromRecords({
+        experimentDir,
+        runs: runsPayload.runs ?? [],
+        comparisons: comparisonsPayload.comparisons ?? [],
+        generatedAt: options.generatedAt
+    });
+}
+export function buildExperimentPlotDataFromRecords(args) {
+    const skippedPoints = [];
+    const warnings = [];
+    const runById = new Map(args.runs.map((run) => [run.runId, run]));
+    const plots = [
+        scatter("token-savings-vs-prompt-length", "Token savings vs prompt length", "Prompt estimated tokens", "Token savings percent"),
+        scatter("time-reduction-vs-prompt-length", "Execution time reduction vs prompt length", "Prompt estimated tokens", "Duration reduction percent"),
+        scatter("token-savings-vs-project-complexity", "Token savings vs project complexity", "Project complexity score", "Token savings percent"),
+        scatter("time-reduction-vs-project-complexity", "Execution time reduction vs project complexity", "Project complexity score", "Duration reduction percent"),
+        { id: "correctness-by-strategy", title: "Correctness score by strategy", xLabel: "Prompt strategy", yLabel: "Correctness score", kind: "bar", points: [], warnings: [] },
+        { id: "run-outcomes-by-agent", title: "Run outcome counts by agent", xLabel: "Agent", yLabel: "Run count", kind: "bar", points: [], warnings: [] }
+    ];
+    const byId = new Map(plots.map((plot) => [plot.id, plot]));
+    for (const comparison of args.comparisons) {
+        const rawRun = comparison.rawRunId ? runById.get(comparison.rawRunId) : undefined;
+        const guidedRun = comparison.myDevKitRunId ? runById.get(comparison.myDevKitRunId) : undefined;
+        const promptTokens = guidedRun?.promptMetrics.promptEstimatedTokens ?? rawRun?.promptMetrics.promptEstimatedTokens;
+        const projectComplexityScore = guidedRun?.projectComplexityScore ?? rawRun?.projectComplexityScore;
+        const label = [comparison.caseId, comparison.benchmarkProject, comparison.complexityLevel].join(" / ");
+        addComparisonPoint({
+            plot: byId.get("token-savings-vs-prompt-length"),
+            skippedPoints,
+            comparison,
+            x: promptTokens,
+            y: comparison.tokenSavingsPercent,
+            label,
+            reason: comparison.tokenComparisonAvailable ? undefined : "Token comparison unavailable."
+        });
+        addComparisonPoint({
+            plot: byId.get("time-reduction-vs-prompt-length"),
+            skippedPoints,
+            comparison,
+            x: promptTokens,
+            y: comparison.durationReductionPercent,
+            label,
+            reason: typeof comparison.durationReductionPercent === "number" ? undefined : "Duration comparison unavailable."
+        });
+        addComparisonPoint({
+            plot: byId.get("token-savings-vs-project-complexity"),
+            skippedPoints,
+            comparison,
+            x: projectComplexityScore,
+            y: comparison.tokenSavingsPercent,
+            label,
+            reason: comparison.tokenComparisonAvailable ? undefined : "Token comparison unavailable."
+        });
+        addComparisonPoint({
+            plot: byId.get("time-reduction-vs-project-complexity"),
+            skippedPoints,
+            comparison,
+            x: projectComplexityScore,
+            y: comparison.durationReductionPercent,
+            label,
+            reason: typeof comparison.durationReductionPercent === "number" ? undefined : "Duration comparison unavailable."
+        });
+    }
+    for (const run of args.runs) {
+        byId.get("correctness-by-strategy").points.push({
+            x: strategyIndex(run.promptStrategy),
+            y: run.correctness.correctnessScore,
+            group: run.agentId,
+            label: [run.caseId, run.benchmarkProject, run.promptComplexityLevel, run.promptStrategy].join(" / "),
+            metadata: { strategy: run.promptStrategy, status: run.status }
+        });
+    }
+    for (const [key, count] of countOutcomeGroups(args.runs)) {
+        const [agentId, status] = key.split("|");
+        byId.get("run-outcomes-by-agent").points.push({
+            x: agentIndex(agentId),
+            y: count,
+            group: status,
+            label: `${agentId} ${status}`,
+            metadata: { agentId, status }
+        });
+    }
+    for (const plot of plots) {
+        if (plot.points.length === 0) {
+            plot.warnings.push("No comparable data available.");
+            warnings.push(`${plot.title}: no comparable data available.`);
+        }
+    }
+    return {
+        generatedAt: args.generatedAt ?? new Date().toISOString(),
+        sourceExperimentDir: args.experimentDir,
+        plots,
+        skippedPoints,
+        warnings
+    };
+}
+function scatter(id, title, xLabel, yLabel) {
+    return { id, title, xLabel, yLabel, kind: "scatter", points: [], warnings: [] };
+}
+function addComparisonPoint(args) {
+    if (args.reason || typeof args.x !== "number" || typeof args.y !== "number" || !Number.isFinite(args.x) || !Number.isFinite(args.y)) {
+        args.skippedPoints.push({ plotId: args.plot.id, label: args.label, reason: args.reason ?? "Point value unavailable." });
+        return;
+    }
+    args.plot.points.push({
+        x: args.x,
+        y: args.y,
+        group: args.comparison.agentId,
+        label: args.label,
+        metadata: {
+            caseId: args.comparison.caseId,
+            benchmarkProject: args.comparison.benchmarkProject,
+            complexityLevel: args.comparison.complexityLevel
+        }
+    });
+}
+function countOutcomeGroups(runs) {
+    const counts = new Map();
+    for (const run of runs) {
+        const key = `${run.agentId}|${run.status}`;
+        counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+    return counts;
+}
+function strategyIndex(strategy) {
+    return strategy === "raw-full-file" ? 1 : 2;
+}
+function agentIndex(agentId) {
+    return agentId === "fake-agent" ? 1 : agentId === "codex" ? 2 : agentId === "claude" ? 3 : 4;
+}

package/dist/src/plots/index.js

@@ -0,0 +1,4 @@
+export * from "./types.js";
+export * from "./buildExperimentPlotData.js";
+export * from "./renderSvgChart.js";
+export * from "./writePlotArtifacts.js";

package/dist/src/plots/renderSvgChart.js

@@ -0,0 +1,82 @@
+const COLORS = ["#2563eb", "#16a34a", "#dc2626", "#9333ea", "#ea580c", "#0891b2"];
+export function renderSvgChart(plot) {
+    const width = 760;
+    const height = 420;
+    const margin = { left: 72, right: 28, top: 58, bottom: 72 };
+    const innerWidth = width - margin.left - margin.right;
+    const innerHeight = height - margin.top - margin.bottom;
+    const groups = [...new Set(plot.points.map((point) => point.group))].sort();
+    const colorFor = (group) => COLORS[Math.max(0, groups.indexOf(group)) % COLORS.length];
+    if (plot.points.length === 0) {
+        return wrapSvg(width, height, `${title(plot.title)}${text(width / 2, height / 2, "No comparable data available", "middle", "empty")}`);
+    }
+    const xs = plot.points.map((point) => point.x);
+    const ys = plot.points.map((point) => point.y);
+    const xDomain = domain(xs);
+    const yDomain = domain(ys);
+    const xScale = (value) => margin.left + ((value - xDomain[0]) / (xDomain[1] - xDomain[0])) * innerWidth;
+    const yScale = (value) => margin.top + innerHeight - ((value - yDomain[0]) / (yDomain[1] - yDomain[0])) * innerHeight;
+    const body = plot.kind === "bar"
+        ? renderBars(plot, margin, innerWidth, innerHeight, colorFor)
+        : plot.points
+            .map((point) => `<circle cx="${round(xScale(point.x))}" cy="${round(yScale(point.y))}" r="5" fill="${colorFor(point.group)}"><title>${escapeXml(point.label)} (${point.x}, ${point.y})</title></circle>`)
+            .join("");
+    return wrapSvg(width, height, [
+        title(plot.title),
+        `<line x1="${margin.left}" y1="${margin.top + innerHeight}" x2="${margin.left + innerWidth}" y2="${margin.top + innerHeight}" stroke="#344054" />`,
+        `<line x1="${margin.left}" y1="${margin.top}" x2="${margin.left}" y2="${margin.top + innerHeight}" stroke="#344054" />`,
+        text(width / 2, height - 20, plot.xLabel, "middle", "axis-label"),
+        `<text x="20" y="${height / 2}" class="axis-label" transform="rotate(-90 20 ${height / 2})" text-anchor="middle">${escapeXml(plot.yLabel)}</text>`,
+        text(margin.left, margin.top + innerHeight + 20, String(round(xDomain[0])), "middle", "tick"),
+        text(margin.left + innerWidth, margin.top + innerHeight + 20, String(round(xDomain[1])), "middle", "tick"),
+        text(margin.left - 10, margin.top + innerHeight, String(round(yDomain[0])), "end", "tick"),
+        text(margin.left - 10, margin.top + 4, String(round(yDomain[1])), "end", "tick"),
+        body,
+        renderLegend(groups, colorFor, width - 180, 70)
+    ].join(""));
+}
+function renderBars(plot, margin, innerWidth, innerHeight, colorFor) {
+    const maxY = Math.max(1, ...plot.points.map((point) => point.y));
+    const barWidth = Math.max(12, innerWidth / Math.max(1, plot.points.length) - 6);
+    return plot.points
+        .map((point, index) => {
+        const height = (point.y / maxY) * innerHeight;
+        const x = margin.left + index * (barWidth + 6);
+        const y = margin.top + innerHeight - height;
+        return `<rect x="${round(x)}" y="${round(y)}" width="${round(barWidth)}" height="${round(height)}" fill="${colorFor(point.group)}"><title>${escapeXml(point.label)}: ${point.y}</title></rect>`;
+    })
+        .join("");
+}
+function renderLegend(groups, colorFor, x, y) {
+    if (groups.length === 0)
+        return "";
+    return groups
+        .map((group, index) => {
+        const rowY = y + index * 20;
+        return `<rect x="${x}" y="${rowY - 10}" width="10" height="10" fill="${colorFor(group)}" /><text x="${x + 16}" y="${rowY}" class="legend">${escapeXml(group)}</text>`;
+    })
+        .join("");
+}
+function wrapSvg(width, height, body) {
+    return `<svg xmlns="http://www.w3.org/2000/svg" role="img" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}"><style>text{font-family:Arial,Helvetica,sans-serif;fill:#17212b}.title{font-size:20px;font-weight:700}.axis-label{font-size:13px}.tick,.legend{font-size:12px}.empty{font-size:18px;fill:#667085}</style>${body}</svg>\n`;
+}
+function title(value) {
+    return text(28, 34, value, "start", "title");
+}
+function text(x, y, value, anchor, className) {
+    return `<text x="${x}" y="${y}" text-anchor="${anchor}" class="${className}">${escapeXml(value)}</text>`;
+}
+function domain(values) {
+    const min = Math.min(...values);
+    const max = Math.max(...values);
+    if (min === max)
+        return [min - 1, max + 1];
+    const padding = (max - min) * 0.1;
+    return [min - padding, max + padding];
+}
+function round(value) {
+    return Math.round(value * 100) / 100;
+}
+function escapeXml(value) {
+    return value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;");
+}

package/dist/src/plots/types.js

@@ -0,0 +1 @@
+export {};