@dailephd/my-dev-kit-lab 0.2.0

package/dist/src/experiments/config.js

@@ -0,0 +1,24 @@
+export function validExperimentConfig(config, warnings = []) {
+    return {
+        valid: true,
+        config,
+        errors: [],
+        warnings,
+    };
+}
+export function invalidExperimentConfig(errors, warnings = []) {
+    return {
+        valid: false,
+        errors,
+        warnings,
+    };
+}
+export function mergeConfig(defaults, config) {
+    if (!isPlainObject(config)) {
+        return { ...defaults };
+    }
+    return { ...defaults, ...config };
+}
+export function isPlainObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/dist/src/experiments/defaultRegistry.js

@@ -0,0 +1,7 @@
+import { ExperimentPluginRegistry } from "./registry.js";
+import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyComparison/index.js";
+export function createDefaultExperimentPluginRegistry() {
+    const registry = new ExperimentPluginRegistry();
+    registry.register(contextStrategyComparisonPlugin);
+    return registry;
+}

package/dist/src/experiments/errors.js

@@ -0,0 +1,18 @@
+export class ExperimentRegistryError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ExperimentRegistryError";
+    }
+}
+export class ExperimentTargetError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ExperimentTargetError";
+    }
+}
+export class ExperimentConfigValidationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ExperimentConfigValidationError";
+    }
+}

package/dist/src/experiments/index.js

@@ -0,0 +1,9 @@
+export * from "./config.js";
+export * from "./defaultRegistry.js";
+export * from "./errors.js";
+export * from "./plugins/index.js";
+export * from "./registry.js";
+export * from "./results.js";
+export * from "./runner.js";
+export * from "./target.js";
+export * from "./types.js";

package/dist/src/experiments/outputPaths.js

@@ -0,0 +1,25 @@
+import path from "node:path";
+export function buildDefaultExperimentOutputRoot(args) {
+    return path.join(args.toolRoot, "lab-output", "experiments", sanitizePathSegment(args.pluginId), targetSlug(args.target), sanitizePathSegment(args.runId));
+}
+export function targetSlug(target) {
+    const name = target.packageName ?? path.basename(target.targetRoot);
+    const version = target.packageVersion ? `-v${target.packageVersion}` : "";
+    const mode = target.isSelf ? "self" : "external";
+    return sanitizePathSegment(`${name}${version}-${mode}`);
+}
+export function sanitizePathSegment(value) {
+    const sanitized = value
+        .replaceAll("\\", "-")
+        .replaceAll("/", "-")
+        .replaceAll(":", "-")
+        .replaceAll("@", "")
+        .replace(/[^a-zA-Z0-9._-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-|-$/g, "");
+    return sanitized || "target";
+}
+export function isPathInside(parent, candidate) {
+    const relative = path.relative(path.resolve(parent), path.resolve(candidate));
+    return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative));
+}

package/dist/src/experiments/plugins/contextStrategyComparison/config.js

@@ -0,0 +1,37 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultContextStrategyComparisonConfig = {
+    casesPath: "examples/token-savings-cases.json",
+    projectProfilesPath: "benchmarks/contracts/benchmark-project-profiles.json",
+    agents: ["fake-agent"],
+    strategies: ["raw-full-file", "my-dev-kit-guided"],
+    complexityLevels: ["short"],
+    outDir: "lab-output/context-strategy-comparison",
+    continueOnFailure: true,
+    includeRealAgents: false,
+};
+export function validateContextStrategyComparisonConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["context strategy comparison config must be an object."]);
+    }
+    const normalized = mergeConfig(defaultContextStrategyComparisonConfig, config);
+    const errors = [];
+    if (!normalized.casesPath || typeof normalized.casesPath !== "string") {
+        errors.push("casesPath is required.");
+    }
+    if (!normalized.outDir || typeof normalized.outDir !== "string") {
+        errors.push("outDir is required.");
+    }
+    if (normalized.agents && !Array.isArray(normalized.agents)) {
+        errors.push("agents must be an array when provided.");
+    }
+    if (normalized.strategies && !Array.isArray(normalized.strategies)) {
+        errors.push("strategies must be an array when provided.");
+    }
+    if (normalized.complexityLevels && !Array.isArray(normalized.complexityLevels)) {
+        errors.push("complexityLevels must be an array when provided.");
+    }
+    if (errors.length > 0) {
+        return invalidExperimentConfig(errors);
+    }
+    return validExperimentConfig(normalized);
+}

package/dist/src/experiments/plugins/contextStrategyComparison/index.js

@@ -0,0 +1,3 @@
+export * from "./config.js";
+export * from "./plugin.js";
+export * from "./resultMapping.js";

package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js

@@ -0,0 +1,83 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { runControlledExperiment } from "../../../evaluation/runControlledExperiment.js";
+import { defaultContextStrategyComparisonConfig, validateContextStrategyComparisonConfig, } from "./config.js";
+import { mapLegacyArtifactsToExperimentRun, } from "./resultMapping.js";
+export const contextStrategyComparisonMetadata = {
+    id: "context-strategy-comparison",
+    name: "Context Strategy Comparison",
+    description: "Compare raw full-file context against my-dev-kit guided retrieval/context strategies.",
+    schemaVersion: "1.0.0",
+    status: "experimental",
+    supportedTargets: ["self", "external-local"],
+    supportedOutputs: ["json", "html", "plot", "screenshot", "artifact"],
+};
+export const contextStrategyComparisonPlugin = {
+    metadata: contextStrategyComparisonMetadata,
+    defaultConfig: defaultContextStrategyComparisonConfig,
+    configDefinition: {
+        fields: [
+            { name: "casesPath", type: "string", required: true },
+            { name: "projectProfilesPath", type: "string" },
+            { name: "agents", type: "array" },
+            { name: "strategies", type: "array" },
+            { name: "complexityLevels", type: "array" },
+            { name: "outDir", type: "string", required: true },
+        ],
+    },
+    validateConfig: validateContextStrategyComparisonConfig,
+    async run(context) {
+        const cases = readInputArray(context.inputs, "cases");
+        const projectProfiles = readInputArray(context.inputs, "projectProfiles");
+        const env = readEnv(context.inputs);
+        const startedAt = context.startedAt.toISOString();
+        const legacyArtifacts = await runControlledExperiment({
+            config: context.config,
+            cases,
+            projectProfiles,
+            repoRoot: context.target.targetRoot,
+            env,
+        });
+        const completedAt = new Date().toISOString();
+        const outDir = context.outputRoot ?? path.resolve(context.toolRoot, context.config.outDir);
+        await mkdir(outDir, { recursive: true });
+        const pluginResultPath = path.join(outDir, "experiment-plugin-result.json");
+        const result = mapLegacyArtifactsToExperimentRun({
+            runId: context.runId,
+            startedAt,
+            completedAt,
+            target: context.target,
+            legacyArtifacts,
+            pluginResultPath,
+        });
+        await writeFile(pluginResultPath, `${JSON.stringify(redactLegacyArtifacts(result), null, 2)}\n`, "utf8");
+        return result;
+    },
+    summarize(result) {
+        if (!result.summary) {
+            throw new Error("Context strategy comparison result is missing a summary.");
+        }
+        return result.summary;
+    },
+};
+function readInputArray(inputs, key) {
+    const value = inputs?.[key];
+    if (!Array.isArray(value)) {
+        throw new Error(`Context strategy comparison requires ${key} input.`);
+    }
+    return value;
+}
+function readEnv(inputs) {
+    const env = inputs?.env;
+    if (!env || typeof env !== "object" || Array.isArray(env)) {
+        return undefined;
+    }
+    return env;
+}
+function redactLegacyArtifacts(result) {
+    const { legacyArtifacts, ...rest } = result;
+    return {
+        ...rest,
+        legacyArtifactPaths: legacyArtifacts.artifactPaths,
+    };
+}

package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js

@@ -0,0 +1,260 @@
+import { summarizeExperimentRun } from "../../results.js";
+export function mapLegacyArtifactsToExperimentRun(args) {
+    const variants = buildVariants(args.legacyArtifacts.runs);
+    const cases = buildCases(args.legacyArtifacts.runs);
+    const warnings = buildWarnings(args.legacyArtifacts);
+    const failures = buildFailures(args.legacyArtifacts.runs);
+    const metrics = buildRunMetrics(args.legacyArtifacts);
+    const artifacts = buildArtifacts(args.legacyArtifacts, args.pluginResultPath);
+    const status = statusFromArtifacts(args.legacyArtifacts);
+    const run = {
+        runId: args.runId,
+        pluginId: "context-strategy-comparison",
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        status,
+        target: args.target,
+        variants,
+        cases,
+        metrics,
+        artifacts,
+        warnings,
+        failures,
+        legacyArtifacts: args.legacyArtifacts,
+        metadata: {
+            pluginName: "Context Strategy Comparison",
+            pluginSchemaVersion: "1.0.0",
+            legacySummaryPath: args.legacyArtifacts.artifactPaths.summaryPath,
+        },
+    };
+    return {
+        ...run,
+        summary: summarizeExperimentRun(run),
+    };
+}
+function buildVariants(runs) {
+    const strategies = [...new Set(runs.map((run) => run.promptStrategy))];
+    return strategies.map((strategy) => ({
+        id: strategy,
+        name: strategy === "raw-full-file" ? "Raw Full File" : "My Dev Kit Guided",
+        description: strategy === "raw-full-file"
+            ? "Full source files are embedded directly in the prompt."
+            : "The prompt asks the agent to use my-dev-kit retrieval/context commands.",
+        metadata: { strategy },
+    }));
+}
+function buildCases(runs) {
+    const caseIds = [...new Set(runs.map((run) => run.caseId))];
+    return caseIds.map((caseId) => {
+        const caseRuns = runs.filter((run) => run.caseId === caseId);
+        return {
+            id: caseId,
+            name: caseId,
+            outcomes: caseRuns.map(mapOutcome),
+            metadata: {
+                benchmarkProject: caseRuns[0]?.benchmarkProject ?? "",
+            },
+        };
+    });
+}
+function mapOutcome(run) {
+    return {
+        id: run.runId,
+        caseId: run.caseId,
+        variantId: run.promptStrategy,
+        status: outcomeStatus(run),
+        metrics: buildOutcomeMetrics(run),
+        artifacts: buildOutcomeArtifacts(run),
+        warnings: run.warnings.map((message) => ({
+            code: "legacy-run-warning",
+            message,
+            caseId: run.caseId,
+            variantId: run.promptStrategy,
+        })),
+        failures: run.errors.map((message) => ({
+            code: run.status,
+            message,
+            caseId: run.caseId,
+            variantId: run.promptStrategy,
+            recoverable: true,
+        })),
+        startedAt: run.startedAt,
+        completedAt: run.endedAt,
+        metadata: {
+            agentId: run.agentId,
+            benchmarkProject: run.benchmarkProject,
+            promptComplexityLevel: run.promptComplexityLevel,
+            promptVariantId: run.promptVariantId,
+            legacyStatus: run.status,
+            statusReason: run.statusReason,
+        },
+    };
+}
+function buildOutcomeMetrics(run) {
+    const metrics = [
+        {
+            id: "correctness-score",
+            name: "Correctness score",
+            value: run.correctness.correctnessScore,
+            variantId: run.promptStrategy,
+            caseId: run.caseId,
+        },
+        {
+            id: "duration-ms",
+            name: "Duration",
+            value: run.durationMs,
+            unit: "ms",
+            variantId: run.promptStrategy,
+            caseId: run.caseId,
+        },
+    ];
+    if (typeof run.tokenUsage.totalTokens === "number") {
+        metrics.push({
+            id: "total-tokens",
+            name: "Total tokens",
+            value: run.tokenUsage.totalTokens,
+            unit: "tokens",
+            variantId: run.promptStrategy,
+            caseId: run.caseId,
+        });
+    }
+    return metrics;
+}
+function buildOutcomeArtifacts(run) {
+    const artifacts = [];
+    if (run.artifactPaths.promptPath) {
+        artifacts.push({
+            id: `${run.runId}-prompt`,
+            label: "Prompt",
+            path: run.artifactPaths.promptPath,
+            kind: "text",
+            caseId: run.caseId,
+            variantId: run.promptStrategy,
+        });
+    }
+    if (run.artifactPaths.agentRunResultPath) {
+        artifacts.push({
+            id: `${run.runId}-agent-result`,
+            label: "Agent run result",
+            path: run.artifactPaths.agentRunResultPath,
+            kind: "json",
+            caseId: run.caseId,
+            variantId: run.promptStrategy,
+        });
+    }
+    if (run.artifactPaths.parsedAnswerPath) {
+        artifacts.push({
+            id: `${run.runId}-parsed-answer`,
+            label: "Parsed answer",
+            path: run.artifactPaths.parsedAnswerPath,
+            kind: "json",
+            caseId: run.caseId,
+            variantId: run.promptStrategy,
+        });
+    }
+    if (run.artifactPaths.correctnessScorePath) {
+        artifacts.push({
+            id: `${run.runId}-correctness`,
+            label: "Correctness score",
+            path: run.artifactPaths.correctnessScorePath,
+            kind: "json",
+            caseId: run.caseId,
+            variantId: run.promptStrategy,
+        });
+    }
+    return artifacts;
+}
+function buildArtifacts(legacyArtifacts, pluginResultPath) {
+    const artifacts = [
+        { id: "legacy-summary", label: "Experiment summary", path: legacyArtifacts.artifactPaths.summaryPath, kind: "json" },
+        { id: "legacy-runs", label: "Experiment runs", path: legacyArtifacts.artifactPaths.runsPath, kind: "json" },
+        { id: "legacy-comparisons", label: "Experiment comparisons", path: legacyArtifacts.artifactPaths.comparisonsPath, kind: "json" },
+        { id: "legacy-config", label: "Experiment config", path: legacyArtifacts.artifactPaths.configPath, kind: "json" },
+    ];
+    if (pluginResultPath) {
+        artifacts.push({
+            id: "plugin-result",
+            label: "Plugin result",
+            path: pluginResultPath,
+            kind: "json",
+        });
+    }
+    return artifacts;
+}
+function buildRunMetrics(legacyArtifacts) {
+    const summary = legacyArtifacts.summary;
+    return [
+        { id: "total-runs", name: "Total runs", value: summary.totalRuns },
+        { id: "completed-runs", name: "Completed runs", value: summary.completedRuns },
+        { id: "failed-runs", name: "Failed runs", value: summary.failedRuns },
+        { id: "total-comparisons", name: "Total comparisons", value: summary.totalComparisons },
+        {
+            id: "average-token-savings-percent",
+            name: "Average token savings",
+            value: summary.averageTokenSavingsPercent,
+            unit: "percent",
+        },
+        {
+            id: "average-duration-reduction-percent",
+            name: "Average duration reduction",
+            value: summary.averageDurationReductionPercent,
+            unit: "percent",
+        },
+        {
+            id: "average-correctness-delta",
+            name: "Average correctness delta",
+            value: summary.averageCorrectnessDelta,
+        },
+    ];
+}
+function buildWarnings(legacyArtifacts) {
+    return legacyArtifacts.warnings.map((message) => ({
+        code: "legacy-artifact-warning",
+        message,
+    }));
+}
+function buildFailures(runs) {
+    return runs.flatMap((run) => run.errors.map((message) => ({
+        code: run.status,
+        message,
+        variantId: run.promptStrategy,
+        caseId: run.caseId,
+        recoverable: true,
+    })));
+}
+function outcomeStatus(run) {
+    if (run.status === "completed")
+        return "completed";
+    if (run.status === "skipped")
+        return "skipped";
+    return "failed";
+}
+function statusFromArtifacts(legacyArtifacts) {
+    const summary = legacyArtifacts.summary;
+    if (summary.totalRuns === 0)
+        return "skipped";
+    if (summary.completedRuns === summary.totalRuns)
+        return "completed";
+    if (summary.completedRuns > 0)
+        return "partial";
+    if (summary.skippedRuns === summary.totalRuns)
+        return "skipped";
+    return "failed";
+}
+export function comparisonMetrics(comparison) {
+    return [
+        {
+            id: `${comparison.comparisonId}-token-savings-percent`,
+            name: "Token savings",
+            value: comparison.tokenSavingsPercent ?? null,
+            unit: "percent",
+            caseId: comparison.caseId,
+        },
+        {
+            id: `${comparison.comparisonId}-correctness-delta`,
+            name: "Correctness delta",
+            value: comparison.correctnessDelta ?? null,
+            caseId: comparison.caseId,
+        },
+    ];
+}

package/dist/src/experiments/plugins/index.js

@@ -0,0 +1 @@
+export * from "./contextStrategyComparison/index.js";

package/dist/src/experiments/registry.js

@@ -0,0 +1,43 @@
+import { ExperimentRegistryError } from "./errors.js";
+export class ExperimentPluginRegistry {
+    plugins = new Map();
+    metadataById = new Map();
+    register(plugin) {
+        const id = plugin.metadata.id;
+        if (!id) {
+            throw new ExperimentRegistryError("Experiment plugin metadata.id is required.");
+        }
+        if (this.plugins.has(id)) {
+            throw new ExperimentRegistryError(`Experiment plugin is already registered: ${id}`);
+        }
+        this.plugins.set(id, plugin);
+        this.metadataById.set(id, cloneMetadata(plugin.metadata));
+    }
+    list() {
+        return [...this.metadataById.values()].map(cloneMetadata);
+    }
+    describe(id) {
+        const metadata = this.metadataById.get(id);
+        if (!metadata) {
+            throw new ExperimentRegistryError(`Experiment plugin not found: ${id}`);
+        }
+        return cloneMetadata(metadata);
+    }
+    find(id) {
+        return this.plugins.get(id);
+    }
+    get(id) {
+        const plugin = this.find(id);
+        if (!plugin) {
+            throw new ExperimentRegistryError(`Experiment plugin not found: ${id}`);
+        }
+        return plugin;
+    }
+}
+function cloneMetadata(metadata) {
+    return {
+        ...metadata,
+        supportedTargets: [...metadata.supportedTargets],
+        supportedOutputs: [...metadata.supportedOutputs],
+    };
+}

package/dist/src/experiments/results.js

@@ -0,0 +1,48 @@
+export const experimentRunStatuses = [
+    "completed",
+    "partial",
+    "failed",
+    "skipped",
+];
+export function summarizeExperimentRun(run) {
+    const caseStatuses = run.cases.map(statusForCase);
+    const completedCases = caseStatuses.filter((status) => status === "completed").length;
+    const partialCases = caseStatuses.filter((status) => status === "partial").length;
+    const failedCases = caseStatuses.filter((status) => status === "failed").length;
+    const skippedCases = caseStatuses.filter((status) => status === "skipped").length;
+    return {
+        status: run.status,
+        totalCases: run.cases.length,
+        completedCases,
+        partialCases,
+        failedCases,
+        skippedCases,
+        metrics: [...run.metrics],
+        warnings: collectWarnings(run),
+        failures: collectFailures(run),
+    };
+}
+function statusForCase(experimentCase) {
+    if (experimentCase.outcomes.length === 0)
+        return "skipped";
+    const statuses = experimentCase.outcomes.map((outcome) => outcome.status);
+    if (statuses.every((status) => status === "completed"))
+        return "completed";
+    if (statuses.every((status) => status === "skipped"))
+        return "skipped";
+    if (statuses.every((status) => status === "failed"))
+        return "failed";
+    return "partial";
+}
+function collectWarnings(run) {
+    return [
+        ...run.warnings,
+        ...run.cases.flatMap((experimentCase) => experimentCase.outcomes.flatMap((outcome) => outcome.warnings)),
+    ];
+}
+function collectFailures(run) {
+    return [
+        ...run.failures,
+        ...run.cases.flatMap((experimentCase) => experimentCase.outcomes.flatMap((outcome) => outcome.failures)),
+    ];
+}