@dailephd/my-dev-kit-lab 0.2.0

package/dist/src/evaluation/benchmarkMetadata.js

@@ -0,0 +1,207 @@
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { PROJECT_COMPLEXITY_FORMULA, calculateProjectComplexityScore } from "./projectComplexity.js";
+export const REQUIRED_BENCHMARK_PROJECT_IDS = [
+    "todo-ts",
+    "todo-python",
+    "todo-js",
+    "todo-mixed-ts-py",
+    "task-workflow-medium-ts",
+    "task-analytics-large-mixed"
+];
+export const VALID_COMPLEXITY_LEVELS = new Set(["small", "medium", "large", "mixed-language"]);
+export async function readBenchmarkProjectProfiles(profilesPath, repoRoot = process.cwd()) {
+    let parsed;
+    try {
+        parsed = JSON.parse(await readFile(profilesPath, "utf8"));
+    }
+    catch (error) {
+        throw new Error(`Failed to parse benchmark project profiles: ${error.message}`);
+    }
+    const profiles = parseBenchmarkProjectProfiles(parsed);
+    const errors = validateBenchmarkProjectProfiles(profiles, repoRoot);
+    if (errors.length > 0) {
+        throw new Error(`Invalid benchmark project profiles:\n${errors.join("\n")}`);
+    }
+    return profiles;
+}
+export function parseBenchmarkProjectProfiles(value) {
+    if (!value || typeof value !== "object") {
+        throw new Error("Benchmark project profiles file must contain an object.");
+    }
+    const contract = value;
+    if (!Array.isArray(contract.profiles)) {
+        throw new Error("Benchmark project profiles file must contain a profiles array.");
+    }
+    return contract.profiles;
+}
+export function validateAnswerKey(answerKey, label) {
+    const errors = [];
+    if (!answerKey || typeof answerKey !== "object") {
+        return [`${label}: answerKey must be an object.`];
+    }
+    const candidate = answerKey;
+    for (const field of ["expectedFiles", "expectedSymbols", "expectedFacts"]) {
+        if (!Array.isArray(candidate[field])) {
+            errors.push(`${label}: answerKey.${field} must be an array.`);
+        }
+    }
+    if (!Number.isInteger(candidate.minimumCorrectFacts) || candidate.minimumCorrectFacts < 0) {
+        errors.push(`${label}: answerKey.minimumCorrectFacts must be a nonnegative integer.`);
+    }
+    if (Array.isArray(candidate.expectedFacts)) {
+        const factIds = new Set();
+        let requiredFactCount = 0;
+        for (const fact of candidate.expectedFacts) {
+            if (!fact || typeof fact !== "object") {
+                errors.push(`${label}: expectedFacts entries must be objects.`);
+                continue;
+            }
+            if (typeof fact.id !== "string" || fact.id.length === 0) {
+                errors.push(`${label}: expectedFacts entries must include id.`);
+            }
+            else if (factIds.has(fact.id)) {
+                errors.push(`${label}: duplicate expected fact id ${fact.id}.`);
+            }
+            else {
+                factIds.add(fact.id);
+            }
+            if (typeof fact.text !== "string" || fact.text.length === 0) {
+                errors.push(`${label}: expected fact ${fact.id ?? "<unknown>"} must include text.`);
+            }
+            if (typeof fact.weight !== "number" || fact.weight <= 0) {
+                errors.push(`${label}: expected fact ${fact.id ?? "<unknown>"} must include positive weight.`);
+            }
+            if (typeof fact.required !== "boolean") {
+                errors.push(`${label}: expected fact ${fact.id ?? "<unknown>"} must include required boolean.`);
+            }
+            if (fact.required === true) {
+                requiredFactCount += 1;
+            }
+        }
+        if (Number.isInteger(candidate.minimumCorrectFacts) && candidate.minimumCorrectFacts > candidate.expectedFacts.length) {
+            errors.push(`${label}: answerKey.minimumCorrectFacts cannot exceed expectedFacts length.`);
+        }
+        if (requiredFactCount > 0 && candidate.minimumCorrectFacts > requiredFactCount + (candidate.expectedFacts.length - requiredFactCount)) {
+            errors.push(`${label}: answerKey.minimumCorrectFacts is not satisfiable.`);
+        }
+    }
+    if (Array.isArray(candidate.expectedFiles) && candidate.expectedFiles.length === 0) {
+        errors.push(`${label}: answerKey.expectedFiles must not be empty.`);
+    }
+    if (Array.isArray(candidate.expectedSymbols) && candidate.expectedSymbols.length === 0) {
+        errors.push(`${label}: answerKey.expectedSymbols must not be empty.`);
+    }
+    return errors;
+}
+export function validateBenchmarkProjectProfiles(profiles, repoRoot = process.cwd()) {
+    const errors = [];
+    const ids = new Set(profiles.map((profile) => profile.projectId));
+    for (const requiredProjectId of REQUIRED_BENCHMARK_PROJECT_IDS) {
+        if (!ids.has(requiredProjectId)) {
+            errors.push(`Missing benchmark project profile: ${requiredProjectId}.`);
+        }
+    }
+    for (const profile of profiles) {
+        errors.push(...validateBenchmarkProjectProfile(profile, repoRoot));
+    }
+    return errors;
+}
+function validateBenchmarkProjectProfile(profile, repoRoot) {
+    const errors = [];
+    const label = `profile ${profile.projectId ?? "<unknown>"}`;
+    for (const field of ["projectId", "displayName", "description", "languageMix", "primaryLanguage", "rootPath", "benchmarkPurpose"]) {
+        if (typeof profile[field] !== "string" || profile[field].length === 0) {
+            errors.push(`${label}: ${field} must be a nonempty string.`);
+        }
+    }
+    for (const field of ["languages", "sourceRoots", "testRoots", "expectedUseCases"]) {
+        if (!Array.isArray(profile[field]) || profile[field].length === 0) {
+            errors.push(`${label}: ${field} must be a nonempty array.`);
+        }
+    }
+    if (!VALID_COMPLEXITY_LEVELS.has(profile.complexityLevel)) {
+        errors.push(`${label}: complexityLevel must be one of ${[...VALID_COMPLEXITY_LEVELS].join(", ")}.`);
+    }
+    if (typeof profile.complexityScore !== "number" || profile.complexityScore < 0 || profile.complexityScore > 100) {
+        errors.push(`${label}: complexityScore must be between 0 and 100.`);
+    }
+    if (!profile.complexityMetrics || typeof profile.complexityMetrics !== "object") {
+        errors.push(`${label}: complexityMetrics must be an object.`);
+    }
+    else {
+        errors.push(...validateComplexityMetrics(profile.complexityMetrics, label));
+        const calculatedScore = calculateProjectComplexityScore(profile.complexityMetrics);
+        if (profile.complexityScore !== calculatedScore) {
+            errors.push(`${label}: complexityScore ${profile.complexityScore} does not match formula score ${calculatedScore}.`);
+        }
+    }
+    if (profile.complexityFormula?.id !== PROJECT_COMPLEXITY_FORMULA.id) {
+        errors.push(`${label}: complexityFormula.id must be ${PROJECT_COMPLEXITY_FORMULA.id}.`);
+    }
+    const projectRoot = path.resolve(repoRoot, profile.rootPath ?? "");
+    if (!existsSync(projectRoot)) {
+        errors.push(`${label}: rootPath does not exist: ${profile.rootPath}.`);
+    }
+    if (!profile.fileTree || !Array.isArray(profile.fileTree.entries)) {
+        errors.push(`${label}: fileTree.entries must be an array.`);
+    }
+    else {
+        errors.push(...validateFileTreeEntries(profile.fileTree.entries, projectRoot, label));
+    }
+    return errors;
+}
+function validateComplexityMetrics(metrics, label) {
+    const errors = [];
+    const requiredMetricFields = [
+        "fileCount",
+        "sourceFileCount",
+        "testFileCount",
+        "totalLinesOfCode",
+        "sourceLinesOfCode",
+        "testLinesOfCode",
+        "languageCount",
+        "dependencyFileCount",
+        "internalImportCount",
+        "exportedSymbolEstimate",
+        "taskCount",
+        "expectedRelevantFilesAverage",
+        "expectedRelevantSymbolsAverage",
+        "maxFileLines",
+        "averageFileLines"
+    ];
+    for (const field of requiredMetricFields) {
+        if (typeof metrics[field] !== "number" || metrics[field] < 0) {
+            errors.push(`${label}: complexityMetrics.${field} must be a nonnegative number.`);
+        }
+    }
+    return errors;
+}
+function validateFileTreeEntries(entries, projectRoot, label) {
+    const errors = [];
+    const paths = new Set();
+    for (const entry of entries) {
+        if (typeof entry.path !== "string" || entry.path.length === 0) {
+            errors.push(`${label}: fileTree entries must include path.`);
+            continue;
+        }
+        if (path.isAbsolute(entry.path) || entry.path.includes("..")) {
+            errors.push(`${label}: fileTree path must be a safe relative path: ${entry.path}.`);
+        }
+        if (paths.has(entry.path)) {
+            errors.push(`${label}: duplicate fileTree path: ${entry.path}.`);
+        }
+        paths.add(entry.path);
+        if (entry.kind !== "file" && entry.kind !== "directory") {
+            errors.push(`${label}: fileTree path ${entry.path} has invalid kind.`);
+        }
+        if (!existsSync(path.join(projectRoot, entry.path))) {
+            errors.push(`${label}: fileTree path does not exist: ${entry.path}.`);
+        }
+        if (entry.kind === "file" && (typeof entry.lines !== "number" || entry.lines < 0)) {
+            errors.push(`${label}: fileTree file ${entry.path} must include nonnegative lines.`);
+        }
+    }
+    return errors;
+}

package/dist/src/evaluation/buildExperimentMatrix.js

@@ -0,0 +1,75 @@
+export const ALL_EXPERIMENT_AGENTS = ["fake-agent", "codex", "claude"];
+export const DEFAULT_EXPERIMENT_AGENTS = ["fake-agent"];
+export const DEFAULT_EXPERIMENT_STRATEGIES = ["raw-full-file", "my-dev-kit-guided"];
+export const DEFAULT_EXPERIMENT_COMPLEXITIES = ["short"];
+const VALID_AGENTS = new Set(ALL_EXPERIMENT_AGENTS);
+const VALID_STRATEGIES = new Set(DEFAULT_EXPERIMENT_STRATEGIES);
+const VALID_COMPLEXITIES = new Set(["short", "medium", "long", "multi-step"]);
+export function buildExperimentMatrix(args) {
+    const agents = args.config.agents ?? DEFAULT_EXPERIMENT_AGENTS;
+    const strategies = args.config.strategies ?? DEFAULT_EXPERIMENT_STRATEGIES;
+    const complexityLevels = args.config.complexityLevels ?? DEFAULT_EXPERIMENT_COMPLEXITIES;
+    validateSelections({ agents, strategies, complexityLevels, includeRealAgents: args.config.includeRealAgents ?? false });
+    const selectedCases = args.cases.filter((evaluationCase) => {
+        const caseMatches = !args.config.caseIds?.length || args.config.caseIds.includes(evaluationCase.id);
+        const projectMatches = !args.config.benchmarkProjects?.length || args.config.benchmarkProjects.includes(evaluationCase.benchmarkProject);
+        return caseMatches && projectMatches;
+    });
+    if (args.config.caseIds?.length) {
+        const found = new Set(selectedCases.map((evaluationCase) => evaluationCase.id));
+        const missing = args.config.caseIds.filter((caseId) => !found.has(caseId));
+        if (missing.length > 0) {
+            throw new Error(`Evaluation case not found: ${missing.join(", ")}`);
+        }
+    }
+    const cells = [];
+    for (const evaluationCase of selectedCases) {
+        for (const agentId of agents) {
+            for (const strategy of strategies) {
+                for (const complexityLevel of complexityLevels) {
+                    cells.push({
+                        caseId: evaluationCase.id,
+                        benchmarkProject: evaluationCase.benchmarkProject,
+                        agentId,
+                        strategy,
+                        complexityLevel,
+                        runId: buildExperimentRunId({
+                            caseId: evaluationCase.id,
+                            benchmarkProject: evaluationCase.benchmarkProject,
+                            agentId,
+                            strategy,
+                            complexityLevel
+                        })
+                    });
+                }
+            }
+        }
+    }
+    return typeof args.config.maxRuns === "number" ? cells.slice(0, args.config.maxRuns) : cells;
+}
+export function buildExperimentRunId(args) {
+    return [args.caseId, args.benchmarkProject, args.agentId, args.strategy, args.complexityLevel].map(safeSegment).join(".");
+}
+function validateSelections(args) {
+    for (const agent of args.agents) {
+        if (!VALID_AGENTS.has(agent)) {
+            throw new Error(`Invalid experiment agent: ${agent}`);
+        }
+        if ((agent === "codex" || agent === "claude") && !args.includeRealAgents) {
+            throw new Error(`Real agent ${agent} requires --include-real-agents.`);
+        }
+    }
+    for (const strategy of args.strategies) {
+        if (!VALID_STRATEGIES.has(strategy)) {
+            throw new Error(`Invalid experiment strategy: ${strategy}`);
+        }
+    }
+    for (const complexity of args.complexityLevels) {
+        if (!VALID_COMPLEXITIES.has(complexity)) {
+            throw new Error(`Invalid prompt complexity level: ${complexity}`);
+        }
+    }
+}
+function safeSegment(value) {
+    return value.replace(/[^a-zA-Z0-9._-]+/g, "-");
+}

package/dist/src/evaluation/classifyAgentRunOutcome.js

@@ -0,0 +1,40 @@
+const LIMIT_PATTERNS = [
+    /usage\s+limit/i,
+    /rate\s+limit/i,
+    /quota/i,
+    /session\s+limit/i,
+    /limit\s+reached/i,
+    /too\s+many\s+requests/i,
+    /exhausted/i,
+    /insufficient\s+quota/i,
+    /daily\s+limit/i,
+    /monthly\s+limit/i
+];
+export function classifyAgentRunOutcome(args) {
+    const result = args.agentRunResult;
+    const warnings = [...result.warnings, ...(args.parsedAnswer?.warnings ?? [])];
+    const errors = [...result.errors];
+    const combinedText = [result.finalAnswerText, ...result.warnings, ...result.errors].join("\n");
+    if (result.status === "skipped" && /not available|unavailable|not found/i.test(combinedText)) {
+        return { status: "agent-unavailable", statusReason: "Agent executable is unavailable.", warnings, errors };
+    }
+    if (LIMIT_PATTERNS.some((pattern) => pattern.test(combinedText))) {
+        return { status: "agent-limit-reached", statusReason: "Agent output indicates an external usage or session limit.", warnings, errors };
+    }
+    if (/timed out after|timeout|timed out/i.test(combinedText)) {
+        return { status: "timeout", statusReason: "Agent command timed out.", warnings, errors };
+    }
+    if (result.status === "failed") {
+        return { status: "failed", statusReason: result.errors[0] ?? "Agent run failed.", warnings, errors };
+    }
+    if (result.status === "skipped") {
+        return { status: "skipped", statusReason: result.warnings[0] ?? "Agent run was skipped.", warnings, errors };
+    }
+    if (result.status === "completed" && args.parsedAnswer?.parseStatus === "failed") {
+        return { status: "invalid-output", statusReason: "Agent completed but answer could not be parsed for scoring.", warnings, errors };
+    }
+    if (result.status === "completed" && args.parsedAnswer?.parseStatus === "partial") {
+        return { status: "invalid-output", statusReason: "Agent completed but answer was only partially parseable for scoring.", warnings, errors };
+    }
+    return { status: "completed", statusReason: "Agent completed and answer was scoreable.", warnings, errors };
+}

package/dist/src/evaluation/compareExperimentRuns.js

@@ -0,0 +1,79 @@
+export function compareExperimentRuns(runs) {
+    const groups = new Map();
+    for (const run of runs) {
+        const key = [run.caseId, run.benchmarkProject, run.agentId, run.promptComplexityLevel].join("|");
+        groups.set(key, [...(groups.get(key) ?? []), run]);
+    }
+    const comparisons = [];
+    for (const groupRuns of groups.values()) {
+        const raw = groupRuns.find((run) => run.promptStrategy === "raw-full-file");
+        const guided = groupRuns.find((run) => run.promptStrategy === "my-dev-kit-guided");
+        const anchor = raw ?? guided;
+        if (!anchor) {
+            continue;
+        }
+        const warnings = [];
+        if (!raw || !guided) {
+            warnings.push("Comparison is missing one paired strategy run.");
+        }
+        const rawTotalTokens = raw?.tokenUsage.totalTokens;
+        const myDevKitTotalTokens = guided?.tokenUsage.totalTokens;
+        const tokenComparisonAvailable = typeof rawTotalTokens === "number" && typeof myDevKitTotalTokens === "number";
+        if (!tokenComparisonAvailable) {
+            warnings.push("Token comparison unavailable because one or both runs lack total token usage.");
+        }
+        const tokenDelta = tokenComparisonAvailable ? rawTotalTokens - myDevKitTotalTokens : undefined;
+        const tokenSavingsPercent = tokenComparisonAvailable && rawTotalTokens !== 0 && typeof tokenDelta === "number" ? round((tokenDelta / rawTotalTokens) * 100) : undefined;
+        const durationDeltaMs = typeof raw?.durationMs === "number" && typeof guided?.durationMs === "number" ? raw.durationMs - guided.durationMs : undefined;
+        const durationReductionPercent = typeof durationDeltaMs === "number" && raw && raw.durationMs !== 0 ? round((durationDeltaMs / raw.durationMs) * 100) : undefined;
+        const correctnessDelta = raw && guided ? round(guided.correctness.correctnessScore - raw.correctness.correctnessScore) : undefined;
+        comparisons.push({
+            comparisonId: [anchor.caseId, anchor.benchmarkProject, anchor.agentId, anchor.promptComplexityLevel].join("."),
+            caseId: anchor.caseId,
+            benchmarkProject: anchor.benchmarkProject,
+            agentId: anchor.agentId,
+            complexityLevel: anchor.promptComplexityLevel,
+            rawRunId: raw?.runId,
+            myDevKitRunId: guided?.runId,
+            rawStatus: raw?.status,
+            myDevKitStatus: guided?.status,
+            rawCorrectnessScore: raw?.correctness.correctnessScore,
+            myDevKitCorrectnessScore: guided?.correctness.correctnessScore,
+            sameCorrectnessPass: Boolean(raw && guided && raw.correctness.passed === guided.correctness.passed),
+            correctnessDelta,
+            rawDurationMs: raw?.durationMs,
+            myDevKitDurationMs: guided?.durationMs,
+            durationDeltaMs,
+            durationReductionPercent,
+            rawTotalTokens,
+            myDevKitTotalTokens,
+            tokenDelta,
+            tokenSavingsPercent,
+            tokenComparisonAvailable,
+            reliabilityLabel: labelReliability(raw, guided, tokenComparisonAvailable),
+            warnings
+        });
+    }
+    return comparisons;
+}
+function round(value) {
+    return Math.round(value * 10000) / 10000;
+}
+function labelReliability(raw, guided, tokenComparisonAvailable) {
+    if (!raw || !guided) {
+        return "partial";
+    }
+    if (raw.status === "agent-limit-reached" || guided.status === "agent-limit-reached") {
+        return "limit-reached";
+    }
+    if (raw.status === "agent-unavailable" || guided.status === "agent-unavailable") {
+        return "unavailable";
+    }
+    if (raw.status === "completed" && guided.status === "completed") {
+        return tokenComparisonAvailable ? "strong" : "correctness-only";
+    }
+    if (raw.status === "failed" || guided.status === "failed" || raw.status === "invalid-output" || guided.status === "invalid-output") {
+        return "failed";
+    }
+    return "partial";
+}

package/dist/src/evaluation/compareTokenSavings.js

@@ -0,0 +1,47 @@
+import { tokenCountMethod } from "../core/countTokens.js";
+export function compareTokenSavings(evaluations) {
+    const cases = evaluations.map(({ evaluationCase, rawBaseline, myDevKit }) => {
+        const tokensSaved = rawBaseline.totalEstimatedTokens - myDevKit.totalEstimatedTokens;
+        const percentSaved = rawBaseline.totalEstimatedTokens === 0 ? 0 : (tokensSaved / rawBaseline.totalEstimatedTokens) * 100;
+        return {
+            caseId: evaluationCase.id,
+            title: evaluationCase.title,
+            benchmarkProject: evaluationCase.benchmarkProject,
+            rawChars: rawBaseline.totalChars,
+            rawEstimatedTokens: rawBaseline.totalEstimatedTokens,
+            myDevKitChars: myDevKit.totalChars,
+            myDevKitEstimatedTokens: myDevKit.totalEstimatedTokens,
+            tokensSaved,
+            percentSaved,
+            filesReadRaw: rawBaseline.totalFiles,
+            filesReadMyDevKit: myDevKit.filesRead.length,
+            commandsRun: myDevKit.commands.length,
+            durationMsRaw: rawBaseline.durationMs,
+            durationMsMyDevKit: myDevKit.durationMs,
+            skipped: myDevKit.skipped,
+            warnings: [...myDevKit.warnings]
+        };
+    });
+    const completed = cases.filter((result) => !result.skipped);
+    const aggregate = (selector) => completed.reduce((sum, item) => sum + selector(item), 0);
+    const average = (selector) => (completed.length === 0 ? 0 : aggregate(selector) / completed.length);
+    return {
+        cases,
+        summary: {
+            caseCount: cases.length,
+            completedCaseCount: completed.length,
+            skippedCaseCount: cases.length - completed.length,
+            averageRawTokens: average((item) => item.rawEstimatedTokens),
+            averageMyDevKitTokens: average((item) => item.myDevKitEstimatedTokens),
+            averageTokensSaved: average((item) => item.tokensSaved),
+            averagePercentSaved: average((item) => item.percentSaved),
+            totalRawTokens: aggregate((item) => item.rawEstimatedTokens),
+            totalMyDevKitTokens: aggregate((item) => item.myDevKitEstimatedTokens),
+            totalTokensSaved: aggregate((item) => item.tokensSaved),
+            totalCommandsRun: cases.reduce((sum, item) => sum + item.commandsRun, 0),
+            totalDurationMs: cases.reduce((sum, item) => sum + item.durationMsRaw + item.durationMsMyDevKit, 0),
+            tokenCountMethod,
+            warnings: cases.flatMap((item) => item.warnings)
+        }
+    };
+}

package/dist/src/evaluation/controlledExperimentTypes.js

@@ -0,0 +1 @@
+export {};

package/dist/src/evaluation/index.js

@@ -0,0 +1,18 @@
+export * from "./types.js";
+export * from "./readEvaluationCases.js";
+export * from "./runRawFullFileBaseline.js";
+export * from "./runMyDevKitRetrieval.js";
+export * from "./compareTokenSavings.js";
+export * from "./writeTokenSavingsArtifacts.js";
+export * from "./renderTokenSavingsReportInput.js";
+export * from "./projectFileTree.js";
+export * from "./projectComplexity.js";
+export * from "./benchmarkMetadata.js";
+export * from "./controlledExperimentTypes.js";
+export * from "./buildExperimentMatrix.js";
+export * from "./classifyAgentRunOutcome.js";
+export * from "./parseAgentAnswer.js";
+export * from "./scoreCorrectness.js";
+export * from "./compareExperimentRuns.js";
+export * from "./runControlledExperiment.js";
+export * from "./writeExperimentArtifacts.js";