@dailephd/my-dev-kit-lab 0.2.0

package/dist/src/plots/writePlotArtifacts.js

@@ -0,0 +1,46 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../core/pathSafety.js";
+import { buildExperimentPlotData } from "./buildExperimentPlotData.js";
+import { renderSvgChart } from "./renderSvgChart.js";
+const chartFiles = {
+    "token-savings-vs-prompt-length": "token-savings-vs-prompt-length.svg",
+    "time-reduction-vs-prompt-length": "time-reduction-vs-prompt-length.svg",
+    "token-savings-vs-project-complexity": "token-savings-vs-project-complexity.svg",
+    "time-reduction-vs-project-complexity": "time-reduction-vs-project-complexity.svg",
+    "correctness-by-strategy": "correctness-by-strategy.svg",
+    "run-outcomes-by-agent": "run-outcomes-by-agent.svg"
+};
+export async function writePlotArtifacts(options) {
+    const data = await buildExperimentPlotData({ experimentDir: options.experimentDir, repoRoot: options.repoRoot });
+    return writePlotArtifactsFromData({ data, outDir: options.outDir });
+}
+export async function writePlotArtifactsFromData(options) {
+    const outDir = path.resolve(options.outDir);
+    const chartsDir = resolveWithinRoot(outDir, "charts");
+    await mkdir(chartsDir, { recursive: true });
+    const summaryPath = resolveWithinRoot(outDir, "plots-summary.json");
+    const dataPath = resolveWithinRoot(outDir, "plot-data.json");
+    const charts = {};
+    for (const plot of options.data.plots) {
+        const fileName = chartFiles[plot.id] ?? `${plot.id}.svg`;
+        const chartPath = resolveWithinRoot(chartsDir, fileName);
+        await writeFile(chartPath, renderSvgChart(plot), "utf8");
+        charts[plot.id] = chartPath;
+    }
+    const summary = {
+        generatedAt: options.data.generatedAt,
+        sourceExperimentDir: options.data.sourceExperimentDir,
+        chartCount: Object.keys(charts).length,
+        skippedPointCount: options.data.skippedPoints.length,
+        warnings: options.data.warnings
+    };
+    await mkdir(outDir, { recursive: true });
+    await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
+    await writeFile(dataPath, `${JSON.stringify(options.data, null, 2)}\n`, "utf8");
+    return {
+        summary,
+        data: options.data,
+        artifactPaths: { summaryPath, dataPath, chartsDir, charts }
+    };
+}

package/dist/src/prompts/buildPromptContext.js

@@ -0,0 +1,68 @@
+export function buildPromptGenerationContext(args) {
+    const profileId = args.evaluationCase.projectProfileRef ?? args.evaluationCase.benchmarkProject;
+    const projectProfile = args.projectProfiles.find((profile) => profile.projectId === profileId);
+    if (!projectProfile) {
+        throw new Error(`No benchmark project profile found for ${profileId}.`);
+    }
+    const answerKey = resolveAnswerKey(args.evaluationCase);
+    return {
+        evaluationCase: args.evaluationCase,
+        projectProfile,
+        answerKey,
+        fileTree: projectProfile.fileTree,
+        complexityLevel: args.complexityLevel,
+        strategy: args.strategy
+    };
+}
+function resolveAnswerKey(evaluationCase) {
+    if (evaluationCase.answerKey) {
+        return evaluationCase.answerKey;
+    }
+    return {
+        expectedFiles: evaluationCase.expectedFiles,
+        expectedSymbols: evaluationCase.expectedSymbols,
+        expectedFacts: evaluationCase.expectedFacts ?? [],
+        minimumCorrectFacts: 0,
+        notes: "Fallback answer key built from legacy evaluation case fields."
+    };
+}
+export function formatCompactFileTree(context) {
+    const maxEntries = context.complexityLevel === "short" ? 6 : context.complexityLevel === "medium" ? 12 : 24;
+    const entries = context.fileTree.entries
+        .filter((entry) => entry.kind === "file")
+        .slice(0, maxEntries)
+        .map((entry) => {
+        const details = [entry.role, entry.language, typeof entry.lines === "number" ? `${entry.lines} lines` : undefined].filter(Boolean).join(", ");
+        return `- ${entry.path}${details ? ` (${details})` : ""}`;
+    });
+    if (context.fileTree.entries.filter((entry) => entry.kind === "file").length > maxEntries) {
+        entries.push(`- ... ${context.fileTree.entries.filter((entry) => entry.kind === "file").length - maxEntries} more files omitted from preview`);
+    }
+    return entries.join("\n");
+}
+export function formatAnswerKeySummary(context) {
+    const facts = context.complexityLevel === "long" || context.complexityLevel === "multi-step"
+        ? context.answerKey.expectedFacts.map((fact) => `- ${fact.id}: ${fact.required ? "required" : "optional"}, weight ${fact.weight}`).join("\n")
+        : `Expected fact count: ${context.answerKey.expectedFacts.length}`;
+    return [
+        `Expected file count: ${context.answerKey.expectedFiles.length}`,
+        `Expected symbol count: ${context.answerKey.expectedSymbols.length}`,
+        `Minimum correct facts later required: ${context.answerKey.minimumCorrectFacts}`,
+        facts
+    ].join("\n");
+}
+export function formatExpectedOutputFields(strategy) {
+    const baseFields = [
+        "answer",
+        "relevantFiles",
+        "relevantSymbols",
+        "expectedFactsFound",
+        "confidence",
+        "tokenUsage, if available",
+        "tokenUsageSource: provider-reported, agent-reported, estimated, or unavailable",
+        "executionTime, if available",
+        "notes"
+    ];
+    const guidedFields = ["commandsRun", "selectedContext", "fullFileReads, if any", "fullFileReadJustifications, if any"];
+    return [...baseFields, ...(strategy === "my-dev-kit-guided" ? guidedFields : [])].map((field) => `- ${field}`).join("\n");
+}

package/dist/src/prompts/generateMyDevKitPrompt.js

@@ -0,0 +1,106 @@
+import { tokenCountMethod } from "../core/countTokens.js";
+import { formatAnswerKeySummary, formatCompactFileTree, formatExpectedOutputFields } from "./buildPromptContext.js";
+import { measurePromptComplexity } from "./measurePromptComplexity.js";
+export function generateMyDevKitPrompt(context) {
+    const promptText = buildMyDevKitPromptText(context);
+    return {
+        id: `${context.evaluationCase.id}.my-dev-kit-guided.${context.complexityLevel}`,
+        caseId: context.evaluationCase.id,
+        benchmarkProject: context.evaluationCase.benchmarkProject,
+        strategy: "my-dev-kit-guided",
+        complexityLevel: context.complexityLevel,
+        title: `${context.evaluationCase.title} - my-dev-kit guided - ${context.complexityLevel}`,
+        promptText,
+        promptMetrics: measurePromptComplexity(promptText, context),
+        expectedAnswerKey: context.answerKey,
+        projectProfile: context.projectProfile,
+        createdFrom: {
+            evaluationCaseId: context.evaluationCase.id,
+            projectProfileId: context.projectProfile.projectId,
+            tokenCountMethod
+        },
+        warnings: []
+    };
+}
+function buildMyDevKitPromptText(context) {
+    const base = [
+        "# my-dev-kit-Guided Benchmark Prompt",
+        "",
+        `Project: ${context.projectProfile.displayName}`,
+        `Project ID: ${context.projectProfile.projectId}`,
+        `Complexity: ${context.projectProfile.complexityLevel}, score ${context.projectProfile.complexityScore}`,
+        `Task: ${context.evaluationCase.title}`,
+        `Query: ${context.evaluationCase.query}`,
+        "",
+        "Instruction: gather targeted context using my-dev-kit.",
+        "Do not read full files by default. Read whole files only if targeted retrieval is insufficient, and explain why.",
+        "Use my-dev-kit index before retrieval.",
+        "Use my-dev-kit search to find candidate files or symbols.",
+        "Use my-dev-kit lookup to inspect selected nodes.",
+        "Use my-dev-kit slice where useful for nearby context.",
+        "Use my-dev-kit source for selected symbols or line ranges.",
+        "Report commandsRun, selected files, selected symbols, and selectedContext.",
+        "Return tokenUsage if the platform exposes it, and state tokenUsageSource as provider-reported, agent-reported, estimated, or unavailable.",
+        "Return executionTime if available.",
+        "",
+        "Expected Output Fields:",
+        formatExpectedOutputFields("my-dev-kit-guided")
+    ];
+    if (context.complexityLevel === "short") {
+        return [
+            ...base,
+            "",
+            `Expected file count: ${context.answerKey.expectedFiles.length}`,
+            `Expected symbol count: ${context.answerKey.expectedSymbols.length}`,
+            "Answer with concise reasoning."
+        ].join("\n");
+    }
+    const medium = [
+        ...base,
+        "",
+        `Description: ${context.projectProfile.description}`,
+        "",
+        "File Tree:",
+        formatCompactFileTree(context),
+        "",
+        "Answer with concise reasoning and cite the my-dev-kit commands that supplied context."
+    ];
+    if (context.complexityLevel === "medium") {
+        return medium.join("\n");
+    }
+    const long = [
+        ...medium,
+        "",
+        "Answer Key Summary:",
+        formatAnswerKeySummary(context),
+        "",
+        "Constraints:",
+        "- Do not create a full-file reading workflow by default.",
+        "- Do not invent files, symbols, commands, or behavior.",
+        "- Prefer search, lookup, slice, and source over broad reads.",
+        "- Include confidence as high, medium, or low.",
+        "- State whether token usage is provider-reported, agent-reported, estimated, or unavailable."
+    ];
+    if (context.complexityLevel === "long") {
+        return long.join("\n");
+    }
+    return [
+        ...long,
+        "",
+        "Workflow Steps:",
+        "1. Run my-dev-kit index for the target project.",
+        "2. Run my-dev-kit search with the benchmark query.",
+        "3. Run my-dev-kit lookup for promising files or symbols.",
+        "4. Run my-dev-kit slice where local context is needed.",
+        "5. Run my-dev-kit source for selected symbols or line ranges.",
+        "6. Avoid full-file reads unless targeted context is insufficient.",
+        "7. Answer the task and report selected files, selected symbols, commandsRun, and any fullFileReads.",
+        "",
+        "Validation Checklist:",
+        "- commandsRun includes index, search, lookup, and any slice/source commands used",
+        "- selectedContext explains why each file or symbol was chosen",
+        "- fullFileReads is empty unless justified",
+        "- expectedFactsFound uses fact IDs when known",
+        "- notes include uncertainty or missing telemetry"
+    ].join("\n");
+}

package/dist/src/prompts/generatePromptVariants.js

@@ -0,0 +1,36 @@
+import { buildPromptGenerationContext } from "./buildPromptContext.js";
+import { generateMyDevKitPrompt } from "./generateMyDevKitPrompt.js";
+import { generateRawFullFilePrompt } from "./generateRawFullFilePrompt.js";
+export const ALL_PROMPT_STRATEGIES = ["raw-full-file", "my-dev-kit-guided"];
+export const ALL_PROMPT_COMPLEXITY_LEVELS = ["short", "medium", "long", "multi-step"];
+export function generatePromptVariants(args) {
+    const strategies = args.strategies ?? ALL_PROMPT_STRATEGIES;
+    const complexityLevels = args.complexityLevels ?? ALL_PROMPT_COMPLEXITY_LEVELS;
+    const variants = [];
+    for (const evaluationCase of args.cases) {
+        for (const strategy of strategies) {
+            for (const complexityLevel of complexityLevels) {
+                const context = buildPromptGenerationContext({
+                    evaluationCase,
+                    projectProfiles: args.projectProfiles,
+                    strategy,
+                    complexityLevel
+                });
+                variants.push(strategy === "raw-full-file" ? generateRawFullFilePrompt(context) : generateMyDevKitPrompt(context));
+            }
+        }
+    }
+    return variants;
+}
+export function parsePromptStrategy(value) {
+    if (value === "raw-full-file" || value === "my-dev-kit-guided") {
+        return value;
+    }
+    throw new Error(`Invalid prompt strategy: ${value}`);
+}
+export function parsePromptComplexityLevel(value) {
+    if (value === "short" || value === "medium" || value === "long" || value === "multi-step") {
+        return value;
+    }
+    throw new Error(`Invalid prompt complexity level: ${value}`);
+}

package/dist/src/prompts/generateRawFullFilePrompt.js

@@ -0,0 +1,97 @@
+import { tokenCountMethod } from "../core/countTokens.js";
+import { formatAnswerKeySummary, formatCompactFileTree, formatExpectedOutputFields } from "./buildPromptContext.js";
+import { measurePromptComplexity } from "./measurePromptComplexity.js";
+export function generateRawFullFilePrompt(context) {
+    const promptText = buildRawPromptText(context);
+    return {
+        id: `${context.evaluationCase.id}.raw-full-file.${context.complexityLevel}`,
+        caseId: context.evaluationCase.id,
+        benchmarkProject: context.evaluationCase.benchmarkProject,
+        strategy: "raw-full-file",
+        complexityLevel: context.complexityLevel,
+        title: `${context.evaluationCase.title} - raw full-file - ${context.complexityLevel}`,
+        promptText,
+        promptMetrics: measurePromptComplexity(promptText, context),
+        expectedAnswerKey: context.answerKey,
+        projectProfile: context.projectProfile,
+        createdFrom: {
+            evaluationCaseId: context.evaluationCase.id,
+            projectProfileId: context.projectProfile.projectId,
+            tokenCountMethod
+        },
+        warnings: []
+    };
+}
+function buildRawPromptText(context) {
+    const base = [
+        "# Raw Full-File Benchmark Prompt",
+        "",
+        `Project: ${context.projectProfile.displayName}`,
+        `Project ID: ${context.projectProfile.projectId}`,
+        `Complexity: ${context.projectProfile.complexityLevel}, score ${context.projectProfile.complexityScore}`,
+        `Task: ${context.evaluationCase.title}`,
+        `Query: ${context.evaluationCase.query}`,
+        "",
+        "Instruction: use the full source files supplied separately by the runner as the primary context.",
+        "Do not use my-dev-kit as the required retrieval method for this strategy.",
+        "Identify the relevant files and relevant symbols before giving the answer.",
+        "Return tokenUsage if the platform exposes it, and state tokenUsageSource as provider-reported, agent-reported, estimated, or unavailable.",
+        "Return executionTime if available.",
+        "",
+        "Expected Output Fields:",
+        formatExpectedOutputFields("raw-full-file")
+    ];
+    if (context.complexityLevel === "short") {
+        return [
+            ...base,
+            "",
+            `Expected file count: ${context.answerKey.expectedFiles.length}`,
+            `Expected symbol count: ${context.answerKey.expectedSymbols.length}`,
+            "Answer with concise reasoning."
+        ].join("\n");
+    }
+    const medium = [
+        ...base,
+        "",
+        `Description: ${context.projectProfile.description}`,
+        "",
+        "File Tree:",
+        formatCompactFileTree(context),
+        "",
+        "Answer with concise reasoning and cite the files/symbols used."
+    ];
+    if (context.complexityLevel === "medium") {
+        return medium.join("\n");
+    }
+    const long = [
+        ...medium,
+        "",
+        "Answer Key Summary:",
+        formatAnswerKeySummary(context),
+        "",
+        "Constraints:",
+        "- Do not invent files, symbols, or behavior.",
+        "- Keep the answer tied to supplied source context.",
+        "- Include confidence as high, medium, or low.",
+        "- State whether token usage is provider-reported, agent-reported, estimated, or unavailable."
+    ];
+    if (context.complexityLevel === "long") {
+        return long.join("\n");
+    }
+    return [
+        ...long,
+        "",
+        "Workflow Steps:",
+        "1. Review the provided full-file context.",
+        "2. Identify candidate files and symbols.",
+        "3. Match the behavior against the benchmark task.",
+        "4. List expected facts found and any missing uncertainty.",
+        "5. Return the structured response fields.",
+        "",
+        "Validation Checklist:",
+        "- relevantFiles are project-relative paths",
+        "- relevantSymbols are concrete functions, classes, exports, or methods",
+        "- expectedFactsFound uses fact IDs when known",
+        "- notes include any uncertainty or missing telemetry"
+    ].join("\n");
+}

package/dist/src/prompts/index.js

@@ -0,0 +1,7 @@
+export * from "./types.js";
+export * from "./buildPromptContext.js";
+export * from "./generateRawFullFilePrompt.js";
+export * from "./generateMyDevKitPrompt.js";
+export * from "./measurePromptComplexity.js";
+export * from "./generatePromptVariants.js";
+export * from "./writePromptArtifacts.js";

package/dist/src/prompts/measurePromptComplexity.js

@@ -0,0 +1,41 @@
+import { countEstimatedTokens, tokenCountMethod } from "../core/countTokens.js";
+const OUTPUT_FIELD_NAMES = [
+    "answer",
+    "relevantFiles",
+    "relevantSymbols",
+    "expectedFactsFound",
+    "confidence",
+    "tokenUsage",
+    "tokenUsageSource",
+    "executionTime",
+    "notes",
+    "commandsRun",
+    "selectedContext",
+    "fullFileReads",
+    "fullFileReadJustifications"
+];
+export function measurePromptComplexity(promptText, context) {
+    return {
+        promptChars: promptText.length,
+        promptEstimatedTokens: countEstimatedTokens(promptText),
+        tokenCountMethod,
+        instructionCount: countMatches(promptText, /\b(instruction|use|return|identify|report|answer|state|include|do not)\b/gi),
+        constraintCount: countMatches(promptText, /\b(must|must not|do not|only|unless|avoid|required|default)\b/gi),
+        requestedOutputFieldCount: OUTPUT_FIELD_NAMES.filter((field) => promptText.includes(field)).length,
+        taskStepCount: countMatches(promptText, /^\d+\./gm),
+        expectedFactCount: context.answerKey.expectedFacts.length,
+        expectedFileCount: context.answerKey.expectedFiles.length,
+        expectedSymbolCount: context.answerKey.expectedSymbols.length,
+        includesFileTree: promptText.includes("File Tree"),
+        includesProjectDescription: promptText.includes(context.projectProfile.description),
+        includesAnswerKeySummary: promptText.includes("Answer Key Summary"),
+        requiresMultipleFiles: context.answerKey.expectedFiles.length > 1,
+        requiresGraphGuidedRetrieval: context.strategy === "my-dev-kit-guided",
+        requiresCommandExecution: context.strategy === "my-dev-kit-guided",
+        requiresTokenReport: promptText.includes("tokenUsage"),
+        requiresTimingReport: promptText.includes("executionTime")
+    };
+}
+function countMatches(text, pattern) {
+    return text.match(pattern)?.length ?? 0;
+}

package/dist/src/prompts/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/prompts/writePromptArtifacts.js

@@ -0,0 +1,43 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { tokenCountMethod } from "../core/countTokens.js";
+import { resolveWithinRoot } from "../core/pathSafety.js";
+export async function writePromptArtifacts(args) {
+    const outDir = path.resolve(args.outDir);
+    const promptDir = resolveWithinRoot(outDir, "prompts");
+    await mkdir(promptDir, { recursive: true });
+    const promptFiles = [];
+    for (const variant of args.variants) {
+        const fileName = promptVariantFileName(variant);
+        const fullPath = resolveWithinRoot(promptDir, fileName);
+        await writeFile(fullPath, variant.promptText, "utf8");
+        promptFiles.push(path.relative(outDir, fullPath).replace(/\\/g, "/"));
+    }
+    const summary = {
+        generatedAt: args.generatedAt ?? new Date(0).toISOString(),
+        caseCount: new Set(args.variants.map((variant) => variant.caseId)).size,
+        promptCount: args.variants.length,
+        strategies: unique(args.variants.map((variant) => variant.strategy)),
+        complexityLevels: unique(args.variants.map((variant) => variant.complexityLevel)),
+        tokenCountMethod,
+        outputPaths: {
+            summaryPath: "prompt-variants-summary.json",
+            variantsPath: "prompt-variants.json",
+            promptDirectory: "prompts",
+            promptFiles
+        },
+        warnings: args.variants.flatMap((variant) => variant.warnings)
+    };
+    await writeFile(resolveWithinRoot(outDir, "prompt-variants.json"), JSON.stringify(args.variants, null, 2), "utf8");
+    await writeFile(resolveWithinRoot(outDir, "prompt-variants-summary.json"), JSON.stringify(summary, null, 2), "utf8");
+    return summary;
+}
+export function promptVariantFileName(variant) {
+    return `${safeSegment(variant.caseId)}.${variant.strategy}.${variant.complexityLevel}.txt`;
+}
+function safeSegment(value) {
+    return value.replace(/[^a-zA-Z0-9._-]+/g, "-");
+}
+function unique(values) {
+    return [...new Set(values)].sort();
+}