@dailephd/my-dev-kit-lab 0.8.1 → 0.9.0

package/dist/src/experiments/plugins/agentSuccessRate/repairPolicy.js

@@ -0,0 +1,88 @@
+/**
+ * Pure repair policy: decides how one evaluated attempt ended and whether a repair may follow. It performs no
+ * filesystem, subprocess or provider work and reads only normalized execution evidence plus already-calculated metrics.
+ */
+/** Categories for which a completed provider attempt produced an evaluable implementation failure. */
+export const AGENT_SUCCESS_REPAIR_ELIGIBLE_CATEGORIES = [
+    "patch-malformed",
+    "patch-policy-rejected",
+    "patch-check-failed",
+    "patch-apply-failed",
+    "task-check-failed",
+    "regression-failed",
+    "required-behavior-unsatisfied"
+];
+function booleanValue(metric) {
+    return metric !== undefined && metric.availability === "available" && typeof metric.value === "boolean" ? metric.value : null;
+}
+/** Classifies an attempt. Infrastructure and provider-health outcomes are never eligible, regardless of task success. */
+export function classifyAgentSuccessAttempt(evidence, metrics) {
+    const taskSuccess = booleanValue(metrics.taskSuccess);
+    const result = (category) => ({
+        category,
+        repairEligible: AGENT_SUCCESS_REPAIR_ELIGIBLE_CATEGORIES.includes(category),
+        taskSuccess
+    });
+    const real = evidence.realAgent;
+    if (evidence.availability === "infrastructure-failure")
+        return result("infrastructure-failure");
+    if (evidence.availability === "baseline-invalid" || evidence.baselineAssessment === null || !evidence.baselineAssessment.evaluable)
+        return result("baseline-invalid");
+    if (real === undefined)
+        return result("evidence-incomplete");
+    if (real.context.availability === "unavailable" && !real.providerInvoked)
+        return result("context-unavailable");
+    if (!real.providerInvoked)
+        return result("provider-not-invoked");
+    switch (real.providerStatus) {
+        case "agent-unavailable":
+            return result("provider-unavailable");
+        case "agent-limit-reached":
+            return result("provider-limit-reached");
+        case "timeout":
+            return result("provider-timeout");
+        case "completed":
+            break;
+        default:
+            return result("provider-failed");
+    }
+    if (!real.finalAnswerAvailable || !evidence.patch.attempted)
+        return result("provider-empty-answer");
+    if (!evidence.cleanup.removed || (real.cwdCleanup.attempted && !real.cwdCleanup.removed))
+        return result("cleanup-failed");
+    if (evidence.protectedIntegrity.status === "unproven")
+        return result("integrity-unproven");
+    if (evidence.availability !== "complete")
+        return result("evidence-incomplete");
+    switch (evidence.patch.outcome) {
+        case "parse-failure":
+            return result("patch-malformed");
+        case "policy-rejection":
+            return result("patch-policy-rejected");
+        case "git-check-failure":
+            return result("patch-check-failed");
+        case "git-apply-failure":
+            return result("patch-apply-failed");
+        case "success":
+            break;
+        default:
+            return result("evidence-incomplete");
+    }
+    if (evidence.protectedIntegrity.status === "mutated")
+        return result("integrity-mutated");
+    if (taskSuccess === true)
+        return result("none");
+    if (taskSuccess === null)
+        return result("evidence-incomplete");
+    if (booleanValue(metrics.taskChecksPassed) === false)
+        return result("task-check-failed");
+    if (booleanValue(metrics.regressionSafe) === false)
+        return result("regression-failed");
+    if (booleanValue(metrics.requiredFactsSatisfied) === false)
+        return result("required-behavior-unsatisfied");
+    return result("task-check-failed");
+}
+/** A repair runs only after an eligible failure, and only while the allowance and the hard total bound both remain. */
+export function shouldRepairAgentSuccessAttempt(args) {
+    return args.classification.repairEligible && args.classification.taskSuccess === false && args.attemptsExecuted <= args.repairAttemptsAllowed && args.attemptsExecuted < 3;
+}

package/dist/src/experiments/plugins/agentSuccessRate/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/experiments/plugins/contextPackGeneration/analysis.js

@@ -0,0 +1,157 @@
+import { calculateRetrievalQualityMetricsFromIdentityEvidence } from "../../../evaluation/retrievalQuality/metrics.js";
+import { calculateBalancedIdentityF1 } from "../retrievalQueryStrategyComparison/analysis.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS } from "../retrievalQueryStrategyComparison/analysisTypes.js";
+import { CONTEXT_PACK_GENERATION_TREATMENT_IDS } from "./metadata.js";
+const noMeasurementF1 = () => ({
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-measurement"
+});
+/** Scores already-completed treatment evidence with the frozen retrieval-quality science. It never influences retrieval. */
+export function analyzeContextPackGenerationTreatment(evaluationCase, treatment) {
+    if (treatment.identityEvidence === null || treatment.size === null) {
+        return {
+            treatmentId: treatment.treatmentId,
+            executionStatus: treatment.status,
+            availability: treatment.availability,
+            quality: null,
+            fileF1: noMeasurementF1(),
+            symbolF1: noMeasurementF1(),
+            estimatedTokens: null,
+            tokenCountMethod: null
+        };
+    }
+    const quality = calculateRetrievalQualityMetricsFromIdentityEvidence({
+        evaluationCase,
+        retrieval: {
+            identityEvidence: treatment.identityEvidence,
+            totalEstimatedTokens: treatment.size.totalEstimatedTokens,
+            tokenCountMethod: treatment.size.tokenCountMethod
+        }
+    });
+    return {
+        treatmentId: treatment.treatmentId,
+        executionStatus: treatment.status,
+        availability: treatment.availability,
+        quality,
+        fileF1: calculateBalancedIdentityF1(quality.file.relevantRetrievedFiles, quality.file.irrelevantRetrievedFiles, quality.file.missedFiles),
+        symbolF1: calculateBalancedIdentityF1(quality.symbol.relevantRetrievedSymbols, quality.symbol.irrelevantRetrievedSymbols, quality.symbol.missedSymbols),
+        estimatedTokens: treatment.size.totalEstimatedTokens,
+        tokenCountMethod: treatment.size.tokenCountMethod
+    };
+}
+/**
+ * Same arithmetic as the existing repository token-savings comparison (tokensSaved = raw - other; percentSaved =
+ * tokensSaved / raw * 100, 0 when raw is 0). Negative savings are valid and never clamped.
+ */
+export function calculateContextPackTokenSavings(rawTokens, packTokens) {
+    const tokensSaved = rawTokens - packTokens;
+    return { tokensSaved, percentSaved: rawTokens === 0 ? 0 : (tokensSaved / rawTokens) * 100 };
+}
+const valueOf = (metric) => (metric && metric.availability === "available" ? metric.value : null);
+function compareTreatments(raw, pack) {
+    const delta = (packValue, rawValue) => (packValue === null || rawValue === null ? null : packValue - rawValue);
+    const factPack = valueOf(pack.quality?.fact.coverage);
+    const factRaw = valueOf(raw.quality?.fact.coverage);
+    const sizes = raw.estimatedTokens !== null && pack.estimatedTokens !== null ? calculateContextPackTokenSavings(raw.estimatedTokens, pack.estimatedTokens) : null;
+    return {
+        fileF1Delta: delta(valueOf(pack.fileF1), valueOf(raw.fileF1)),
+        symbolF1Delta: delta(valueOf(pack.symbolF1), valueOf(raw.symbolF1)),
+        factCoverageDelta: delta(factPack, factRaw),
+        estimatedTokenDelta: delta(pack.estimatedTokens, raw.estimatedTokens),
+        tokensSaved: sizes?.tokensSaved ?? null,
+        percentSaved: sizes?.percentSaved ?? null
+    };
+}
+export function analyzeContextPackGenerationCase(evaluationCase, executionCase) {
+    if (evaluationCase.id !== executionCase.caseId)
+        throw new Error("Context pack generation analysis case identity mismatch.");
+    const order = executionCase.treatments.map((treatment) => treatment.treatmentId);
+    if (order.length !== CONTEXT_PACK_GENERATION_TREATMENT_IDS.length || order.some((id, index) => id !== CONTEXT_PACK_GENERATION_TREATMENT_IDS[index])) {
+        throw new Error("Context pack generation analysis requires the two canonical treatments in canonical order.");
+    }
+    const treatments = executionCase.treatments.map((treatment) => analyzeContextPackGenerationTreatment(evaluationCase, treatment));
+    return {
+        caseId: executionCase.caseId,
+        benchmarkProject: executionCase.benchmarkProject,
+        taskLocality: executionCase.taskLocality,
+        treatments,
+        comparison: compareTreatments(treatments[0], treatments[1])
+    };
+}
+function isComparable(treatment) {
+    const quality = treatment.quality;
+    return (quality !== null &&
+        treatment.fileF1.availability === "available" &&
+        treatment.symbolF1.availability === "available" &&
+        quality.fact.coverage.availability === "available" &&
+        typeof quality.retrievedTokenCount === "number" &&
+        Number.isFinite(quality.retrievedTokenCount) &&
+        quality.retrievedTokenCount >= 0);
+}
+/** Matched design: a case counts only when BOTH treatments expose all four required measures. */
+export function isContextPackCaseComparable(caseAnalysis) {
+    return caseAnalysis.treatments.length === CONTEXT_PACK_GENERATION_TREATMENT_IDS.length && caseAnalysis.treatments.every(isComparable);
+}
+const mean = (values) => values.reduce((sum, value) => sum + value, 0) / values.length;
+function casesForScope(cases, scopeId) {
+    return scopeId === "overall" ? [...cases] : cases.filter((entry) => entry.taskLocality === scopeId);
+}
+export function aggregateContextPackGenerationScope(scopeId, cases) {
+    const scopeCases = casesForScope(cases, scopeId);
+    const included = scopeCases.filter(isContextPackCaseComparable);
+    const excluded = scopeCases.filter((entry) => !isContextPackCaseComparable(entry));
+    const treatmentSummaries = CONTEXT_PACK_GENERATION_TREATMENT_IDS.map((treatmentId, index) => {
+        const countStatus = (status) => scopeCases.filter((entry) => entry.treatments[index].executionStatus === status).length;
+        const rows = included.map((entry) => entry.treatments[index]);
+        return {
+            treatmentId,
+            completedCaseCount: countStatus("completed"),
+            partialCaseCount: countStatus("partial"),
+            failedCaseCount: countStatus("failed"),
+            objectives: rows.length === 0
+                ? null
+                : {
+                    meanFileF1: mean(rows.map((row) => row.fileF1.value)),
+                    meanSymbolF1: mean(rows.map((row) => row.symbolF1.value)),
+                    meanFactCoverage: mean(rows.map((row) => row.quality.fact.coverage.value)),
+                    meanEstimatedTokenCount: mean(rows.map((row) => row.quality.retrievedTokenCount))
+                }
+        };
+    });
+    const [rawSummary, packSummary] = treatmentSummaries;
+    const rawObjectives = rawSummary.objectives;
+    const packObjectives = packSummary.objectives;
+    const paired = rawObjectives && packObjectives;
+    return {
+        scopeId,
+        caseCount: scopeCases.length,
+        includedCaseCount: included.length,
+        excludedCaseCount: excluded.length,
+        includedCaseIds: included.map((entry) => entry.caseId),
+        excludedCaseIds: excluded.map((entry) => entry.caseId),
+        treatmentSummaries,
+        pairedDeltas: paired
+            ? {
+                meanFileF1Delta: mean(included.map((entry) => entry.comparison.fileF1Delta)),
+                meanSymbolF1Delta: mean(included.map((entry) => entry.comparison.symbolF1Delta)),
+                meanFactCoverageDelta: mean(included.map((entry) => entry.comparison.factCoverageDelta)),
+                meanEstimatedTokenDelta: mean(included.map((entry) => entry.comparison.estimatedTokenDelta))
+            }
+            : null,
+        tokenSavings: paired
+            ? {
+                meanTokensSaved: mean(included.map((entry) => entry.comparison.tokensSaved)),
+                percentSavedOfMeans: calculateContextPackTokenSavings(rawObjectives.meanEstimatedTokenCount, packObjectives.meanEstimatedTokenCount).percentSaved
+            }
+            : null
+    };
+}
+export function analyzeContextPackGeneration(evaluationCases, executionCases) {
+    if (evaluationCases.length !== executionCases.length)
+        throw new Error("Context pack generation analysis requires one execution case per evaluation case.");
+    const cases = evaluationCases.map((evaluationCase, index) => analyzeContextPackGenerationCase(evaluationCase, executionCases[index]));
+    return { cases, scopes: RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS.map((scopeId) => aggregateContextPackGenerationScope(scopeId, cases)) };
+}

package/dist/src/experiments/plugins/contextPackGeneration/analysisArtifact.js

@@ -0,0 +1,22 @@
+export const CONTEXT_PACK_GENERATION_ANALYSIS_ARTIFACT_FILE = "context-pack-generation-analysis.json";
+export const CONTEXT_PACK_GENERATION_ANALYSIS_SCHEMA_VERSION = "my-dev-kit-lab-context-pack-generation-analysis-v1";
+/** Single owner of the frozen methodology. No ranking, winner, composite score or Pareto front exists for this experiment. */
+export const CONTEXT_PACK_GENERATION_METHODOLOGY = {
+    fileF1: "balanced-f1",
+    symbolF1: "balanced-f1",
+    aggregation: "matched-complete-case-macro-mean",
+    treatmentComparison: "paired-descriptive-delta",
+    sizeMeasure: "estimated-tokens-of-rendered-text"
+};
+export function buildContextPackGenerationAnalysisArtifact(args) {
+    return {
+        schemaVersion: CONTEXT_PACK_GENERATION_ANALYSIS_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        methodology: { ...CONTEXT_PACK_GENERATION_METHODOLOGY },
+        analysis: structuredClone(args.analysis)
+    };
+}

package/dist/src/experiments/plugins/contextPackGeneration/analysisTypes.js

@@ -0,0 +1 @@
+export {};

package/dist/src/experiments/plugins/contextPackGeneration/buildContextPack.js

@@ -0,0 +1,84 @@
+import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../../../core/countTokens.js";
+import { compareCodeUnits } from "../../../evaluation/indexSnapshot.js";
+import { buildContextPackSelectionPolicy, selectCallRelationships, selectRelevantFiles, selectRelevantSymbols, selectSourceSlices, selectTestFiles } from "./packSelectionPolicy.js";
+import { renderContextPack } from "./renderContextPack.js";
+import { CONTEXT_PACK_SCHEMA_VERSION, CONTEXT_PACK_SECTION_IDS } from "./types.js";
+/**
+ * Pure context-pack composition. No filesystem, process, environment, network, clock or randomness: identical input
+ * yields structurally identical output. It selects within the frozen caps, renders deterministically, and measures size
+ * from the exact rendered text.
+ */
+export function buildContextPack(input) {
+    const files = selectRelevantFiles(input.fileCandidates);
+    const symbols = selectRelevantSymbols(input.symbolCandidates);
+    const sourceSlices = selectSourceSlices(input.sourceSliceCandidates);
+    const calls = selectCallRelationships(input.callCandidates);
+    const tests = selectTestFiles(input.testCandidates);
+    const evidenceNotes = [
+        ...new Set([
+            ...(input.evidenceNotes ?? []),
+            ...files.notes,
+            ...symbols.notes,
+            ...sourceSlices.notes,
+            ...calls.notes,
+            ...tests.notes
+        ])
+    ].sort(compareCodeUnits);
+    const sectionAvailability = {
+        task: { availability: "available", reason: null },
+        files: { ...input.sectionAvailability.files },
+        symbols: { ...input.sectionAvailability.symbols },
+        sourceSlices: { ...input.sectionAvailability.sourceSlices },
+        callRelationships: { ...input.sectionAvailability.callRelationships },
+        tests: { ...input.sectionAvailability.tests },
+        evidenceNotes: { ...input.sectionAvailability.evidenceNotes }
+    };
+    const itemCounts = {
+        task: 1,
+        files: files.items.length,
+        symbols: symbols.items.length,
+        sourceSlices: sourceSlices.items.length,
+        callRelationships: calls.items.length,
+        tests: tests.items.length,
+        evidenceNotes: evidenceNotes.length
+    };
+    const rendered = renderContextPack({
+        task: input.task,
+        files: files.items,
+        symbols: symbols.items,
+        sourceSlices: sourceSlices.items,
+        callRelationships: calls.items,
+        tests: tests.items,
+        evidenceNotes,
+        sectionAvailability
+    });
+    return {
+        schemaVersion: CONTEXT_PACK_SCHEMA_VERSION,
+        caseId: input.caseId,
+        benchmarkProject: input.benchmarkProject,
+        taskLocality: input.taskLocality,
+        task: { title: input.task.title, summary: input.task.summary },
+        selectionPolicy: buildContextPackSelectionPolicy(input.myDevKitVersion),
+        files: files.items,
+        symbols: symbols.items,
+        sourceSlices: sourceSlices.items,
+        callRelationships: calls.items,
+        tests: tests.items,
+        evidenceNotes,
+        sections: CONTEXT_PACK_SECTION_IDS.map((id) => ({
+            id,
+            availability: sectionAvailability[id].availability,
+            reason: sectionAvailability[id].reason,
+            itemCount: itemCounts[id],
+            estimatedTokens: countEstimatedTokens(rendered.sections[id])
+        })),
+        size: {
+            totalChars: countTextChars(rendered.renderedText),
+            totalEstimatedTokens: countEstimatedTokens(rendered.renderedText),
+            tokenCountMethod
+        },
+        availability: input.availability,
+        reason: input.reason,
+        renderedText: rendered.renderedText
+    };
+}

package/dist/src/experiments/plugins/contextPackGeneration/config.js

@@ -0,0 +1,39 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultContextPackGenerationConfig = {
+    outDir: "lab-output/context-pack-generation",
+    kitCommand: "npx @dailephd/my-dev-kit@latest"
+};
+const SUPPORTED_FIELDS = ["outDir", "kitCommand", "caseIds", "benchmarkProjects"];
+export const contextPackGenerationConfigDefinition = {
+    fields: [
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
+        { name: "caseIds", type: "array", description: "Bundled evaluation case IDs to include; corpus order is preserved." },
+        { name: "benchmarkProjects", type: "array", description: "Bundled benchmark project IDs to include; corpus order is preserved." }
+    ]
+};
+export function validateContextPackGenerationConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["context pack generation config must be an object."]);
+    }
+    const errors = [];
+    const unsupported = Object.keys(config ?? {}).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    if (unsupported.length > 0) {
+        errors.push(`Unsupported context-pack-generation config field(s): ${unsupported.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultContextPackGenerationConfig, config);
+    for (const field of ["outDir", "kitCommand"]) {
+        const value = normalized[field];
+        if (typeof value !== "string" || !value.trim())
+            errors.push(`${field} must be a non-empty string.`);
+    }
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+    }
+    return errors.length > 0 ? invalidExperimentConfig(errors) : validExperimentConfig(normalized);
+}