@dailephd/my-dev-kit-lab 0.8.0 → 0.8.2

package/dist/src/evaluation/runMyDevKitRetrieval.js

@@ -3,6 +3,7 @@ import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/
 import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
 import { interpretToolVersionOutput } from "./indexSnapshot.js";
 import { buildRetrievalEvidenceFromCommands } from "./retrievalQuality/buildRetrievalEvidence.js";
+import { buildQueryStrategyEvidenceFromRetrievalEvidence } from "./retrievalQueryStrategyEvidence.js";
 const VERSION_PROBE_TIMEOUT_MS = 30_000;
 function parseJsonIfPossible(text) {
     try {
@@ -182,6 +183,7 @@ export async function buildMyDevKitIndex(options) {
             ...(options.excludePaths ?? []).flatMap((excludePath) => ["--exclude", excludePath]),
             "--out",
             options.indexDir,
+            ...(options.callGraph === true ? ["--call-graph"] : []),
             ...(mode.kind === "incremental" ? ["--incremental", "--refresh-scope", mode.refreshScope] : []),
             "--json"
         ]
@@ -239,11 +241,31 @@ export async function probeMyDevKitVersion(options) {
         });
     }
 }
+function selectRetrievalStrategyContextText(strategyId, outputs) {
+    const ordered = strategyId === "keyword-search"
+        ? [outputs.search]
+        : strategyId === "symbol-lookup"
+            ? [outputs.lookup, outputs.search]
+            : strategyId === "graph-neighborhood"
+                ? [outputs.slice, outputs.search]
+                : strategyId === "source-slice"
+                    ? [outputs.source, outputs.search]
+                    : [outputs.source, outputs.slice, outputs.lookup, outputs.search];
+    return ordered.find((text) => text && text.trim().length > 0) ?? "";
+}
+const STRATEGY_DOWNSTREAM_COMMANDS = {
+    "keyword-search": [],
+    "symbol-lookup": ["lookup"],
+    "graph-neighborhood": ["slice"],
+    "source-slice": ["source"],
+    "combined-graph-guided": ["lookup", "slice", "source"]
+};
 /**
- * Runs search -> lookup -> slice -> source against an already prepared index. Never invokes
- * `index`; `durationMs` covers retrieval only.
+ * Executes exactly one core retrieval query strategy against an already prepared index. Every
+ * strategy starts with `search` and expands at most the first candidate. Never invokes `index`;
+ * `durationMs` covers retrieval only.
  */
-export async function runMyDevKitRetrievalFromIndex(options) {
+export async function runMyDevKitRetrievalStrategyFromIndex(options) {
     const started = Date.now();
     const warnings = [];
     const commands = [];
@@ -274,58 +296,37 @@ export async function runMyDevKitRetrievalFromIndex(options) {
     const selectedNodeId = candidate.nodeId;
     const selectedFile = candidate.file;
     const selectedSymbol = candidate.symbol;
-    let lookupOutput = "";
-    let sliceOutput = "";
-    let sourceOutput = "";
-    if (selectedNodeId) {
-        const lookupCommand = await runMeasuredCommand({
-            commandId: "lookup",
-            commandString: options.kitCommand,
-            cwd: process.cwd(),
-            outDir: commandsDir,
-            extraArgs: ["lookup", "--index", indexDir, "--node", selectedNodeId, "--json"]
-        });
-        commands.push(lookupCommand);
-        if (lookupCommand.ok) {
-            lookupOutput = lookupCommand.stdout;
-        }
-        else {
-            warnings.push("my-dev-kit lookup command failed.");
-        }
-        const sliceCommand = await runMeasuredCommand({
-            commandId: "slice",
-            commandString: options.kitCommand,
-            cwd: process.cwd(),
-            outDir: commandsDir,
-            extraArgs: ["slice", "--index", indexDir, "--node", selectedNodeId, "--json"]
-        });
-        commands.push(sliceCommand);
-        if (sliceCommand.ok) {
-            sliceOutput = sliceCommand.stdout;
-        }
-        else {
-            warnings.push("my-dev-kit slice command failed.");
-        }
-        const sourceCommand = await runMeasuredCommand({
-            commandId: "source",
-            commandString: options.kitCommand,
-            cwd: process.cwd(),
-            outDir: commandsDir,
-            extraArgs: ["source", "--index", indexDir, "--node", selectedNodeId, "--max-lines", "160", "--format", "numbered"]
-        });
-        commands.push(sourceCommand);
-        if (sourceCommand.ok) {
-            sourceOutput = sourceCommand.stdout;
+    const outputs = { search: searchCommand.stdout, lookup: "", slice: "", source: "" };
+    const downstream = STRATEGY_DOWNSTREAM_COMMANDS[options.strategyId];
+    if (downstream.length > 0) {
+        if (selectedNodeId) {
+            for (const commandId of downstream) {
+                const extraArgs = commandId === "source"
+                    ? ["source", "--index", indexDir, "--node", selectedNodeId, "--max-lines", "160", "--format", "numbered"]
+                    : [commandId, "--index", indexDir, "--node", selectedNodeId, "--json"];
+                const command = await runMeasuredCommand({
+                    commandId,
+                    commandString: options.kitCommand,
+                    cwd: process.cwd(),
+                    outDir: commandsDir,
+                    extraArgs
+                });
+                commands.push(command);
+                if (command.ok) {
+                    outputs[commandId] = command.stdout;
+                }
+                else {
+                    warnings.push(`my-dev-kit ${commandId} command failed.`);
+                }
+            }
         }
         else {
-            warnings.push("my-dev-kit source command failed.");
+            warnings.push("No my-dev-kit node id was available after search.");
         }
     }
-    else {
-        warnings.push("No my-dev-kit node id was available after search.");
-    }
-    const contextText = [sourceOutput, sliceOutput, lookupOutput, searchCommand.stdout].find((text) => text && text.trim().length > 0) ?? "";
+    const contextText = selectRetrievalStrategyContextText(options.strategyId, outputs);
     const filesRead = selectedFile ? [selectedFile] : [];
+    const retrievalEvidence = buildRetrievalEvidenceFromCommands(commands, { nodeId: selectedNodeId, file: selectedFile });
     return {
         caseId: options.evaluationCase.id,
         skipped: contextText.length === 0,
@@ -339,10 +340,18 @@ export async function runMyDevKitRetrievalFromIndex(options) {
         selectedNodeId,
         selectedFile,
         selectedSymbol,
-        retrievalEvidence: buildRetrievalEvidenceFromCommands(commands, { nodeId: selectedNodeId, file: selectedFile }),
+        retrievalEvidence,
+        queryStrategyEvidence: buildQueryStrategyEvidenceFromRetrievalEvidence(options.strategyId, retrievalEvidence),
         durationMs: Date.now() - started
     };
 }
+/**
+ * v0.8.0 compatibility entrypoint: the historical search -> lookup -> slice -> source lifecycle,
+ * delegating to the `combined-graph-guided` strategy. Never invokes `index`.
+ */
+export async function runMyDevKitRetrievalFromIndex(options) {
+    return runMyDevKitRetrievalStrategyFromIndex({ ...options, strategyId: "combined-graph-guided" });
+}
 /**
  * Legacy per-case lifecycle: builds a case-specific index and retrieves from it. `commands`
  * starts with the index command and `durationMs` covers the whole lifecycle including indexing.

package/dist/src/evaluation/runSemanticRetrievalStrategy.js

@@ -0,0 +1,303 @@
+/**
+ * Semantic retrieval strategies (`data-model-graph`, `model-view-lineage`) executed against an
+ * already prepared my-dev-kit index through `data-model` queries.
+ *
+ * The target is chosen from the task query and the upstream data-model graph only; the answer key is
+ * never read. Output is normalized into `retrieval-query-strategy-evidence-v1`, never into
+ * `retrieval-evidence-v1`, because these commands are not search/lookup/slice/source.
+ */
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
+import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
+import { compareCodeUnits } from "./indexSnapshot.js";
+import { normalizeRetrievedRepositoryPath } from "./retrievalQuality/buildRetrievalEvidence.js";
+import { canonicalizeQueryStrategyIdentities } from "./retrievalQueryStrategyEvidence.js";
+const REJECTED_REASON = "some-semantic-identities-were-rejected";
+function isRecord(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function parseJsonIfPossible(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Lowercases, splits on every run of non ASCII alphanumerics, drops tokens shorter than two
+ * characters, deduplicates, and sorts in code-unit order. No stemming, synonyms, or stop words.
+ */
+export function tokenizeSemanticRetrievalQuery(query) {
+    const tokens = query
+        .toLowerCase()
+        .split(/[^A-Za-z0-9]+/)
+        .filter((token) => token.length >= 2);
+    return [...new Set(tokens)].sort(compareCodeUnits);
+}
+/** Count of unique query tokens that occur exactly as label tokens. */
+export function scoreSemanticLabel(queryTokens, label) {
+    const labelTokens = new Set(tokenizeSemanticRetrievalQuery(label));
+    let score = 0;
+    for (const token of new Set(queryTokens)) {
+        if (labelTokens.has(token))
+            score += 1;
+    }
+    return score;
+}
+function step(kind, succeeded, evidenceAvailable, reason) {
+    return { kind, succeeded, evidenceAvailable, reason };
+}
+function buildEvidence(strategyId, availability, availabilityReason, identities, steps) {
+    const canonical = canonicalizeQueryStrategyIdentities(identities.files.map((file) => ({ path: file })), identities.symbols);
+    return {
+        schemaVersion: "retrieval-query-strategy-evidence-v1",
+        strategyId,
+        availability,
+        availabilityReason,
+        files: canonical.files,
+        symbols: canonical.symbols,
+        steps
+    };
+}
+function unavailableEvidence(strategyId, reason, steps) {
+    return buildEvidence(strategyId, "unavailable", reason, { files: [], symbols: [] }, steps);
+}
+async function readDataModelGraph(indexDir) {
+    let text;
+    try {
+        text = await readFile(path.join(indexDir, "data-model-graph.json"), "utf8");
+    }
+    catch {
+        return { ok: false, reason: "data-model-graph-artifact-missing" };
+    }
+    const payload = parseJsonIfPossible(text);
+    if (!isRecord(payload))
+        return { ok: false, reason: "data-model-graph-artifact-malformed" };
+    if (payload.artifactKind !== "my-dev-kit-v1-data-model-graph" ||
+        typeof payload.schemaVersion !== "string" ||
+        !payload.schemaVersion.startsWith("1.")) {
+        return { ok: false, reason: "data-model-graph-schema-unsupported" };
+    }
+    if (!Array.isArray(payload.nodes) || !Array.isArray(payload.edges)) {
+        return { ok: false, reason: "data-model-graph-artifact-malformed" };
+    }
+    const nodes = [];
+    for (const node of payload.nodes) {
+        if (isRecord(node) &&
+            typeof node.id === "string" &&
+            typeof node.label === "string" &&
+            (node.kind === "entity" || node.kind === "field")) {
+            nodes.push({ id: node.id, kind: node.kind, label: node.label });
+        }
+    }
+    return { ok: true, nodes };
+}
+/** Highest score wins; ties break on node id in code-unit order. No candidate means no match. */
+function selectSemanticCandidate(queryTokens, nodes) {
+    let best;
+    for (const node of nodes) {
+        const score = scoreSemanticLabel(queryTokens, node.label);
+        if (score <= 0)
+            continue;
+        if (!best || score > best.score || (score === best.score && compareCodeUnits(node.id, best.node.id) < 0)) {
+            best = { node, score };
+        }
+    }
+    return best?.node;
+}
+/** Normalizes `filePath` of every ref; unsafe values are counted, never kept. */
+function collectSafeFiles(refs) {
+    const files = [];
+    let rejected = 0;
+    for (const ref of refs) {
+        const normalized = isRecord(ref) ? normalizeRetrievedRepositoryPath(ref.filePath) : null;
+        if (normalized === null) {
+            rejected += 1;
+        }
+        else {
+            files.push(normalized);
+        }
+    }
+    return { files, rejected };
+}
+function uniqueFileOrNull(files) {
+    const unique = [...new Set(files)];
+    return unique.length === 1 ? unique[0] : null;
+}
+function readEntityOutput(payload, expectedMode) {
+    if (!isRecord(payload) || payload.status !== "ok" || typeof payload.mode !== "string")
+        return "malformed";
+    if (payload.mode !== expectedMode)
+        return "mismatched";
+    return payload;
+}
+function readIdentity(value) {
+    return isRecord(value) && typeof value.id === "string" && typeof value.name === "string"
+        ? { id: value.id, name: value.name }
+        : undefined;
+}
+function buildDataModelGraphStrategyEvidence(selectedNode, lookupPayload, generationCommand, lookupCommand) {
+    const strategyId = "data-model-graph";
+    const generationStep = step("data-model", generationCommand.ok, generationCommand.ok, null);
+    const failed = (reason) => unavailableEvidence(strategyId, reason, [generationStep, step("data-model", lookupCommand.ok, false, reason)]);
+    if (!lookupCommand.ok)
+        return failed("data-model-lookup-command-failed");
+    const expectedMode = selectedNode.kind === "entity" ? "entity" : "field";
+    const output = readEntityOutput(lookupPayload, expectedMode);
+    if (output === "malformed")
+        return failed("data-model-lookup-output-malformed");
+    if (output === "mismatched")
+        return failed("data-model-lookup-output-mismatched");
+    const selected = readIdentity(expectedMode === "entity" ? output.entity : output.field);
+    if (!selected || !Array.isArray(output.sourceRefs))
+        return failed("data-model-lookup-output-malformed");
+    const { files, rejected } = collectSafeFiles(output.sourceRefs);
+    const symbol = {
+        name: selected.name,
+        nodeId: selected.id,
+        file: uniqueFileOrNull(files)
+    };
+    const partial = rejected > 0;
+    return buildEvidence(strategyId, partial ? "partial" : "available", partial ? REJECTED_REASON : null, { files, symbols: [symbol] }, [generationStep, step("data-model", true, !partial, partial ? REJECTED_REASON : null)]);
+}
+function buildModelViewLineageStrategyEvidence(tracePayload, generationCommand, traceCommand, expectedMode) {
+    const strategyId = "model-view-lineage";
+    const generationStep = step("data-model", generationCommand.ok, generationCommand.ok, null);
+    const failed = (reason) => unavailableEvidence(strategyId, reason, [generationStep, step("model-view-lineage", traceCommand.ok, false, reason)]);
+    if (!traceCommand.ok)
+        return failed("model-view-lineage-command-failed");
+    const output = readEntityOutput(tracePayload, expectedMode);
+    if (output === "malformed")
+        return failed("model-view-lineage-output-malformed");
+    if (output === "mismatched")
+        return failed("model-view-lineage-output-mismatched");
+    const lineage = output.lineage;
+    if (!readIdentity(output.entity) || !isRecord(lineage) || !Array.isArray(lineage.nodes) || !Array.isArray(lineage.edges)) {
+        return failed("model-view-lineage-output-malformed");
+    }
+    const files = [];
+    const symbols = [];
+    let rejected = 0;
+    for (const node of lineage.nodes) {
+        if (!isRecord(node) || typeof node.id !== "string" || typeof node.label !== "string") {
+            rejected += 1;
+            continue;
+        }
+        let nodeFiles = [];
+        if (Array.isArray(node.evidenceRefs)) {
+            const collected = collectSafeFiles(node.evidenceRefs);
+            nodeFiles = collected.files;
+            rejected += collected.rejected;
+        }
+        else if (node.evidenceRefs !== undefined) {
+            rejected += 1;
+        }
+        files.push(...nodeFiles);
+        symbols.push({ name: node.label, nodeId: node.id, file: uniqueFileOrNull(nodeFiles) });
+    }
+    const partial = rejected > 0;
+    return buildEvidence(strategyId, partial ? "partial" : "available", partial ? REJECTED_REASON : null, { files, symbols }, [generationStep, step("model-view-lineage", true, !partial, partial ? REJECTED_REASON : null)]);
+}
+async function runDataModelCommand(options, commandId, args, commands) {
+    const command = await runMeasuredCommand({
+        commandId,
+        commandString: options.kitCommand,
+        cwd: process.cwd(),
+        outDir: options.commandsDir,
+        extraArgs: ["data-model", "--index", options.indexDir, ...args, "--json"]
+    });
+    commands.push(command);
+    if (!command.ok && options.requireKit) {
+        throw new Error(command.error || `my-dev-kit data-model failed with exit code ${command.exitCode}`);
+    }
+    return command;
+}
+function buildResult(options, started, commands, evidence, contextText) {
+    const warnings = [];
+    if (evidence.availability === "unavailable") {
+        warnings.push(`my-dev-kit semantic retrieval was unavailable: ${evidence.availabilityReason}.`);
+    }
+    else if (evidence.availability === "partial") {
+        warnings.push(`my-dev-kit semantic retrieval was partially interpreted: ${evidence.availabilityReason}.`);
+    }
+    const text = evidence.availability === "unavailable" ? "" : contextText;
+    return {
+        caseId: options.evaluationCase.id,
+        strategyId: options.strategyId,
+        skipped: text.length === 0,
+        warnings,
+        totalChars: countTextChars(text),
+        totalEstimatedTokens: countEstimatedTokens(text),
+        tokenCountMethod,
+        contextText: text,
+        commands,
+        queryStrategyEvidence: evidence,
+        durationMs: Date.now() - started
+    };
+}
+/** Shared by both strategies: generate artifacts, validate the graph, select one semantic node. */
+async function resolveSemanticTarget(options, commands) {
+    const generation = await runDataModelCommand(options, "data-model", [], commands);
+    if (!generation.ok) {
+        return {
+            kind: "done",
+            evidence: unavailableEvidence(options.strategyId, "data-model-command-failed", [
+                step("data-model", false, false, "data-model-command-failed")
+            ])
+        };
+    }
+    const generationStep = step("data-model", true, true, null);
+    const graph = await readDataModelGraph(options.indexDir);
+    if (!graph.ok) {
+        return {
+            kind: "done",
+            evidence: unavailableEvidence(options.strategyId, graph.reason, [step("data-model", true, false, graph.reason)])
+        };
+    }
+    const node = selectSemanticCandidate(tokenizeSemanticRetrievalQuery(options.evaluationCase.query), graph.nodes);
+    if (!node) {
+        return {
+            kind: "done",
+            evidence: buildEvidence(options.strategyId, "available", null, { files: [], symbols: [] }, [generationStep])
+        };
+    }
+    return { kind: "node", node, generation };
+}
+async function executeDataModelGraphStrategy(options, started) {
+    const commands = [];
+    const resolved = await resolveSemanticTarget(options, commands);
+    if (resolved.kind === "done")
+        return buildResult(options, started, commands, resolved.evidence, "");
+    const selector = resolved.node.kind === "entity" ? ["--entity", resolved.node.label] : ["--field", resolved.node.label];
+    const lookup = await runDataModelCommand(options, "data-model-selected", selector, commands);
+    const evidence = buildDataModelGraphStrategyEvidence(resolved.node, lookup.ok ? parseJsonIfPossible(lookup.stdout) : undefined, resolved.generation, lookup);
+    return buildResult(options, started, commands, evidence, lookup.stdout);
+}
+async function executeModelViewLineageStrategy(options, started) {
+    const commands = [];
+    const resolved = await resolveSemanticTarget(options, commands);
+    if (resolved.kind === "done")
+        return buildResult(options, started, commands, resolved.evidence, "");
+    const isEntity = resolved.node.kind === "entity";
+    const selector = isEntity ? ["--trace-view", resolved.node.label] : ["--field", resolved.node.label, "--trace-view"];
+    const trace = await runDataModelCommand(options, "model-view-lineage", selector, commands);
+    const evidence = buildModelViewLineageStrategyEvidence(trace.ok ? parseJsonIfPossible(trace.stdout) : undefined, resolved.generation, trace, isEntity ? "trace-entity" : "trace-field");
+    return buildResult(options, started, commands, evidence, trace.stdout);
+}
+/**
+ * Executes one semantic retrieval strategy against an already prepared index. Never invokes `index`,
+ * never reads the answer key, never runs agents.
+ */
+export async function runSemanticRetrievalStrategyFromIndex(options) {
+    const started = Date.now();
+    switch (options.strategyId) {
+        case "data-model-graph":
+            return executeDataModelGraphStrategy(options, started);
+        case "model-view-lineage":
+            return executeModelViewLineageStrategy(options, started);
+        default:
+            throw new Error(`Unsupported semantic retrieval strategy: ${String(options.strategyId)}`);
+    }
+}

package/dist/src/experiments/defaultRegistry.js

@@ -1,8 +1,10 @@
 import { ExperimentPluginRegistry } from "./registry.js";
 import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyComparison/index.js";
+import { contextPackGenerationPlugin } from "./plugins/contextPackGeneration/index.js";
 import { contextWindowScalingPlugin } from "./plugins/contextWindowScaling/index.js";
 import { incrementalChangeStalenessPlugin } from "./plugins/incrementalChangeStaleness/index.js";
 import { retrievalPrecisionRecallPlugin } from "./plugins/retrievalPrecisionRecall/index.js";
+import { retrievalQueryStrategyComparisonPlugin } from "./plugins/retrievalQueryStrategyComparison/index.js";
 import { warmIndexReusePlugin } from "./plugins/warmIndexReuse/index.js";
 export function createDefaultExperimentPluginRegistry() {
     const registry = new ExperimentPluginRegistry();
@@ -11,5 +13,7 @@ export function createDefaultExperimentPluginRegistry() {
     registry.register(incrementalChangeStalenessPlugin);
     registry.register(contextWindowScalingPlugin);
     registry.register(retrievalPrecisionRecallPlugin);
+    registry.register(retrievalQueryStrategyComparisonPlugin);
+    registry.register(contextPackGenerationPlugin);
     return registry;
 }

package/dist/src/experiments/plugins/contextPackGeneration/analysis.js

@@ -0,0 +1,157 @@
+import { calculateRetrievalQualityMetricsFromIdentityEvidence } from "../../../evaluation/retrievalQuality/metrics.js";
+import { calculateBalancedIdentityF1 } from "../retrievalQueryStrategyComparison/analysis.js";
+import { RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS } from "../retrievalQueryStrategyComparison/analysisTypes.js";
+import { CONTEXT_PACK_GENERATION_TREATMENT_IDS } from "./metadata.js";
+const noMeasurementF1 = () => ({
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-measurement"
+});
+/** Scores already-completed treatment evidence with the frozen retrieval-quality science. It never influences retrieval. */
+export function analyzeContextPackGenerationTreatment(evaluationCase, treatment) {
+    if (treatment.identityEvidence === null || treatment.size === null) {
+        return {
+            treatmentId: treatment.treatmentId,
+            executionStatus: treatment.status,
+            availability: treatment.availability,
+            quality: null,
+            fileF1: noMeasurementF1(),
+            symbolF1: noMeasurementF1(),
+            estimatedTokens: null,
+            tokenCountMethod: null
+        };
+    }
+    const quality = calculateRetrievalQualityMetricsFromIdentityEvidence({
+        evaluationCase,
+        retrieval: {
+            identityEvidence: treatment.identityEvidence,
+            totalEstimatedTokens: treatment.size.totalEstimatedTokens,
+            tokenCountMethod: treatment.size.tokenCountMethod
+        }
+    });
+    return {
+        treatmentId: treatment.treatmentId,
+        executionStatus: treatment.status,
+        availability: treatment.availability,
+        quality,
+        fileF1: calculateBalancedIdentityF1(quality.file.relevantRetrievedFiles, quality.file.irrelevantRetrievedFiles, quality.file.missedFiles),
+        symbolF1: calculateBalancedIdentityF1(quality.symbol.relevantRetrievedSymbols, quality.symbol.irrelevantRetrievedSymbols, quality.symbol.missedSymbols),
+        estimatedTokens: treatment.size.totalEstimatedTokens,
+        tokenCountMethod: treatment.size.tokenCountMethod
+    };
+}
+/**
+ * Same arithmetic as the existing repository token-savings comparison (tokensSaved = raw - other; percentSaved =
+ * tokensSaved / raw * 100, 0 when raw is 0). Negative savings are valid and never clamped.
+ */
+export function calculateContextPackTokenSavings(rawTokens, packTokens) {
+    const tokensSaved = rawTokens - packTokens;
+    return { tokensSaved, percentSaved: rawTokens === 0 ? 0 : (tokensSaved / rawTokens) * 100 };
+}
+const valueOf = (metric) => (metric && metric.availability === "available" ? metric.value : null);
+function compareTreatments(raw, pack) {
+    const delta = (packValue, rawValue) => (packValue === null || rawValue === null ? null : packValue - rawValue);
+    const factPack = valueOf(pack.quality?.fact.coverage);
+    const factRaw = valueOf(raw.quality?.fact.coverage);
+    const sizes = raw.estimatedTokens !== null && pack.estimatedTokens !== null ? calculateContextPackTokenSavings(raw.estimatedTokens, pack.estimatedTokens) : null;
+    return {
+        fileF1Delta: delta(valueOf(pack.fileF1), valueOf(raw.fileF1)),
+        symbolF1Delta: delta(valueOf(pack.symbolF1), valueOf(raw.symbolF1)),
+        factCoverageDelta: delta(factPack, factRaw),
+        estimatedTokenDelta: delta(pack.estimatedTokens, raw.estimatedTokens),
+        tokensSaved: sizes?.tokensSaved ?? null,
+        percentSaved: sizes?.percentSaved ?? null
+    };
+}
+export function analyzeContextPackGenerationCase(evaluationCase, executionCase) {
+    if (evaluationCase.id !== executionCase.caseId)
+        throw new Error("Context pack generation analysis case identity mismatch.");
+    const order = executionCase.treatments.map((treatment) => treatment.treatmentId);
+    if (order.length !== CONTEXT_PACK_GENERATION_TREATMENT_IDS.length || order.some((id, index) => id !== CONTEXT_PACK_GENERATION_TREATMENT_IDS[index])) {
+        throw new Error("Context pack generation analysis requires the two canonical treatments in canonical order.");
+    }
+    const treatments = executionCase.treatments.map((treatment) => analyzeContextPackGenerationTreatment(evaluationCase, treatment));
+    return {
+        caseId: executionCase.caseId,
+        benchmarkProject: executionCase.benchmarkProject,
+        taskLocality: executionCase.taskLocality,
+        treatments,
+        comparison: compareTreatments(treatments[0], treatments[1])
+    };
+}
+function isComparable(treatment) {
+    const quality = treatment.quality;
+    return (quality !== null &&
+        treatment.fileF1.availability === "available" &&
+        treatment.symbolF1.availability === "available" &&
+        quality.fact.coverage.availability === "available" &&
+        typeof quality.retrievedTokenCount === "number" &&
+        Number.isFinite(quality.retrievedTokenCount) &&
+        quality.retrievedTokenCount >= 0);
+}
+/** Matched design: a case counts only when BOTH treatments expose all four required measures. */
+export function isContextPackCaseComparable(caseAnalysis) {
+    return caseAnalysis.treatments.length === CONTEXT_PACK_GENERATION_TREATMENT_IDS.length && caseAnalysis.treatments.every(isComparable);
+}
+const mean = (values) => values.reduce((sum, value) => sum + value, 0) / values.length;
+function casesForScope(cases, scopeId) {
+    return scopeId === "overall" ? [...cases] : cases.filter((entry) => entry.taskLocality === scopeId);
+}
+export function aggregateContextPackGenerationScope(scopeId, cases) {
+    const scopeCases = casesForScope(cases, scopeId);
+    const included = scopeCases.filter(isContextPackCaseComparable);
+    const excluded = scopeCases.filter((entry) => !isContextPackCaseComparable(entry));
+    const treatmentSummaries = CONTEXT_PACK_GENERATION_TREATMENT_IDS.map((treatmentId, index) => {
+        const countStatus = (status) => scopeCases.filter((entry) => entry.treatments[index].executionStatus === status).length;
+        const rows = included.map((entry) => entry.treatments[index]);
+        return {
+            treatmentId,
+            completedCaseCount: countStatus("completed"),
+            partialCaseCount: countStatus("partial"),
+            failedCaseCount: countStatus("failed"),
+            objectives: rows.length === 0
+                ? null
+                : {
+                    meanFileF1: mean(rows.map((row) => row.fileF1.value)),
+                    meanSymbolF1: mean(rows.map((row) => row.symbolF1.value)),
+                    meanFactCoverage: mean(rows.map((row) => row.quality.fact.coverage.value)),
+                    meanEstimatedTokenCount: mean(rows.map((row) => row.quality.retrievedTokenCount))
+                }
+        };
+    });
+    const [rawSummary, packSummary] = treatmentSummaries;
+    const rawObjectives = rawSummary.objectives;
+    const packObjectives = packSummary.objectives;
+    const paired = rawObjectives && packObjectives;
+    return {
+        scopeId,
+        caseCount: scopeCases.length,
+        includedCaseCount: included.length,
+        excludedCaseCount: excluded.length,
+        includedCaseIds: included.map((entry) => entry.caseId),
+        excludedCaseIds: excluded.map((entry) => entry.caseId),
+        treatmentSummaries,
+        pairedDeltas: paired
+            ? {
+                meanFileF1Delta: mean(included.map((entry) => entry.comparison.fileF1Delta)),
+                meanSymbolF1Delta: mean(included.map((entry) => entry.comparison.symbolF1Delta)),
+                meanFactCoverageDelta: mean(included.map((entry) => entry.comparison.factCoverageDelta)),
+                meanEstimatedTokenDelta: mean(included.map((entry) => entry.comparison.estimatedTokenDelta))
+            }
+            : null,
+        tokenSavings: paired
+            ? {
+                meanTokensSaved: mean(included.map((entry) => entry.comparison.tokensSaved)),
+                percentSavedOfMeans: calculateContextPackTokenSavings(rawObjectives.meanEstimatedTokenCount, packObjectives.meanEstimatedTokenCount).percentSaved
+            }
+            : null
+    };
+}
+export function analyzeContextPackGeneration(evaluationCases, executionCases) {
+    if (evaluationCases.length !== executionCases.length)
+        throw new Error("Context pack generation analysis requires one execution case per evaluation case.");
+    const cases = evaluationCases.map((evaluationCase, index) => analyzeContextPackGenerationCase(evaluationCase, executionCases[index]));
+    return { cases, scopes: RETRIEVAL_QUERY_STRATEGY_COMPARISON_SCOPE_IDS.map((scopeId) => aggregateContextPackGenerationScope(scopeId, cases)) };
+}

package/dist/src/experiments/plugins/contextPackGeneration/analysisArtifact.js

@@ -0,0 +1,22 @@
+export const CONTEXT_PACK_GENERATION_ANALYSIS_ARTIFACT_FILE = "context-pack-generation-analysis.json";
+export const CONTEXT_PACK_GENERATION_ANALYSIS_SCHEMA_VERSION = "my-dev-kit-lab-context-pack-generation-analysis-v1";
+/** Single owner of the frozen methodology. No ranking, winner, composite score or Pareto front exists for this experiment. */
+export const CONTEXT_PACK_GENERATION_METHODOLOGY = {
+    fileF1: "balanced-f1",
+    symbolF1: "balanced-f1",
+    aggregation: "matched-complete-case-macro-mean",
+    treatmentComparison: "paired-descriptive-delta",
+    sizeMeasure: "estimated-tokens-of-rendered-text"
+};
+export function buildContextPackGenerationAnalysisArtifact(args) {
+    return {
+        schemaVersion: CONTEXT_PACK_GENERATION_ANALYSIS_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        methodology: { ...CONTEXT_PACK_GENERATION_METHODOLOGY },
+        analysis: structuredClone(args.analysis)
+    };
+}

package/dist/src/experiments/plugins/contextPackGeneration/analysisTypes.js

@@ -0,0 +1 @@
+export {};