@dailephd/my-dev-kit-lab 0.7.2 → 0.8.1

package/dist/src/evaluation/runSemanticRetrievalStrategy.js

@@ -0,0 +1,303 @@
+/**
+ * Semantic retrieval strategies (`data-model-graph`, `model-view-lineage`) executed against an
+ * already prepared my-dev-kit index through `data-model` queries.
+ *
+ * The target is chosen from the task query and the upstream data-model graph only; the answer key is
+ * never read. Output is normalized into `retrieval-query-strategy-evidence-v1`, never into
+ * `retrieval-evidence-v1`, because these commands are not search/lookup/slice/source.
+ */
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
+import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
+import { compareCodeUnits } from "./indexSnapshot.js";
+import { normalizeRetrievedRepositoryPath } from "./retrievalQuality/buildRetrievalEvidence.js";
+import { canonicalizeQueryStrategyIdentities } from "./retrievalQueryStrategyEvidence.js";
+const REJECTED_REASON = "some-semantic-identities-were-rejected";
+function isRecord(value) {
+    return !!value && typeof value === "object" && !Array.isArray(value);
+}
+function parseJsonIfPossible(text) {
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Lowercases, splits on every run of non ASCII alphanumerics, drops tokens shorter than two
+ * characters, deduplicates, and sorts in code-unit order. No stemming, synonyms, or stop words.
+ */
+export function tokenizeSemanticRetrievalQuery(query) {
+    const tokens = query
+        .toLowerCase()
+        .split(/[^A-Za-z0-9]+/)
+        .filter((token) => token.length >= 2);
+    return [...new Set(tokens)].sort(compareCodeUnits);
+}
+/** Count of unique query tokens that occur exactly as label tokens. */
+export function scoreSemanticLabel(queryTokens, label) {
+    const labelTokens = new Set(tokenizeSemanticRetrievalQuery(label));
+    let score = 0;
+    for (const token of new Set(queryTokens)) {
+        if (labelTokens.has(token))
+            score += 1;
+    }
+    return score;
+}
+function step(kind, succeeded, evidenceAvailable, reason) {
+    return { kind, succeeded, evidenceAvailable, reason };
+}
+function buildEvidence(strategyId, availability, availabilityReason, identities, steps) {
+    const canonical = canonicalizeQueryStrategyIdentities(identities.files.map((file) => ({ path: file })), identities.symbols);
+    return {
+        schemaVersion: "retrieval-query-strategy-evidence-v1",
+        strategyId,
+        availability,
+        availabilityReason,
+        files: canonical.files,
+        symbols: canonical.symbols,
+        steps
+    };
+}
+function unavailableEvidence(strategyId, reason, steps) {
+    return buildEvidence(strategyId, "unavailable", reason, { files: [], symbols: [] }, steps);
+}
+async function readDataModelGraph(indexDir) {
+    let text;
+    try {
+        text = await readFile(path.join(indexDir, "data-model-graph.json"), "utf8");
+    }
+    catch {
+        return { ok: false, reason: "data-model-graph-artifact-missing" };
+    }
+    const payload = parseJsonIfPossible(text);
+    if (!isRecord(payload))
+        return { ok: false, reason: "data-model-graph-artifact-malformed" };
+    if (payload.artifactKind !== "my-dev-kit-v1-data-model-graph" ||
+        typeof payload.schemaVersion !== "string" ||
+        !payload.schemaVersion.startsWith("1.")) {
+        return { ok: false, reason: "data-model-graph-schema-unsupported" };
+    }
+    if (!Array.isArray(payload.nodes) || !Array.isArray(payload.edges)) {
+        return { ok: false, reason: "data-model-graph-artifact-malformed" };
+    }
+    const nodes = [];
+    for (const node of payload.nodes) {
+        if (isRecord(node) &&
+            typeof node.id === "string" &&
+            typeof node.label === "string" &&
+            (node.kind === "entity" || node.kind === "field")) {
+            nodes.push({ id: node.id, kind: node.kind, label: node.label });
+        }
+    }
+    return { ok: true, nodes };
+}
+/** Highest score wins; ties break on node id in code-unit order. No candidate means no match. */
+function selectSemanticCandidate(queryTokens, nodes) {
+    let best;
+    for (const node of nodes) {
+        const score = scoreSemanticLabel(queryTokens, node.label);
+        if (score <= 0)
+            continue;
+        if (!best || score > best.score || (score === best.score && compareCodeUnits(node.id, best.node.id) < 0)) {
+            best = { node, score };
+        }
+    }
+    return best?.node;
+}
+/** Normalizes `filePath` of every ref; unsafe values are counted, never kept. */
+function collectSafeFiles(refs) {
+    const files = [];
+    let rejected = 0;
+    for (const ref of refs) {
+        const normalized = isRecord(ref) ? normalizeRetrievedRepositoryPath(ref.filePath) : null;
+        if (normalized === null) {
+            rejected += 1;
+        }
+        else {
+            files.push(normalized);
+        }
+    }
+    return { files, rejected };
+}
+function uniqueFileOrNull(files) {
+    const unique = [...new Set(files)];
+    return unique.length === 1 ? unique[0] : null;
+}
+function readEntityOutput(payload, expectedMode) {
+    if (!isRecord(payload) || payload.status !== "ok" || typeof payload.mode !== "string")
+        return "malformed";
+    if (payload.mode !== expectedMode)
+        return "mismatched";
+    return payload;
+}
+function readIdentity(value) {
+    return isRecord(value) && typeof value.id === "string" && typeof value.name === "string"
+        ? { id: value.id, name: value.name }
+        : undefined;
+}
+function buildDataModelGraphStrategyEvidence(selectedNode, lookupPayload, generationCommand, lookupCommand) {
+    const strategyId = "data-model-graph";
+    const generationStep = step("data-model", generationCommand.ok, generationCommand.ok, null);
+    const failed = (reason) => unavailableEvidence(strategyId, reason, [generationStep, step("data-model", lookupCommand.ok, false, reason)]);
+    if (!lookupCommand.ok)
+        return failed("data-model-lookup-command-failed");
+    const expectedMode = selectedNode.kind === "entity" ? "entity" : "field";
+    const output = readEntityOutput(lookupPayload, expectedMode);
+    if (output === "malformed")
+        return failed("data-model-lookup-output-malformed");
+    if (output === "mismatched")
+        return failed("data-model-lookup-output-mismatched");
+    const selected = readIdentity(expectedMode === "entity" ? output.entity : output.field);
+    if (!selected || !Array.isArray(output.sourceRefs))
+        return failed("data-model-lookup-output-malformed");
+    const { files, rejected } = collectSafeFiles(output.sourceRefs);
+    const symbol = {
+        name: selected.name,
+        nodeId: selected.id,
+        file: uniqueFileOrNull(files)
+    };
+    const partial = rejected > 0;
+    return buildEvidence(strategyId, partial ? "partial" : "available", partial ? REJECTED_REASON : null, { files, symbols: [symbol] }, [generationStep, step("data-model", true, !partial, partial ? REJECTED_REASON : null)]);
+}
+function buildModelViewLineageStrategyEvidence(tracePayload, generationCommand, traceCommand, expectedMode) {
+    const strategyId = "model-view-lineage";
+    const generationStep = step("data-model", generationCommand.ok, generationCommand.ok, null);
+    const failed = (reason) => unavailableEvidence(strategyId, reason, [generationStep, step("model-view-lineage", traceCommand.ok, false, reason)]);
+    if (!traceCommand.ok)
+        return failed("model-view-lineage-command-failed");
+    const output = readEntityOutput(tracePayload, expectedMode);
+    if (output === "malformed")
+        return failed("model-view-lineage-output-malformed");
+    if (output === "mismatched")
+        return failed("model-view-lineage-output-mismatched");
+    const lineage = output.lineage;
+    if (!readIdentity(output.entity) || !isRecord(lineage) || !Array.isArray(lineage.nodes) || !Array.isArray(lineage.edges)) {
+        return failed("model-view-lineage-output-malformed");
+    }
+    const files = [];
+    const symbols = [];
+    let rejected = 0;
+    for (const node of lineage.nodes) {
+        if (!isRecord(node) || typeof node.id !== "string" || typeof node.label !== "string") {
+            rejected += 1;
+            continue;
+        }
+        let nodeFiles = [];
+        if (Array.isArray(node.evidenceRefs)) {
+            const collected = collectSafeFiles(node.evidenceRefs);
+            nodeFiles = collected.files;
+            rejected += collected.rejected;
+        }
+        else if (node.evidenceRefs !== undefined) {
+            rejected += 1;
+        }
+        files.push(...nodeFiles);
+        symbols.push({ name: node.label, nodeId: node.id, file: uniqueFileOrNull(nodeFiles) });
+    }
+    const partial = rejected > 0;
+    return buildEvidence(strategyId, partial ? "partial" : "available", partial ? REJECTED_REASON : null, { files, symbols }, [generationStep, step("model-view-lineage", true, !partial, partial ? REJECTED_REASON : null)]);
+}
+async function runDataModelCommand(options, commandId, args, commands) {
+    const command = await runMeasuredCommand({
+        commandId,
+        commandString: options.kitCommand,
+        cwd: process.cwd(),
+        outDir: options.commandsDir,
+        extraArgs: ["data-model", "--index", options.indexDir, ...args, "--json"]
+    });
+    commands.push(command);
+    if (!command.ok && options.requireKit) {
+        throw new Error(command.error || `my-dev-kit data-model failed with exit code ${command.exitCode}`);
+    }
+    return command;
+}
+function buildResult(options, started, commands, evidence, contextText) {
+    const warnings = [];
+    if (evidence.availability === "unavailable") {
+        warnings.push(`my-dev-kit semantic retrieval was unavailable: ${evidence.availabilityReason}.`);
+    }
+    else if (evidence.availability === "partial") {
+        warnings.push(`my-dev-kit semantic retrieval was partially interpreted: ${evidence.availabilityReason}.`);
+    }
+    const text = evidence.availability === "unavailable" ? "" : contextText;
+    return {
+        caseId: options.evaluationCase.id,
+        strategyId: options.strategyId,
+        skipped: text.length === 0,
+        warnings,
+        totalChars: countTextChars(text),
+        totalEstimatedTokens: countEstimatedTokens(text),
+        tokenCountMethod,
+        contextText: text,
+        commands,
+        queryStrategyEvidence: evidence,
+        durationMs: Date.now() - started
+    };
+}
+/** Shared by both strategies: generate artifacts, validate the graph, select one semantic node. */
+async function resolveSemanticTarget(options, commands) {
+    const generation = await runDataModelCommand(options, "data-model", [], commands);
+    if (!generation.ok) {
+        return {
+            kind: "done",
+            evidence: unavailableEvidence(options.strategyId, "data-model-command-failed", [
+                step("data-model", false, false, "data-model-command-failed")
+            ])
+        };
+    }
+    const generationStep = step("data-model", true, true, null);
+    const graph = await readDataModelGraph(options.indexDir);
+    if (!graph.ok) {
+        return {
+            kind: "done",
+            evidence: unavailableEvidence(options.strategyId, graph.reason, [step("data-model", true, false, graph.reason)])
+        };
+    }
+    const node = selectSemanticCandidate(tokenizeSemanticRetrievalQuery(options.evaluationCase.query), graph.nodes);
+    if (!node) {
+        return {
+            kind: "done",
+            evidence: buildEvidence(options.strategyId, "available", null, { files: [], symbols: [] }, [generationStep])
+        };
+    }
+    return { kind: "node", node, generation };
+}
+async function executeDataModelGraphStrategy(options, started) {
+    const commands = [];
+    const resolved = await resolveSemanticTarget(options, commands);
+    if (resolved.kind === "done")
+        return buildResult(options, started, commands, resolved.evidence, "");
+    const selector = resolved.node.kind === "entity" ? ["--entity", resolved.node.label] : ["--field", resolved.node.label];
+    const lookup = await runDataModelCommand(options, "data-model-selected", selector, commands);
+    const evidence = buildDataModelGraphStrategyEvidence(resolved.node, lookup.ok ? parseJsonIfPossible(lookup.stdout) : undefined, resolved.generation, lookup);
+    return buildResult(options, started, commands, evidence, lookup.stdout);
+}
+async function executeModelViewLineageStrategy(options, started) {
+    const commands = [];
+    const resolved = await resolveSemanticTarget(options, commands);
+    if (resolved.kind === "done")
+        return buildResult(options, started, commands, resolved.evidence, "");
+    const isEntity = resolved.node.kind === "entity";
+    const selector = isEntity ? ["--trace-view", resolved.node.label] : ["--field", resolved.node.label, "--trace-view"];
+    const trace = await runDataModelCommand(options, "model-view-lineage", selector, commands);
+    const evidence = buildModelViewLineageStrategyEvidence(trace.ok ? parseJsonIfPossible(trace.stdout) : undefined, resolved.generation, trace, isEntity ? "trace-entity" : "trace-field");
+    return buildResult(options, started, commands, evidence, trace.stdout);
+}
+/**
+ * Executes one semantic retrieval strategy against an already prepared index. Never invokes `index`,
+ * never reads the answer key, never runs agents.
+ */
+export async function runSemanticRetrievalStrategyFromIndex(options) {
+    const started = Date.now();
+    switch (options.strategyId) {
+        case "data-model-graph":
+            return executeDataModelGraphStrategy(options, started);
+        case "model-view-lineage":
+            return executeModelViewLineageStrategy(options, started);
+        default:
+            throw new Error(`Unsupported semantic retrieval strategy: ${String(options.strategyId)}`);
+    }
+}

package/dist/src/experiments/defaultRegistry.js

@@ -2,6 +2,8 @@ import { ExperimentPluginRegistry } from "./registry.js";
 import { contextStrategyComparisonPlugin } from "./plugins/contextStrategyComparison/index.js";
 import { contextWindowScalingPlugin } from "./plugins/contextWindowScaling/index.js";
 import { incrementalChangeStalenessPlugin } from "./plugins/incrementalChangeStaleness/index.js";
+import { retrievalPrecisionRecallPlugin } from "./plugins/retrievalPrecisionRecall/index.js";
+import { retrievalQueryStrategyComparisonPlugin } from "./plugins/retrievalQueryStrategyComparison/index.js";
 import { warmIndexReusePlugin } from "./plugins/warmIndexReuse/index.js";
 export function createDefaultExperimentPluginRegistry() {
     const registry = new ExperimentPluginRegistry();
@@ -9,5 +11,7 @@ export function createDefaultExperimentPluginRegistry() {
     registry.register(warmIndexReusePlugin);
     registry.register(incrementalChangeStalenessPlugin);
     registry.register(contextWindowScalingPlugin);
+    registry.register(retrievalPrecisionRecallPlugin);
+    registry.register(retrievalQueryStrategyComparisonPlugin);
     return registry;
 }

package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js

@@ -83,6 +83,8 @@ export function projectRunForExternalLocalPersistence(run, target) {
         metadata.outputRoot = EXTERNAL_LOCAL_REDACTED_VALUE;
     if (typeof metadata.executionArtifactPath === "string")
         metadata.executionArtifactPath = path.basename(metadata.executionArtifactPath);
+    if (typeof metadata.analysisArtifactPath === "string")
+        metadata.analysisArtifactPath = path.basename(metadata.analysisArtifactPath);
     return { ...run, target, metadata: metadata };
 }
 const GENERIC_ISSUE_TEXT = {

package/dist/src/experiments/plugins/index.js

@@ -2,3 +2,5 @@ export * from "./contextStrategyComparison/index.js";
 export * from "./contextWindowScaling/index.js";
 export * from "./incrementalChangeStaleness/index.js";
 export * from "./warmIndexReuse/index.js";
+export * from "./retrievalPrecisionRecall/index.js";
+export * from "./retrievalQueryStrategyComparison/index.js";

package/dist/src/experiments/plugins/retrievalPrecisionRecall/config.js

@@ -0,0 +1,39 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultRetrievalPrecisionRecallConfig = {
+    outDir: "lab-output/retrieval-precision-recall",
+    kitCommand: "npx @dailephd/my-dev-kit@latest"
+};
+const SUPPORTED_FIELDS = ["outDir", "kitCommand", "caseIds", "benchmarkProjects"];
+export const retrievalPrecisionRecallConfigDefinition = {
+    fields: [
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
+        { name: "caseIds", type: "array", description: "Bundled evaluation case IDs to include (corpus order is preserved)." },
+        { name: "benchmarkProjects", type: "array", description: "Bundled benchmark project IDs to include (corpus order is preserved)." }
+    ]
+};
+export function validateRetrievalPrecisionRecallConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["retrieval precision/recall config must be an object."]);
+    }
+    const errors = [];
+    const unsupported = Object.keys(config ?? {}).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    if (unsupported.length > 0) {
+        errors.push(`Unsupported retrieval-precision-recall config field(s): ${unsupported.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultRetrievalPrecisionRecallConfig, config);
+    for (const field of ["outDir", "kitCommand"]) {
+        const value = normalized[field];
+        if (typeof value !== "string" || !value.trim())
+            errors.push(`${field} must be a non-empty string.`);
+    }
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+    }
+    return errors.length > 0 ? invalidExperimentConfig(errors) : validExperimentConfig(normalized);
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/execution.js

@@ -0,0 +1,152 @@
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { buildMyDevKitIndex, runMyDevKitRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { calculateRetrievalQualityMetrics, validateRetrievalPrecisionRecallCase } from "../../../evaluation/retrievalQuality/index.js";
+import { groupWarmIndexCases, taskOutputSegment } from "../warmIndexReuse/selection.js";
+const MAX_MESSAGE_LENGTH = 300;
+const MAX_WARNINGS = 20;
+/** Bounds a message and replaces machine-local paths so persisted errors never carry them. */
+export function boundedRetrievalSafeMessage(message, knownRoots = []) {
+    let text = message;
+    for (const root of knownRoots) {
+        if (!root)
+            continue;
+        text = text.split(root).join("<path>").split(root.replace(/\\/g, "/")).join("<path>");
+    }
+    text = text
+        .replace(/[A-Za-z]:[\\/][^\s"'`]*/g, "<path>")
+        .replace(/(?:^|(?<=[\s"'`(]))\/(?:Users|home|tmp|var|private|opt|mnt)\/[^\s"'`]*/g, "<path>")
+        .replace(/[\u0000-\u001f\u007f]/g, " ");
+    return text.length > MAX_MESSAGE_LENGTH ? `${text.slice(0, MAX_MESSAGE_LENGTH)}...` : text;
+}
+const errorText = (error) => (error instanceof Error ? error.message : String(error));
+function failedCase(evaluationCase, code, message) {
+    return {
+        caseId: evaluationCase.id,
+        caseName: evaluationCase.title,
+        benchmarkProject: evaluationCase.benchmarkProject,
+        taskLocality: evaluationCase.taskLocality ?? null,
+        status: "failed",
+        retrieval: null,
+        quality: null,
+        errors: [{ code, message }]
+    };
+}
+export function summarizeRetrieval(result) {
+    const evidence = result.retrievalEvidence;
+    return {
+        skipped: result.skipped,
+        durationMs: result.durationMs,
+        totalEstimatedTokens: result.totalEstimatedTokens,
+        tokenCountMethod: result.tokenCountMethod,
+        warnings: result.warnings.slice(0, MAX_WARNINGS).map((warning) => boundedRetrievalSafeMessage(warning)),
+        evidenceAvailability: evidence?.availability ?? "missing",
+        evidenceAvailabilityReason: evidence?.availabilityReason ?? (evidence === undefined ? "retrieval-evidence-missing" : null),
+        retrievedFileCount: evidence?.files.length ?? null,
+        retrievedSymbolCount: evidence?.symbols.length ?? null,
+        commands: evidence ? evidence.commands.map((command) => ({ ...command })) : []
+    };
+}
+/**
+ * One full index per benchmark project, then one retrieval per case from that project's prepared index, using the
+ * existing lifecycle owners. A failed project fails only its own cases. No agent runs. Only bounded, context-free
+ * evidence is returned: retrieval results (and their contextText) never leave this function.
+ */
+export async function executeRetrievalPrecisionRecall(options) {
+    const buildIndex = options.dependencies?.buildIndex ?? buildMyDevKitIndex;
+    const retrieveFromIndex = options.dependencies?.retrieveFromIndex ?? runMyDevKitRetrievalFromIndex;
+    const results = new Map();
+    const knownRoots = [options.outputRoot, ...options.cases.map((evaluationCase) => evaluationCase.absoluteTargetRoot)];
+    const validCases = [];
+    for (const evaluationCase of options.cases) {
+        const groundTruthErrors = validateRetrievalPrecisionRecallCase(evaluationCase);
+        if (groundTruthErrors.length > 0) {
+            results.set(evaluationCase.id, failedCase(evaluationCase, "ground-truth-invalid", boundedRetrievalSafeMessage(groundTruthErrors.join(" "), knownRoots)));
+        }
+        else {
+            validCases.push(evaluationCase);
+        }
+    }
+    for (const group of groupWarmIndexCases(validCases)) {
+        if (group.structuralErrors.length > 0) {
+            for (const evaluationCase of group.cases) {
+                results.set(evaluationCase.id, failedCase(evaluationCase, "project-group-inconsistent", "Cases of one benchmark project disagree on target root or source roots."));
+            }
+            continue;
+        }
+        const indexDir = resolveWithinRoot(options.outputRoot, path.join("indexes", group.projectSegment));
+        const commandsDir = resolveWithinRoot(options.outputRoot, path.join("commands", group.projectSegment));
+        let indexFailure;
+        try {
+            // requireKit: false keeps a failed index as measurable evidence instead of throwing.
+            const build = await buildIndex({
+                target: { absoluteTargetRoot: group.targetRoot, sourceRoots: [...group.sourceRoots] },
+                kitCommand: options.kitCommand,
+                indexDir,
+                commandsDir: path.join(commandsDir, "index"),
+                requireKit: false
+            });
+            if (!build.ok)
+                indexFailure = build.warnings.join(" ") || "my-dev-kit index setup failed.";
+        }
+        catch (error) {
+            indexFailure = errorText(error);
+        }
+        if (indexFailure !== undefined) {
+            const message = boundedRetrievalSafeMessage(indexFailure, knownRoots);
+            for (const evaluationCase of group.cases) {
+                results.set(evaluationCase.id, failedCase(evaluationCase, "project-index-failed", message));
+            }
+            continue;
+        }
+        for (const evaluationCase of group.cases) {
+            results.set(evaluationCase.id, await executeCase({
+                evaluationCase,
+                kitCommand: options.kitCommand,
+                indexDir,
+                commandsDir: path.join(commandsDir, taskOutputSegment(evaluationCase.id)),
+                retrieveFromIndex,
+                knownRoots
+            }));
+        }
+    }
+    // Selection order is the corpus order after filtering, regardless of how projects were grouped.
+    return options.cases.map((evaluationCase) => results.get(evaluationCase.id));
+}
+async function executeCase(args) {
+    const { evaluationCase } = args;
+    let retrieval;
+    try {
+        retrieval = await args.retrieveFromIndex({
+            evaluationCase,
+            kitCommand: args.kitCommand,
+            indexDir: args.indexDir,
+            commandsDir: args.commandsDir,
+            requireKit: false
+        });
+    }
+    catch (error) {
+        return failedCase(evaluationCase, "retrieval-failed", boundedRetrievalSafeMessage(errorText(error), args.knownRoots));
+    }
+    return buildCaseEvidenceFromRetrieval(evaluationCase, retrieval);
+}
+/**
+ * The single place a returned retrieval becomes bounded case evidence: Batch 2 metrics are calculated here and the
+ * case is COMPLETED only when the retrieval evidence is available, otherwise PARTIAL. Shared by the bundled and the
+ * external-local execution paths so neither duplicates the status rule or the metric call.
+ */
+export function buildCaseEvidenceFromRetrieval(evaluationCase, retrieval) {
+    const quality = calculateRetrievalQualityMetrics({ evaluationCase, retrieval });
+    const status = retrieval.retrievalEvidence?.availability === "available" ? "completed" : "partial";
+    const errors = [];
+    return {
+        caseId: evaluationCase.id,
+        caseName: evaluationCase.title,
+        benchmarkProject: evaluationCase.benchmarkProject,
+        taskLocality: evaluationCase.taskLocality ?? null,
+        status,
+        retrieval: summarizeRetrieval(retrieval),
+        quality,
+        errors
+    };
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/executionArtifact.js

@@ -0,0 +1,15 @@
+export const RETRIEVAL_PRECISION_RECALL_EXECUTION_ARTIFACT_FILE = "retrieval-precision-recall-execution.json";
+export const RETRIEVAL_PRECISION_RECALL_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-retrieval-precision-recall-execution-v1";
+export function buildRetrievalPrecisionRecallExecutionArtifact(args) {
+    return {
+        schemaVersion: RETRIEVAL_PRECISION_RECALL_EXECUTION_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        tokenCountMethod: args.cases.find((entry) => entry.retrieval !== null)?.retrieval?.tokenCountMethod ?? null,
+        cases: args.cases.map((entry) => structuredClone(entry)),
+        aggregate: structuredClone(args.aggregate)
+    };
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/index.js

@@ -0,0 +1,9 @@
+export * from "./config.js";
+export * from "./execution.js";
+export * from "./localSubjectExecution.js";
+export * from "./localSubjectPrivacy.js";
+export * from "./executionArtifact.js";
+export * from "./metadata.js";
+export * from "./metrics.js";
+export * from "./plugin.js";
+export * from "./types.js";