@dailephd/my-dev-kit-lab 0.7.2 → 0.8.1

package/dist/src/experiments/plugins/retrievalPrecisionRecall/localSubjectExecution.js

@@ -0,0 +1,139 @@
+import path from "node:path";
+import { relativeWithinRoot } from "../../../core/pathSafety.js";
+import { normalizeRetrievedRepositoryPath, validateRetrievalPrecisionRecallCase } from "../../../evaluation/retrievalQuality/index.js";
+import { buildMyDevKitIndex, runMyDevKitRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { captureTargetSnapshot, compareTargetSnapshots } from "../../../evaluation/targetImmutability/index.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { deriveGuidedIndexExclusions } from "../contextWindowScaling/localSubjectExclusions.js";
+import { assertWorkRootOutsideTarget, createPrivateScratch, defaultPrivateScratchIo, removePrivateScratch } from "../contextWindowScaling/localSubjectScratch.js";
+import { buildCaseEvidenceFromRetrieval } from "./execution.js";
+function toRepositoryRelative(repositoryRoot, file) {
+    if (path.isAbsolute(file)) {
+        try {
+            return relativeWithinRoot(repositoryRoot, file);
+        }
+        catch {
+            return file.replace(/\\/g, "/");
+        }
+    }
+    return normalizeRetrievedRepositoryPath(file) ?? file.replace(/\\/g, "/");
+}
+/**
+ * Counts distinct retrieved identities outside the eligible subject universe. It inspects every file identity the
+ * retrieval exposes: evidence files, evidence symbol files, the selected file, and the files read. Only the count
+ * leaves this function; an offending identity is never named.
+ */
+export function countRetrievedIdentitiesOutsideEligibleUniverse(retrieval, eligibleFiles, repositoryRoot) {
+    const identities = new Set();
+    for (const file of retrieval.retrievalEvidence?.files ?? [])
+        identities.add(toRepositoryRelative(repositoryRoot, file.path));
+    for (const symbol of retrieval.retrievalEvidence?.symbols ?? []) {
+        if (symbol.file !== undefined)
+            identities.add(toRepositoryRelative(repositoryRoot, symbol.file));
+    }
+    if (retrieval.selectedFile)
+        identities.add(toRepositoryRelative(repositoryRoot, retrieval.selectedFile));
+    for (const file of retrieval.filesRead)
+        identities.add(toRepositoryRelative(repositoryRoot, file));
+    return [...identities].filter((identity) => !eligibleFiles.has(identity)).length;
+}
+/**
+ * Internal programmatic seam: runs a loaded local subject through the existing index and retrieval owners, one
+ * private index per configured case (each case keeps its own exact source roots), with a private scratch outside the
+ * target and pre/post target-immutability enforcement. Order: validate work root, derive exclusions, validate
+ * ground truth, capture before, create scratch, execute cases, capture after, compare, remove scratch, then decide.
+ * Success is returned only when every safety gate passed. A mutated target is never reverted.
+ *
+ * Every issue message is safe by construction: fixed text plus logical case ids and counts, never a file name,
+ * symbol, fact id, path, or caught error text.
+ */
+export async function executeLocalRepositorySubjectRetrievalPrecisionRecall(args) {
+    const { subject } = args;
+    const buildIndex = args.dependencies?.buildIndex ?? buildMyDevKitIndex;
+    const retrieveFromIndex = args.dependencies?.retrieveFromIndex ?? runMyDevKitRetrievalFromIndex;
+    await assertWorkRootOutsideTarget(args.workRoot, subject.repositoryRoot);
+    const guidedExclusions = deriveGuidedIndexExclusions({
+        eligibleFiles: subject.eligibleFiles,
+        excludedFiles: [...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles]
+    });
+    const invalidCaseIds = subject.evaluationCases.filter((evaluationCase) => validateRetrievalPrecisionRecallCase(evaluationCase).length > 0).map((evaluationCase) => evaluationCase.id);
+    if (invalidCaseIds.length > 0) {
+        throw new LocalSubjectExecutionError([
+            { code: "GROUND_TRUTH_INVALID", message: `retrieval ground truth is incomplete for case(s): ${invalidCaseIds.join(", ")}.` }
+        ]);
+    }
+    const snapshotConfig = { targetRootPath: subject.repositoryRoot, relativeFilePaths: subject.eligibleFiles };
+    const snapshotOptions = { externalLocalSafe: { maxHashedFileBytes: subject.manifest.safetyPolicy.maxFileBytes } };
+    const before = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!before.ok) {
+        throw new LocalSubjectExecutionError([{ code: "BEFORE_SNAPSHOT_FAILED", message: `target snapshot before execution failed (${before.code}).` }]);
+    }
+    const scratch = await createPrivateScratch(args.workRoot, subject.repositoryRoot);
+    const issues = [];
+    const caseEvidence = [];
+    const eligible = new Set(subject.eligibleFiles);
+    for (const [position, evaluationCase] of subject.evaluationCases.entries()) {
+        // Short, fixed segments: private case or repository names never become directory names.
+        const indexDir = path.join(scratch.path, `i${position + 1}`);
+        const commandsDir = path.join(scratch.path, `c${position + 1}`);
+        try {
+            // requireKit: false keeps a failed index as a result we can classify instead of an arbitrary thrown message.
+            const build = await buildIndex({
+                target: { absoluteTargetRoot: evaluationCase.absoluteTargetRoot, sourceRoots: [...evaluationCase.sourceRoots] },
+                kitCommand: args.kitCommand,
+                indexDir,
+                commandsDir: path.join(commandsDir, "index"),
+                requireKit: false,
+                excludePaths: guidedExclusions
+            });
+            if (!build.ok) {
+                issues.push({ code: "EXECUTION_FAILED", message: `case ${evaluationCase.id}: the my-dev-kit index could not be prepared.` });
+                break;
+            }
+            const retrieval = await retrieveFromIndex({
+                evaluationCase,
+                kitCommand: args.kitCommand,
+                indexDir,
+                commandsDir: path.join(commandsDir, "retrieval"),
+                requireKit: false
+            });
+            const outside = countRetrievedIdentitiesOutsideEligibleUniverse(retrieval, eligible, subject.repositoryRoot);
+            if (outside > 0) {
+                issues.push({
+                    code: "RETRIEVAL_OUTSIDE_ELIGIBLE_UNIVERSE",
+                    message: `case ${evaluationCase.id}: retrieval exposed ${outside} file identit${outside === 1 ? "y" : "ies"} outside the eligible subject universe.`
+                });
+                break;
+            }
+            caseEvidence.push(buildCaseEvidenceFromRetrieval(evaluationCase, retrieval));
+        }
+        catch {
+            issues.push({ code: "EXECUTION_FAILED", message: `case ${evaluationCase.id}: execution threw before completing (details withheld).` });
+            break;
+        }
+    }
+    let comparison = null;
+    const after = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!after.ok) {
+        issues.push({ code: "AFTER_SNAPSHOT_FAILED", message: `target snapshot after execution failed (${after.code}).` });
+    }
+    else {
+        comparison = compareTargetSnapshots(before.snapshot, after.snapshot);
+        if (comparison.status === "mutated") {
+            issues.push({ code: "TARGET_MUTATED", message: "the target changed during execution; it was not restored." });
+        }
+    }
+    const cleanupFailure = await removePrivateScratch(scratch, args.scratchIo ?? defaultPrivateScratchIo);
+    if (cleanupFailure) {
+        issues.push({ code: "SCRATCH_CLEANUP_FAILED", message: "the private scratch could not be removed." });
+    }
+    if (issues.length > 0 || !comparison) {
+        throw new LocalSubjectExecutionError(issues.length > 0 ? issues : [{ code: "EXECUTION_FAILED", message: "execution did not complete." }], {
+            immutability: comparison
+        });
+    }
+    return {
+        caseEvidence,
+        immutability: { status: "unchanged", preExistingGitStatusEntryCount: comparison.preExistingGitStatusEntryCount, newMutationCount: 0 }
+    };
+}

package/dist/src/experiments/plugins/retrievalPrecisionRecall/localSubjectPrivacy.js

@@ -0,0 +1,119 @@
+import { redactedFileList } from "../contextWindowScaling/localSubjectPrivacy.js";
+/** Single fixed placeholder for a withheld case title. */
+export const REDACTED_CASE_TITLE = "<redacted case title>";
+/** The one V1 marker stamped on every externally projected case, so placeholders are never mistaken for real values. */
+export const RETRIEVAL_IDENTITY_REDACTION = {
+    fileIdentities: "redacted",
+    symbolIdentities: "redacted",
+    factIdentities: "redacted",
+    warningText: "redacted",
+    caseTitle: "redacted"
+};
+/** One opaque placeholder per identity: counts stay exact; null stays null; an empty list stays empty. */
+function placeholders(kind, count) {
+    return Array.from({ length: count }, (_, index) => `<redacted ${kind} ${index + 1}>`);
+}
+const filePlaceholders = (list) => (list === null ? null : redactedFileList(list.length));
+const symbolPlaceholders = (list) => (list === null ? null : placeholders("symbol", list.length));
+const factPlaceholders = (list) => (list === null ? null : placeholders("fact", list.length));
+/** Plugin-owned text per error code; a caught Error.message is never copied into durable external evidence. */
+const FIXED_ERROR_TEXT = {
+    "ground-truth-invalid": "The case ground truth was invalid.",
+    "project-group-inconsistent": "The case source configuration was inconsistent.",
+    "project-index-failed": "The my-dev-kit index could not be prepared.",
+    "retrieval-failed": "The retrieval did not complete."
+};
+/** Only identity lists change; every number, availability and fixed-vocabulary reason is carried through untouched. */
+function projectQuality(quality) {
+    return {
+        ...quality,
+        file: {
+            ...quality.file,
+            relevantRetrievedFiles: filePlaceholders(quality.file.relevantRetrievedFiles),
+            irrelevantRetrievedFiles: filePlaceholders(quality.file.irrelevantRetrievedFiles),
+            missedFiles: filePlaceholders(quality.file.missedFiles)
+        },
+        symbol: {
+            ...quality.symbol,
+            relevantRetrievedSymbols: symbolPlaceholders(quality.symbol.relevantRetrievedSymbols),
+            irrelevantRetrievedSymbols: symbolPlaceholders(quality.symbol.irrelevantRetrievedSymbols),
+            missedSymbols: symbolPlaceholders(quality.symbol.missedSymbols)
+        },
+        fact: {
+            ...quality.fact,
+            coveredFactIds: factPlaceholders(quality.fact.coveredFactIds),
+            uncoveredFactIds: factPlaceholders(quality.fact.uncoveredFactIds)
+        }
+    };
+}
+/**
+ * Projects case evidence for durable persistence of an external-local run. Redacts file, symbol and fact identity
+ * lists, the case title, and all retrieval warning prose; keeps logical ids, locality, statuses, numbers and
+ * fixed-vocabulary evidence. It performs no metric calculation.
+ */
+export function projectRetrievalEvidenceForExternalLocalPersistence(evidence) {
+    return evidence.map((entry) => {
+        const cloned = structuredClone(entry);
+        return {
+            ...cloned,
+            caseName: REDACTED_CASE_TITLE,
+            retrieval: cloned.retrieval === null
+                ? null
+                : { ...cloned.retrieval, warnings: placeholders("warning", cloned.retrieval.warnings.length) },
+            quality: cloned.quality === null ? null : projectQuality(cloned.quality),
+            errors: cloned.errors.map((error) => ({ code: error.code, message: FIXED_ERROR_TEXT[error.code] })),
+            identityRedaction: { ...RETRIEVAL_IDENTITY_REDACTION }
+        };
+    });
+}
+/**
+ * Defense in depth: fails closed if a known private file path, case title, or warning text survived projection.
+ * It deliberately does not substring-search short symbol or fact identifiers, which would collide with ordinary words.
+ */
+export function assertExternalLocalProjectionIsPrivate(projected, privateValues) {
+    const serialized = JSON.stringify(projected);
+    const candidates = [...privateValues.filePaths, ...privateValues.caseTitles, ...privateValues.warnings].filter((value) => value.length > 0);
+    const leaked = candidates.some((value) => serialized.includes(value) || serialized.includes(JSON.stringify(value).slice(1, -1)));
+    if (leaked) {
+        throw new Error("External-local privacy projection check failed; no durable output was written.");
+    }
+}
+const GENERIC_ISSUE_TEXT = {
+    WORK_ROOT_INSIDE_TARGET: "the output directory must be outside the local subject repository",
+    BEFORE_SNAPSHOT_FAILED: "the target snapshot before execution could not be captured",
+    AFTER_SNAPSHOT_FAILED: "the target snapshot after execution could not be captured",
+    SCRATCH_CLEANUP_FAILED: "the private scratch could not be removed",
+    GUIDED_EXCLUSION_LIMIT: "the guided-index exclusions exceeded the safe limit",
+    GUIDED_EXCLUSION_UNREPRESENTABLE: "the guided-index exclusions could not be expressed exactly"
+};
+/**
+ * Safe, persistable description of an external-local failure: issue codes, logical case ids, counts and mutation
+ * kinds only. The issue messages built by the execution seam are safe by construction and are included for the
+ * three case-scoped codes; nothing else (no file name, symbol, fact id, path, or caught error text) can appear.
+ */
+export function describeRetrievalLocalSubjectFailureForPersistence(error) {
+    const parts = [];
+    const seen = new Set();
+    for (const issue of error.issues) {
+        if (issue.code === "EXECUTION_FAILED" || issue.code === "GROUND_TRUTH_INVALID" || issue.code === "RETRIEVAL_OUTSIDE_ELIGIBLE_UNIVERSE") {
+            if (!seen.has(issue.message)) {
+                seen.add(issue.message);
+                parts.push(issue.message.replace(/\.$/, ""));
+            }
+            continue;
+        }
+        if (seen.has(issue.code))
+            continue;
+        seen.add(issue.code);
+        if (issue.code === "TARGET_MUTATED") {
+            const kinds = [...new Set((error.immutability?.mutations ?? []).map((mutation) => mutation.kind))].sort();
+            parts.push(`the target changed during execution${kinds.length > 0 ? ` (${kinds.join(", ")})` : ""}; it was not restored`);
+        }
+        else {
+            parts.push(GENERIC_ISSUE_TEXT[issue.code] ?? `issue ${issue.code}`);
+        }
+    }
+    return `Local subject execution failed (${error.code}): ${parts.join("; ")}.`;
+}
+/** Fixed message for any non-LocalSubjectExecutionError thrown inside the external path: no caught text is exposed. */
+export const UNEXPECTED_EXTERNAL_FAILURE_MESSAGE = "Local subject execution failed (EXECUTION_FAILED): execution failed before completing (details withheld).";

package/dist/src/experiments/plugins/retrievalPrecisionRecall/metadata.js

@@ -0,0 +1,21 @@
+export const RETRIEVAL_PRECISION_RECALL_PLUGIN_ID = "retrieval-precision-recall";
+export const RETRIEVAL_PRECISION_RECALL_VARIANT_ID = "my-dev-kit-retrieval";
+/** The frozen bundled corpus; the public plugin offers no way to substitute another. */
+export const RETRIEVAL_PRECISION_RECALL_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
+export const RETRIEVAL_PRECISION_RECALL_PROJECT_PROFILES_RESOURCE = "benchmarks/contracts/benchmark-project-profiles.json";
+export const retrievalPrecisionRecallMetadata = {
+    id: RETRIEVAL_PRECISION_RECALL_PLUGIN_ID,
+    name: "Retrieval Precision/Recall",
+    description: "Measure deterministic file, symbol, fact and irrelevant-context retrieval quality for the existing my-dev-kit retrieval lifecycle without agents.",
+    schemaVersion: "1.0.0",
+    status: "experimental",
+    supportedTargets: ["self", "external-local"],
+    supportedOutputs: ["json", "html", "text", "artifact"]
+};
+export const RETRIEVAL_PRECISION_RECALL_VARIANTS = [
+    {
+        id: RETRIEVAL_PRECISION_RECALL_VARIANT_ID,
+        name: "my-dev-kit retrieval",
+        description: "search, then lookup, slice and source for the top candidate, from one index prepared per benchmark project."
+    }
+];

package/dist/src/experiments/plugins/retrievalPrecisionRecall/metrics.js

@@ -0,0 +1,158 @@
+import { RETRIEVAL_PRECISION_RECALL_RATIO_KEYS } from "./types.js";
+const UNAVAILABLE_RATIO = {
+    availability: "unavailable",
+    numerator: null,
+    denominator: null,
+    value: null,
+    reason: "no-retrieval-measurement"
+};
+/** Per-case ratio selector. A failed case has no quality result and is counted as unavailable, never as zero. */
+export function retrievalRatioOf(quality, key) {
+    if (quality === null)
+        return UNAVAILABLE_RATIO;
+    switch (key) {
+        case "filePrecision":
+            return quality.file.precision;
+        case "fileRecall":
+            return quality.file.recall;
+        case "symbolPrecision":
+            return quality.symbol.precision;
+        case "symbolRecall":
+            return quality.symbol.recall;
+        case "factCoverage":
+            return quality.fact.coverage;
+        case "irrelevantContextRatio":
+            return quality.irrelevantContextRatio;
+    }
+}
+function aggregateRatio(metrics) {
+    const available = metrics.filter((metric) => metric.availability === "available" && typeof metric.value === "number");
+    return {
+        availableCount: available.length,
+        unavailableCount: metrics.filter((metric) => metric.availability === "unavailable").length,
+        notApplicableCount: metrics.filter((metric) => metric.availability === "not-applicable").length,
+        meanValue: available.length === 0 ? null : available.reduce((sum, metric) => sum + metric.value, 0) / available.length
+    };
+}
+function aggregateTokens(cases) {
+    const counts = cases
+        .map((entry) => entry.quality?.retrievedTokenCount)
+        .filter((count) => typeof count === "number" && Number.isFinite(count) && count >= 0);
+    const total = counts.reduce((sum, count) => sum + count, 0);
+    return {
+        availableCount: counts.length,
+        unavailableCount: cases.length - counts.length,
+        totalTokens: counts.length === 0 ? null : total,
+        meanTokens: counts.length === 0 ? null : total / counts.length,
+        minTokens: counts.length === 0 ? null : Math.min(...counts),
+        maxTokens: counts.length === 0 ? null : Math.max(...counts)
+    };
+}
+const countStatus = (cases, status) => cases.filter((entry) => entry.status === status).length;
+/**
+ * Pure macro aggregation over per-case Batch 2 results. It never recalculates a per-case metric, never weights by
+ * denominators, and never produces a score, winner, ranking or threshold.
+ */
+export function aggregateRetrievalPrecisionRecall(cases) {
+    const ratios = Object.fromEntries(RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => [key, aggregateRatio(cases.map((entry) => retrievalRatioOf(entry.quality, key)))]));
+    const sumLists = (select) => {
+        let total = 0;
+        let evidenceCases = 0;
+        for (const entry of cases) {
+            const list = entry.quality === null ? null : select(entry.quality);
+            if (list === null)
+                continue;
+            total += list.length;
+            evidenceCases += 1;
+        }
+        return { total, evidenceCases };
+    };
+    const missedFiles = sumLists((quality) => quality.file.missedFiles);
+    const irrelevantFiles = sumLists((quality) => quality.file.irrelevantRetrievedFiles);
+    const missedSymbols = sumLists((quality) => quality.symbol.missedSymbols);
+    const irrelevantSymbols = sumLists((quality) => quality.symbol.irrelevantRetrievedSymbols);
+    const uncoveredFacts = sumLists((quality) => quality.fact.uncoveredFactIds);
+    const evidenceCount = (availability) => cases.filter((entry) => entry.retrieval?.evidenceAvailability === availability).length;
+    return {
+        runSummary: {
+            projectCount: new Set(cases.map((entry) => entry.benchmarkProject)).size,
+            caseCount: cases.length,
+            completedCaseCount: countStatus(cases, "completed"),
+            partialCaseCount: countStatus(cases, "partial"),
+            failedCaseCount: countStatus(cases, "failed"),
+            retrievalEvidenceAvailableCaseCount: evidenceCount("available"),
+            retrievalEvidencePartialCaseCount: evidenceCount("partial"),
+            retrievalEvidenceUnavailableCaseCount: cases.filter((entry) => entry.retrieval !== null && entry.retrieval.evidenceAvailability !== "available" && entry.retrieval.evidenceAvailability !== "partial").length
+        },
+        ratios,
+        tokens: aggregateTokens(cases),
+        occurrences: {
+            totalMissedFileOccurrences: missedFiles.total,
+            totalMissedSymbolOccurrences: missedSymbols.total,
+            totalUncoveredFactOccurrences: uncoveredFacts.total,
+            totalIrrelevantRetrievedFileOccurrences: irrelevantFiles.total,
+            totalIrrelevantRetrievedSymbolOccurrences: irrelevantSymbols.total,
+            fileOccurrenceEvidenceCaseCount: missedFiles.evidenceCases,
+            symbolOccurrenceEvidenceCaseCount: missedSymbols.evidenceCases,
+            factOccurrenceEvidenceCaseCount: uncoveredFacts.evidenceCases
+        }
+    };
+}
+const RATIO_METRIC_IDS = {
+    filePrecision: { id: "file-precision", name: "File precision" },
+    fileRecall: { id: "file-recall", name: "File recall" },
+    symbolPrecision: { id: "symbol-precision", name: "Symbol precision" },
+    symbolRecall: { id: "symbol-recall", name: "Symbol recall" },
+    factCoverage: { id: "fact-coverage", name: "Fact coverage" },
+    irrelevantContextRatio: { id: "irrelevant-context-ratio", name: "Irrelevant context ratio" }
+};
+/** Run-level generic metric projection. Ratios stay canonical (0..1, never percent); null means no available evidence. */
+export function toRetrievalPrecisionRecallRunMetrics(aggregate) {
+    const summary = aggregate.runSummary;
+    const occurrences = aggregate.occurrences;
+    return [
+        { id: "case-count", name: "Cases", value: summary.caseCount, unit: "count" },
+        { id: "completed-case-count", name: "Completed cases", value: summary.completedCaseCount, unit: "count" },
+        { id: "partial-case-count", name: "Partial cases", value: summary.partialCaseCount, unit: "count" },
+        { id: "failed-case-count", name: "Failed cases", value: summary.failedCaseCount, unit: "count" },
+        ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map((key) => ({
+            id: `mean-${RATIO_METRIC_IDS[key].id}`,
+            name: `Mean ${RATIO_METRIC_IDS[key].name.toLowerCase()}`,
+            value: aggregate.ratios[key].meanValue,
+            unit: "ratio"
+        })),
+        { id: "total-retrieved-tokens", name: "Total retrieved tokens", value: aggregate.tokens.totalTokens, unit: "tokens" },
+        { id: "mean-retrieved-tokens", name: "Mean retrieved tokens", value: aggregate.tokens.meanTokens, unit: "tokens" },
+        { id: "total-missed-file-occurrences", name: "Total missed file occurrences", value: occurrences.totalMissedFileOccurrences, unit: "count" },
+        { id: "total-missed-symbol-occurrences", name: "Total missed symbol occurrences", value: occurrences.totalMissedSymbolOccurrences, unit: "count" },
+        { id: "total-uncovered-fact-occurrences", name: "Total uncovered fact occurrences", value: occurrences.totalUncoveredFactOccurrences, unit: "count" },
+        { id: "total-irrelevant-file-occurrences", name: "Total irrelevant file occurrences", value: occurrences.totalIrrelevantRetrievedFileOccurrences, unit: "count" }
+    ];
+}
+/** Per-case metric projection for the single variant; unavailable and not-applicable ratios project as null. */
+export function toRetrievalPrecisionRecallOutcomeMetrics(quality, variantId, caseId) {
+    const base = { variantId, caseId };
+    const ratio = (key) => ({
+        ...base,
+        id: RATIO_METRIC_IDS[key].id,
+        name: RATIO_METRIC_IDS[key].name,
+        value: retrievalRatioOf(quality, key).value,
+        unit: "ratio"
+    });
+    const count = (id, name, value) => ({
+        ...base,
+        id,
+        name,
+        value: value ?? null,
+        unit: "count"
+    });
+    return [
+        ...RETRIEVAL_PRECISION_RECALL_RATIO_KEYS.map(ratio),
+        { ...base, id: "retrieved-token-count", name: "Retrieved token count", value: quality?.retrievedTokenCount ?? null, unit: "tokens" },
+        count("missed-file-count", "Missed file count", quality?.file.missedFileCount),
+        count("missed-symbol-count", "Missed symbol count", quality?.symbol.missedSymbolCount),
+        count("uncovered-fact-count", "Uncovered fact count", quality?.fact.uncoveredFactCount),
+        count("irrelevant-retrieved-file-count", "Irrelevant retrieved file count", quality?.file.irrelevantRetrievedFiles?.length),
+        count("irrelevant-retrieved-symbol-count", "Irrelevant retrieved symbol count", quality?.symbol.irrelevantRetrievedSymbols?.length)
+    ];
+}