@dailephd/my-dev-kit-lab 0.8.1 → 0.8.2

package/dist/src/report/experiments/renderPluginExperimentReportText.js

@@ -5,6 +5,7 @@ import { REINDEX_RECOMMENDATION_EXPLANATIONS } from "./buildWarmIndexReuseReport
 import { renderContextWindowScalingTextLines } from "./renderContextWindowScalingText.js";
 import { renderRetrievalPrecisionRecallTextLines } from "./renderRetrievalPrecisionRecallText.js";
 import { renderRetrievalQueryStrategyComparisonTextLines } from "./renderRetrievalQueryStrategyComparisonText.js";
+import { renderContextPackGenerationTextLines } from "./renderContextPackGenerationText.js";
 function sanitizeScalar(value) {
     const text = String(value);
     let result = "";
@@ -689,6 +690,12 @@ export function renderPluginExperimentReportText(report) {
     else {
         lines.push(...renderRetrievalQueryStrategyComparisonTextLines(retrievalQueryStrategyComparison));
     }
+    // Additive: other plugins keep their existing text output unchanged.
+    const contextPackGeneration = report.contextPackGeneration ?? null;
+    if (contextPackGeneration !== null) {
+        pushSection(lines, "Context Pack Generation");
+        lines.push(...renderContextPackGenerationTextLines(contextPackGeneration));
+    }
     pushSection(lines, "Warnings, Skips, And Failures");
     pushDashList(lines, report.findings.map((finding) => `${finding.severity} ${finding.code} ${finding.message} variant=${finding.variantId ?? ""} case=${finding.caseId ?? ""}`));
     pushSection(lines, "Artifacts");

package/dist/src/report/experiments/renderRetrievalQueryStrategyComparisonHtml.js

@@ -4,9 +4,9 @@ export function renderRetrievalQueryStrategyComparisonHtml(section) {
         return "";
     const methodology = section.methodology;
     const objective = (value, digits) => (value === undefined ? "unavailable" : digits ? value.toFixed(4) : String(value));
-    return `<section>
-    <h2>Retrieval Query Strategy Comparison</h2>
-    <h3>Methodology</h3>
+    return `<section>
+    <h2>Retrieval Query Strategy Comparison</h2>
+    <h3>Methodology</h3>
     ${table(["Field", "Value"], [
         ["Schema", section.schemaVersion],
         ["File F1", methodology.fileF1],
@@ -15,9 +15,9 @@ export function renderRetrievalQueryStrategyComparisonHtml(section) {
         ["Multi-objective comparison", methodology.multiObjectiveComparison],
         ["Unique-best rule", methodology.uniqueBestRule],
         ["Strategy order", section.strategyOrder.join(", ")]
-    ])}
-    ${section.identityRedaction !== null ? `<p><strong>Identity redaction.</strong> ${escapeHtml(RETRIEVAL_QUERY_STRATEGY_EXTERNAL_PRIVACY_NOTE)}</p>` : ""}
-    <h3>Task-Type Comparison</h3>
+    ])}
+    ${section.identityRedaction !== null ? `<p><strong>Identity redaction.</strong> ${escapeHtml(RETRIEVAL_QUERY_STRATEGY_EXTERNAL_PRIVACY_NOTE)}</p>` : ""}
+    <h3>Task-Type Comparison</h3>
     ${table(["Scope", "Cases", "Matched Cases", "Excluded Cases", "Interpretation", "Best Strategy", "Pareto Front"], section.scopes.map((scope) => [
         scope.scopeId,
         String(scope.caseCount),
@@ -26,11 +26,11 @@ export function renderRetrievalQueryStrategyComparisonHtml(section) {
         scope.interpretation,
         scope.bestStrategyId ?? "none",
         formatRetrievalQueryStrategyList(scope.paretoFrontStrategyIds)
-    ]))}
-    ${listHtml(section.scopes.map(describeRetrievalQueryStrategyScope))}
-    <h3>Per-Scope Strategy Objectives</h3>
+    ]))}
+    ${listHtml(section.scopes.map(describeRetrievalQueryStrategyScope))}
+    <h3>Per-Scope Strategy Objectives</h3>
     ${section.scopes
-        .map((scope) => `<h4>${escapeHtml(scope.scopeId)}</h4>
+        .map((scope) => `<h4>${escapeHtml(scope.scopeId)}</h4>
     ${table(["Strategy", "Completed", "Partial", "Failed", "Mean File F1", "Mean Symbol F1", "Mean Fact Coverage", "Mean Retrieved Tokens"], scope.strategySummaries.map((summary) => [
         summary.strategyId,
         String(summary.completedCaseCount),
@@ -41,8 +41,8 @@ export function renderRetrievalQueryStrategyComparisonHtml(section) {
         objective(summary.objectives?.meanFactCoverage, true),
         objective(summary.objectives?.meanRetrievedTokenCount, false)
     ]))}`)
-        .join("\n    ")}
-    <h3>Per-Case Treatments</h3>
+        .join("\n    ")}
+    <h3>Per-Case Treatments</h3>
     ${table([
         "Case",
         "Task Type",
@@ -73,9 +73,9 @@ export function renderRetrievalQueryStrategyComparisonHtml(section) {
         formatRetrievalQueryStrategyMetric(treatment.factCoverage),
         formatRetrievalQueryStrategyMetric(treatment.irrelevantContextRatio),
         formatRetrievalQueryStrategyNumber(treatment.retrievedTokenCount)
-    ])))}
-    <h3>Interpretation Limits</h3>
-    ${listHtml(section.limitations)}
+    ])))}
+    <h3>Interpretation Limits</h3>
+    ${listHtml(section.limitations)}
   </section>`;
 }
 function listHtml(items) {

package/dist/src/report/experiments/writePluginExperimentReports.js

@@ -2,6 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { resolveWithinRoot } from "../../core/pathSafety.js";
 import { buildPluginExperimentReport } from "./buildPluginExperimentReport.js";
+import { loadContextPackArtifacts } from "./loadContextPackArtifacts.js";
 import { renderPluginExperimentReportHtml } from "./renderPluginExperimentReportHtml.js";
 import { renderPluginExperimentReportText } from "./renderPluginExperimentReportText.js";
 export async function writePluginExperimentReports(args) {
@@ -22,6 +23,7 @@ export async function writePluginExperimentReports(args) {
         plugin: args.plugin,
         outputRoot: outputPaths.outDir,
         generatedAt: args.generatedAt,
+        contextPacks: await loadContextPackArtifacts(args.run, outputPaths.outDir),
     });
     const report = args.redactOutputRoot
         ? { ...builtReport, metadata: { ...builtReport.metadata, outputRoot: "[redacted]" } }

package/docs/METRICS.md

@@ -1,6 +1,6 @@
 # Metrics
 
-This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, `incremental-change-staleness`, `context-window-scaling`, `retrieval-precision-recall`, and `retrieval-query-strategy-comparison`).
+This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, `incremental-change-staleness`, `context-window-scaling`, `retrieval-precision-recall`, `retrieval-query-strategy-comparison`, and `context-pack-generation`).
 
 Related documentation:
 - [ARCHITECTURE.md](ARCHITECTURE.md) — how metrics flow through the pipeline
@@ -841,3 +841,30 @@ Retrieved token count is `ceil(character count / 4)` through the existing estima
 `retrieval-query-strategy-comparison-execution.json` holds execution evidence only. `retrieval-query-strategy-comparison-analysis.json` is the scientific truth owner for the metrics, objective vectors, matched scopes, Pareto fronts, and best-strategy interpretation. `report.json`, `report.html`, and `report.txt` present the precomputed analysis and do not recalculate it. The generic experiment metrics (per-treatment values and per-scope objective means) are projections of the same precomputed values.
 
 See [COMMANDS.md](COMMANDS.md#retrieval-query-strategy-comparison-v081), [WORKFLOWS.md](WORKFLOWS.md#retrieval-query-strategy-comparison-experiment-v081), and [ARCHITECTURE.md](ARCHITECTURE.md#retrieval-query-strategy-comparison-architecture-v081).
+
+## Context-pack generation metrics (v0.8.2)
+
+Status: released in v0.8.2. The `context-pack-generation` plugin runs the fixed paired treatments `raw-full-file` and `context-pack`. It calculates each treatment once and persists the execution and analysis artifacts; reports present those values without recalculating them.
+
+### Reused retrieval-quality measures
+
+The plugin reuses the v0.8.0 identity-based measures and availability semantics:
+
+- fact coverage from the answer key's explicit fact-to-context mappings;
+- file precision, recall, and balanced F1;
+- symbol precision, recall, and balanced F1; and
+- irrelevant-context ratio where the existing retrieval-quality calculation applies.
+
+Precision, recall, F1, fact coverage, and irrelevant-context ratio retain their existing numerator, denominator, unavailable, and not-applicable behavior. There is no new composite quality score.
+
+### Estimated size and paired comparison
+
+Both treatments report estimated context size using the existing estimated-token semantics (`ceil(character count / 4)`), not a provider tokenizer or billing measurement. For each matched case, `tokensSaved = rawEstimatedTokens - packEstimatedTokens`; `percentSaved = tokensSaved / rawEstimatedTokens * 100`, with zero when the raw estimate is zero. Negative savings are valid and are not clamped.
+
+For each case, the analysis reports pack-minus-raw objective deltas for file F1, symbol F1, fact coverage, and estimated tokens. A case contributes to scope means only when both treatments have all four required measures available. Scope summaries report included and excluded case counts, treatment objective means, mean tokens saved, and percent saved from the means. Existing task-locality scopes are `overall`, `localized`, `cross-module`, and `broad-change`.
+
+There is no composite score, winner, or ranking. Tests and call relationships may be included descriptively in a context pack, but the frozen answer key does not provide expected test or call-edge identities; v0.8.2 therefore defines no test-coverage or call-edge-coverage score.
+
+In external-local mode, analysis uses real identities before the privacy projection. Redacted placeholders never enter metric calculations. Durable external-local output has no pack body; the report preview presents redacted identities.
+
+See [COMMANDS.md](COMMANDS.md#context-pack-generation-v082), [WORKFLOWS.md](WORKFLOWS.md#context-pack-generation-experiment), and [ARCHITECTURE.md](ARCHITECTURE.md#context-pack-generation-architecture-v082).

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.8.1",
+  "version": "0.8.2",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {