@dailephd/my-dev-kit-lab 0.9.1 → 0.9.3

package/dist/src/securityValidation/validate/runSecurityValidation.js

@@ -80,7 +80,7 @@ export async function runSecurityValidation(options) {
     // missing package.json gracefully.
     if (selectedChecks.has("package")) {
         try {
-            const pkgOutput = await runPackageChecks({ cwd: target.targetRoot, config });
+            const pkgOutput = await runPackageChecks({ cwd: target.targetRoot, config, target });
             collectMany(pkgOutput.checks);
         }
         catch (err) {

package/docs/METRICS.md

@@ -1,6 +1,6 @@
 # Metrics
 
-This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, `incremental-change-staleness`, `context-window-scaling`, `retrieval-precision-recall`, `retrieval-query-strategy-comparison`, and `context-pack-generation`).
+This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, `incremental-change-staleness`, `context-window-scaling`, `retrieval-precision-recall`, `retrieval-query-strategy-comparison`, `context-pack-generation`, and `agent-success-rate`).
 
 Related documentation:
 - [ARCHITECTURE.md](ARCHITECTURE.md) — how metrics flow through the pipeline
@@ -940,10 +940,20 @@ Cumulative versus final: durations and tokens accumulate over all provider attem
 - Provider telemetry can be unavailable, and in deterministic-fixture mode both treatments receive the same reference patch, so those results measure the pipeline only.
 
 
-## Campaign progress evidence (v0.9.1 implemented; unreleased)
+## Campaign progress evidence (v0.9.1 released)
 
 Token evidence levels are `provider-reported`, `cli-reported`, `parsed-from-output`, `unavailable`, and `estimated`. The original telemetry source is retained. Numeric availability is recorded per token field, so partial measurements remain partial and normalization never fills missing fields or invents totals; unattributed OpenTelemetry evidence is unavailable. Duration provenance is `lab-wall-clock` for valid observed elapsed time and `unavailable` for missing or invalid values; campaign comparison time is Lab-measured wall-clock duration, not provider-native time.
 
 The plugin-specific progress projection is `my-dev-kit-lab-agent-success-rate-campaign-progress-v1`; the writer publishes immutable revision-specific JSON, HTML, and text snapshots. It accounts for selected cases and treatment units, while execution epochs remain recovery history and are excluded from selected-unit counts. Paused provider-limit and uncertain states remain distinct. Partial progress is not a complete scientific analysis artifact.
 
 Evidence quality is metric-specific: task outcomes, token measurements, and duration can have different strong, partial, or unavailable classifications. Strong describes the completeness and provenance of the available evidence; it is not statistical significance. Partial evidence does not mean task failure. Missing token evidence is unavailable, never zero; estimated tokens are not provider usage. Duration is Lab-measured wall-clock time, not provider-native time. Comparisons use matched completed cases and state their denominator and exclusions; the existing paired direction is preserved. These descriptive comparisons establish no causal effect, winner, p-value, or general provider performance.
+
+## v0.9.3 report metric evidence and prompt comparison
+
+The plugin report projects recorded metrics into value, unit, availability, reason, provenance, reliability, variant/case identity, and an optional glossary anchor. It adds deterministic caveats for unavailable/not-applicable values, estimates, unknown token provenance/reliability, partial campaign telemetry, and missing glossary metadata. These are presentation projections; the existing JSON report and scientific formulas are unchanged, and the text report presents the same caveats from the same projection. Unknown provenance remains unknown and is never promoted to high reliability.
+
+The descriptor registry in `src/report/experiments/metricGlossary.ts` covers every metric id emitted by the registered experiment plugins. Static ids are listed explicitly; bounded dynamic families are matched by closed patterns: context-window per-budget ids (`budget-<tokens>-utilization/correctness/success` and `context-budget-<tokens>-fit/too-large/successful/success-rate/mean-correctness/mean-utilization`), per task-locality scope ids (`overall|localized|cross-module|broad-change`-prefixed comparison counts and means for retrieval-query-strategy-comparison and context-pack-generation) and per-comparison `-token-savings-percent` / `-correctness-delta` ids of the context-strategy comparison. Definitions are condensed from the calculation owners (and this document); an id outside the static and family sets is rendered without a definition or link and produces a missing-metadata caveat. Agent-success attempt, repair, optional-fact, `patchApplied` and per-attempt provider duration/token metrics are defined from `analysis.ts`/`repairAnalysis.ts`; incremental-change count, upstream-refresh and run-level metrics are defined from the plugin's own metric descriptions. Per-attempt token provenance and reliability are preserved where the execution evidence records them, an aggregate (`totalProviderTokens`, cumulative warm-index tokens) is attributed a provenance only when every contributing measurement is known and agrees, and an aggregate reliability only when all contributors share it.
+
+The standalone prompt comparison reports matched first-attempt invalid-output rates and reports persisted task-success verdicts separately. The numerator requires persisted proof of a completed provider response that violated the prompt/response contract (`OUTPUT_CONTRACT_*` error, verified malformed diff, or a named empty completed answer); a bare provider `invalid-output` status, provider timeouts, unavailability, limits, transport/infrastructure failures, policy rejections and failed task checks are excluded or valid, never counted as invalid. The denominator is eligible completed first-attempt responses on units matched on both sides. Evidence is verified through the campaign store's complete checkpoint validation against trusted task definitions authenticated by task digest. A V1 identity does not bind a template, and a self-validating sidecar plus intact prompt hash is insufficient to prove historical template identity. Without independent trustworthy run-bound proof, V1 identity is `UNVERIFIED` and the comparison is `INCOMPARABLE`.
+
+No matched paid Codex/Claude smoke was authorized. Observed invalid-output-rate improvement remains **NOT_VERIFIED** and must not be claimed. The v0.9.3 release gate covers reproducible prompt-contract classification, trustworthy run-bound identity, matched comparison eligibility/denominators, and report correctness; real-provider effectiveness is a separate, opt-in empirical objective, not a readiness dependency. Prompt comparison capability alone does not demonstrate improvement.

package/examples/tutorial-browser/prepare.mjs

@@ -1,14 +1,14 @@
-import fs from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-const sourceRoot = path.dirname(fileURLToPath(import.meta.url));
-const targetRoot = process.argv[2];
-if (!targetRoot) {
-  console.error("prepare requires a target root argument");
-  process.exit(2);
-}
-fs.mkdirSync(path.join(targetRoot, "app"), { recursive: true });
-fs.mkdirSync(path.join(targetRoot, "out"), { recursive: true });
-for (const file of ["index.html", "server.mjs"]) fs.copyFileSync(path.join(sourceRoot, file), path.join(targetRoot, "app", file));
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const sourceRoot = path.dirname(fileURLToPath(import.meta.url));
+const targetRoot = process.argv[2];
+if (!targetRoot) {
+  console.error("prepare requires a target root argument");
+  process.exit(2);
+}
+fs.mkdirSync(path.join(targetRoot, "app"), { recursive: true });
+fs.mkdirSync(path.join(targetRoot, "out"), { recursive: true });
+for (const file of ["index.html", "server.mjs"]) fs.copyFileSync(path.join(sourceRoot, file), path.join(targetRoot, "app", file));
 fs.writeFileSync(path.join(targetRoot, "out", "report.json"), JSON.stringify({ prepared: true, counts: { widgets: 3 } }, null, 2), "utf8");

package/examples/tutorial-browser/server.mjs

@@ -1,14 +1,14 @@
-import http from "node:http";
-import fs from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-const root = path.dirname(fileURLToPath(import.meta.url));
-const port = Number(process.env.TUTORIAL_FIXTURE_PORT || 0);
-const server = http.createServer((req, res) => {
-  const url = new URL(req.url ?? "/", "http://127.0.0.1");
-  if (url.pathname === "/api/status") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true, widgets: 3 })); return; }
-  const filePath = url.pathname === "/" ? path.join(root, "index.html") : path.join(root, url.pathname);
-  fs.readFile(filePath, (error, contents) => { if (error) { res.writeHead(404); res.end("not found"); return; } res.writeHead(200, { "content-type": filePath.endsWith(".html") ? "text/html" : "application/octet-stream" }); res.end(contents); });
-});
+import http from "node:http";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = path.dirname(fileURLToPath(import.meta.url));
+const port = Number(process.env.TUTORIAL_FIXTURE_PORT || 0);
+const server = http.createServer((req, res) => {
+  const url = new URL(req.url ?? "/", "http://127.0.0.1");
+  if (url.pathname === "/api/status") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true, widgets: 3 })); return; }
+  const filePath = url.pathname === "/" ? path.join(root, "index.html") : path.join(root, url.pathname);
+  fs.readFile(filePath, (error, contents) => { if (error) { res.writeHead(404); res.end("not found"); return; } res.writeHead(200, { "content-type": filePath.endsWith(".html") ? "text/html" : "application/octet-stream" }); res.end(contents); });
+});
 server.listen(port, "127.0.0.1", () => console.log("tutorial fixture listening on " + server.address().port));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {