@dailephd/my-dev-kit-lab 0.6.1 → 0.6.3

package/dist/src/securityValidation/staticScans/codeql.js

@@ -1,18 +1,18 @@
 import { runSecurityCommand } from "../commandRunner.js";
 import { resolveCommand } from "../../core/resolveCommand.js";
 import { skippedCheck } from "../cliAdversarial/runAdversarialCheck.js";
-// CodeQL is primarily a GitHub Actions / code-scanning integration.
-// Local CodeQL CLI is optional — if unavailable the check is skipped.
-// Absence is not a release blocker; it surfaces as "ready except optional checks".
+// This local check only verifies CodeQL CLI availability and integration.
+// Full CodeQL analysis is owned by .github/workflows/codeql.yml.
+// Release readiness requires successful GitHub analysis for the exact candidate SHA.
 export async function runCodeqlCheck(options) {
     const { cwd, timeoutMs } = options;
     const resolved = resolveCommand("codeql", { cwd, env: process.env });
     if (resolved.resolutionKind === "unavailable") {
         return skippedCheck({
             id: "codeql-scan",
-            name: "CodeQL static analysis",
+            name: "CodeQL CLI availability preflight",
             category: "static-scan",
-            reason: "CodeQL CLI not found in PATH. Full analysis runs via GitHub Actions code-scanning workflow. Install the CodeQL CLI locally for local analysis.",
+            reason: "Local CodeQL CLI is unavailable, so the local availability preflight is skipped. Full release-readiness coverage requires a successful GitHub CodeQL advanced-setup analysis for the exact candidate SHA.",
         });
     }
     const startedAt = new Date().toISOString();
@@ -29,7 +29,7 @@ export async function runCodeqlCheck(options) {
     if (versionCmd.exitCode !== 0 || versionCmd.exitCode === null) {
         return {
             id: "codeql-scan",
-            name: "CodeQL static analysis",
+            name: "CodeQL CLI availability preflight",
             category: "static-scan",
             status: "failed",
             severity: "major",
@@ -50,10 +50,10 @@ export async function runCodeqlCheck(options) {
             command: "codeql version --format terse",
         };
     }
-    // CLI is present and functional. Full analysis runs in GitHub Actions.
+    // The CLI is present and functional; this is not repository-wide analysis.
     return {
         id: "codeql-scan",
-        name: "CodeQL static analysis",
+        name: "CodeQL CLI availability preflight",
         category: "static-scan",
         status: "passed",
         severity: "informational",

package/docs/METRICS.md

@@ -1,6 +1,6 @@
 # Metrics
 
-This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports.
+This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, and `incremental-change-staleness`).
 
 Related documentation:
 - [ARCHITECTURE.md](ARCHITECTURE.md) — how metrics flow through the pipeline
@@ -595,3 +595,69 @@ All six metrics are warm-side only (they are not on the raw-full-file side, beca
 **Report presentation.** `report.warmIndexReuse.affectedNeighborhoodSummary` holds presentation counts (assessed and unassessed tasks; complete, partial, and unavailable assessments; related, unrelated, and unknown relationships; recommended, not-indicated, and unknown recommendations). They are counts, not metric formulas. Each task's `affectedNeighborhood` block shows the statuses, the six metric objects, the relationship, the recommendation with a fixed explanation, and bounded lists (at most 20 affected node IDs, 20 edge IDs, 10 unresolved and 10 ambiguous task mappings, and 10 warnings, each with its total and omitted count). The full identity arrays stay in `warm-index-execution.json`.
 
 **Observational only.** The evidence never triggers reindexing, suppresses or alters warm retrieval, or changes execution, provider, agent, correctness, or token-evidence status. It uses only the baseline graph of the prepared index: there is no graph-diff, refreshed-index, or stale-versus-refreshed comparison.
+
+## Incremental-change and staleness evidence (v0.6.2)
+
+The `incremental-change-staleness` plugin reuses the six existing v0.6.1 affected-neighborhood numeric metrics unchanged (`changedFileCount`, `changedSymbolCount`, `affectedNodeCount`, `affectedEdgeCount`, `taskOverlapCount`, `taskOverlapPercent`; see "Affected-neighborhood metrics (v0.6.1)" above) and adds no new numeric formula.
+
+**Correctness relation** (per scenario, comparing the `stale-index` and `full-refresh` treatments' deterministic fake-agent correctness):
+
+- `stale-worse`: `stale-index` correctness is lower than `full-refresh` correctness.
+- `same`: both treatments have available, equal correctness.
+- `stale-better`: `stale-index` correctness is higher than `full-refresh` correctness.
+- `unknown`: either treatment's correctness is unavailable.
+
+**Required-file status** (per treatment, an answer-key file-presence check against that treatment's actual retrieved-files evidence — not retrieval precision, recall, F1, MRR, NDCG, or symbol recall):
+
+- `present`: every required file was observed.
+- `missing`: at least one required file was not observed, and the observed-files evidence was complete.
+- `unknown`: the actual-files-read evidence was not complete enough to establish presence or absence.
+
+**Required-file relation** (per scenario, comparing the two treatments' required-file status):
+
+- `stale-worse`: `stale-index` is `missing` while `full-refresh` is `present`.
+- `same`: both treatments have the same status (including `missing`/`missing`).
+- `stale-better`: `stale-index` is `present` while `full-refresh` is `missing`.
+- `unknown`: either treatment's status is `unknown`.
+
+**Stale-risk classification** (per scenario, the frozen precedence over the two relations above):
+
+1. `observed-stale-regression` — the correctness relation or the required-file relation is `stale-worse`.
+2. `inconclusive` — otherwise, if the correctness relation or the required-file relation is `unknown`.
+3. `no-observed-stale-regression` — otherwise.
+
+`staleRiskClassification` is not a numeric risk score, not a winner, not a ranking, and not an automatic reindex decision. `no-observed-stale-regression` means only that this matched scenario did not observe a stale-specific regression under these two frozen evidence dimensions; it does not establish that stale indexes are generally safe. `reindexRecommendation` (the released v0.6.1 categorical evidence) remains separate observational evidence about the affected-neighborhood analysis; it does not select or trigger either treatment and is not an input to the stale-risk classification above.
+
+**Report presentation.** `report.incrementalChangeStaleness` holds `scenarioCount`, `readyScenarioCount`, `failedScenarioCount`, `observedStaleRegressionCount`, `noObservedStaleRegressionCount`, `inconclusiveCount`, per-scenario evidence (lifecycle, both treatments in order, and the comparison), and fixed limitations text. There is no overall score, grade, winner, best-treatment, or safe-to-skip-reindex field anywhere in the report or the persisted execution artifact.
+
+### Partial-refresh treatment evidence (v0.6.3, current release)
+
+The `incremental-change-staleness` plugin now compares four treatments (`stale-index`, `changed-files-refresh`, `affected-neighborhood-refresh`, `full-refresh`). It defines no new formula. Two evidence families must not be merged:
+
+- **Lab task-relationship affected-neighborhood evidence** (released v0.6.1, unchanged): the six numeric metrics `changedFileCount`, `changedSymbolCount`, `affectedNodeCount`, `affectedEdgeCount`, `taskOverlapCount`, `taskOverlapPercent`, plus the categorical `relationship` and `reindexRecommendation`. They describe the baseline graph neighborhood of the controlled change relative to the task. `changedSymbolCount` still means baseline symbol identities belonging to confirmed changed indexed files, not proof that any symbol body changed, and `reindexRecommendation` stays observational: it selects and executes no treatment.
+- **Upstream refresh execution evidence** (new): what my-dev-kit actually did during a partial refresh invocation. It does not depend on the task and is not a task-relevance measure.
+
+**Numeric upstream-refresh metrics** (unit `count`; emitted as `ExperimentMetric` entries on the outcome of an incremental treatment, `changed-files-refresh` or `affected-neighborhood-refresh`, and never on `stale-index` or `full-refresh`):
+
+| Metric ID | Meaning |
+|---|---|
+| `upstream-refresh-fresh-extraction-file-count` | Files upstream freshly re-extracted during the refresh invocation. |
+| `upstream-refresh-reused-file-count` | Files upstream reused from the previous index. |
+| `upstream-refresh-forced-neighbor-reanalysis-file-count` | Neighboring files upstream forced to re-analyze during the refresh. |
+| `upstream-refresh-seed-file-count` | Seed files upstream selected for the affected-neighborhood refresh. |
+| `upstream-refresh-seed-symbol-count` | Seed symbols upstream selected. |
+| `upstream-refresh-affected-node-count` | Graph nodes upstream treated as affected. |
+| `upstream-refresh-affected-edge-count` | Graph edges upstream treated as affected. |
+
+Availability: the first three are always present for a validated incremental treatment. The seed, node, and edge counts appear only when upstream supplied a non-null value; a `null` upstream value is unavailable evidence and the metric is omitted, never emitted as zero. These counts describe upstream execution only: they do not prove a treatment is better, faster, safer, or more accurate, and they are distinct from the Lab `affectedNodeCount`/`affectedEdgeCount` above (different metric IDs).
+
+**Categorical refresh evidence** (not numeric metrics, never converted to numbers): `requestedScope` (`changed-files` or `affected-neighborhood`), `appliedScope` (the scope upstream actually applied: `changed-files`, `affected-neighborhood`, `full`, or `none`), `selectionStatus` (`applied`, `fallback-full`, or `not-needed`), `fallbackReason` (an upstream reason or `null`), `forcedNeighborSample` (a bounded sample of path strings, capped by upstream, so its length can be less than the forced-neighbor count), and the Lab `refreshRealization` (`NO_REFRESH`, `APPLIED_PARTIAL`, `FALLBACK_FULL`, `FULL_REFRESH`). Only `changed-files-refresh` and `affected-neighborhood-refresh` can be `APPLIED_PARTIAL` or `FALLBACK_FULL`.
+
+**Reference comparisons** (exactly three per ready scenario, each against `full-refresh`):
+
+- `stale-index` vs `full-refresh` uses the v0.6.2 stale-risk vocabulary and precedence unchanged (`observed-stale-regression`, `no-observed-stale-regression`, `inconclusive`).
+- `changed-files-refresh` vs `full-refresh` and `affected-neighborhood-refresh` vs `full-refresh` use neutral candidate relations (`candidate-worse`, `same`, `candidate-better`, `unknown`) for deterministic fake-agent correctness and for the required-file status, with the frozen precedence: (1) `not-comparable-as-partial-refresh` if the candidate fell back to a full rebuild (`FALLBACK_FULL`); (2) `observed-regression-relative-to-full` if correctness or required-file presence is `candidate-worse`; (3) `inconclusive` if either relation is `unknown`; (4) otherwise `no-observed-regression-relative-to-full`.
+
+`no-observed-regression-relative-to-full` means only that the executed scenario showed no regression under those two evidence dimensions. It does not establish equivalence or safety and does not show that full refresh is unnecessary. `not-comparable-as-partial-refresh` says the run gives no evidence about the requested partial behavior. There is no winner, ranking, composite score, safety score, safe-to-skip-refresh flag, or stale-risk percentage. Ordinary per-treatment correctness `score` remains the existing deterministic fake-agent evidence.
+
+**Report presentation.** `report.incrementalChangeStaleness` (schema `my-dev-kit-lab-incremental-change-staleness-report-v2`) holds the summary counts (including per-treatment applied-partial, fallback, and classification counts), per-scenario lifecycle, four treatments and three comparisons identified by explicit treatment IDs, and fixed limitations text. Unavailable evidence is written as `unavailable` or `not applicable`, never zero. Historical V1 reports keep their original two-treatment presentation.

package/examples/tutorial-browser/prepare.mjs

@@ -1,14 +1,14 @@
-import fs from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-const sourceRoot = path.dirname(fileURLToPath(import.meta.url));
-const targetRoot = process.argv[2];
-if (!targetRoot) {
-  console.error("prepare requires a target root argument");
-  process.exit(2);
-}
-fs.mkdirSync(path.join(targetRoot, "app"), { recursive: true });
-fs.mkdirSync(path.join(targetRoot, "out"), { recursive: true });
-for (const file of ["index.html", "server.mjs"]) fs.copyFileSync(path.join(sourceRoot, file), path.join(targetRoot, "app", file));
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const sourceRoot = path.dirname(fileURLToPath(import.meta.url));
+const targetRoot = process.argv[2];
+if (!targetRoot) {
+  console.error("prepare requires a target root argument");
+  process.exit(2);
+}
+fs.mkdirSync(path.join(targetRoot, "app"), { recursive: true });
+fs.mkdirSync(path.join(targetRoot, "out"), { recursive: true });
+for (const file of ["index.html", "server.mjs"]) fs.copyFileSync(path.join(sourceRoot, file), path.join(targetRoot, "app", file));
 fs.writeFileSync(path.join(targetRoot, "out", "report.json"), JSON.stringify({ prepared: true, counts: { widgets: 3 } }, null, 2), "utf8");

package/examples/tutorial-browser/server.mjs

@@ -1,14 +1,14 @@
-import http from "node:http";
-import fs from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-const root = path.dirname(fileURLToPath(import.meta.url));
-const port = Number(process.env.TUTORIAL_FIXTURE_PORT || 0);
-const server = http.createServer((req, res) => {
-  const url = new URL(req.url ?? "/", "http://127.0.0.1");
-  if (url.pathname === "/api/status") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true, widgets: 3 })); return; }
-  const filePath = url.pathname === "/" ? path.join(root, "index.html") : path.join(root, url.pathname);
-  fs.readFile(filePath, (error, contents) => { if (error) { res.writeHead(404); res.end("not found"); return; } res.writeHead(200, { "content-type": filePath.endsWith(".html") ? "text/html" : "application/octet-stream" }); res.end(contents); });
-});
+import http from "node:http";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = path.dirname(fileURLToPath(import.meta.url));
+const port = Number(process.env.TUTORIAL_FIXTURE_PORT || 0);
+const server = http.createServer((req, res) => {
+  const url = new URL(req.url ?? "/", "http://127.0.0.1");
+  if (url.pathname === "/api/status") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true, widgets: 3 })); return; }
+  const filePath = url.pathname === "/" ? path.join(root, "index.html") : path.join(root, url.pathname);
+  fs.readFile(filePath, (error, contents) => { if (error) { res.writeHead(404); res.end("not found"); return; } res.writeHead(200, { "content-type": filePath.endsWith(".html") ? "text/html" : "application/octet-stream" }); res.end(contents); });
+});
 server.listen(port, "127.0.0.1", () => console.log("tutorial fixture listening on " + server.address().port));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.6.1",
+  "version": "0.6.3",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {