@dailephd/my-dev-kit-lab 0.9.2 → 0.9.3

package/dist/src/securityValidation/attackScenarios/scenarios/subprocessInjectionScenario.js

@@ -1,9 +1,6 @@
-import fs from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
 import { makeEvidence } from "../exploitEvidence.js";
 import { getPayloadsForGroup } from "../payloadCorpus.js";
-import { runSecurityCommand } from "../../commandRunner.js";
+import { buildSecurityCommandSpawnOptions, runSecurityCommand } from "../../commandRunner.js";
 // ---------------------------------------------------------------------------
 // v0.2.2 Batch 3 — subprocess injection scenario.
 //
@@ -15,31 +12,60 @@ import { runSecurityCommand } from "../../commandRunner.js";
 // that no shell re-interpreted it (splitting it into multiple commands via
 // `;`, `&&`, backticks, `$()`, or `|`).
 //
-// Also does a bounded static check that the command-execution helpers this
-// scenario exercises are built with `shell: false` (source inspection of a
-// known-small file, not a general code scanner).
+// Also verifies at runtime, through the compiled command runner's own
+// spawn-options builder (the exact value runSecurityCommand() spawns with),
+// that the runner does not enable a shell. No TypeScript source is read, so
+// the proof works from an installed package.
 // ---------------------------------------------------------------------------
 const ECHO_ARGV_SCRIPT = "process.stdout.write(JSON.stringify(process.argv.slice(1)))";
-function commandRunnerSourcePath() {
-    const here = path.dirname(fileURLToPath(import.meta.url));
-    return path.resolve(here, "..", "..", "commandRunner.ts");
+const DEFAULT_DEPS = {
+    runCommand: runSecurityCommand,
+    readSpawnOptions: (cwd) => buildSecurityCommandSpawnOptions({ cwd, env: process.env }),
+};
+// Runtime proof that the applicable runner's spawn path does not use a shell.
+// A strict `shell === false` is the only accepted safe value; a truthy shell
+// is demonstrated unsafe; a missing/malformed/throwing proof is unavailable.
+function proveShellDisabled(deps, cwd) {
+    let options;
+    try {
+        options = deps.readSpawnOptions(cwd);
+    }
+    catch (error) {
+        return {
+            state: "unavailable",
+            detail: `Spawn options could not be obtained from the command runner: ${error instanceof Error ? error.message : String(error)}`,
+        };
+    }
+    if (typeof options !== "object" || options === null || !("shell" in options)) {
+        return { state: "unavailable", detail: "Command runner spawn options did not declare a shell setting." };
+    }
+    const shell = options.shell;
+    if (shell === false)
+        return { state: "safe" };
+    if (shell === true || typeof shell === "string") {
+        return { state: "unsafe", detail: "Command runner spawn options enable a shell." };
+    }
+    return { state: "unavailable", detail: "Command runner spawn options declared a malformed shell setting." };
 }
 export const SUBPROCESS_INJECTION_SCENARIO = {
     id: "subprocess-injection-safety",
     title: "Subprocess injection: shell metacharacter payloads are passed as literal arguments",
-    description: "Runs a benign argv-echoing Node script with each subprocess-injection payload as a single spawn argument and confirms it round-trips as exactly one literal string, proving no shell re-interpretation. Also checks that commandRunner.ts constructs subprocess calls with shell:false.",
+    description: "Runs a benign argv-echoing Node script with each subprocess-injection payload as a single spawn argument and confirms it round-trips as exactly one literal string, proving no shell re-interpretation. Also verifies at runtime that the command runner constructs subprocess calls with shell:false.",
     checkId: "subprocess",
     applicableProfiles: [],
     severityBaseline: "blocker",
     verdictImpact: "tool-framework-blocker",
-    expectedSafeBehavior: "Every subprocess-injection payload round-trips as a single literal argv element; commandRunner.ts uses shell:false.",
+    expectedSafeBehavior: "Every subprocess-injection payload round-trips as a single literal argv element; the command runner uses shell:false.",
     evidenceRequirements: ["command", "observation"],
-    run: async (ctx) => {
+    run: (ctx) => runSubprocessInjectionScenario(ctx, DEFAULT_DEPS),
+};
+export async function runSubprocessInjectionScenario(ctx, deps) {
+    {
         const evidence = [];
         const notLiteral = [];
         const payloads = getPayloadsForGroup("subprocess-injection");
         for (const p of payloads) {
-            const cmd = await runSecurityCommand({
+            const cmd = await deps.runCommand({
                 command: process.execPath,
                 args: ["-e", ECHO_ARGV_SCRIPT, p.value],
                 cwd: ctx.toolRoot,
@@ -70,42 +96,34 @@ export const SUBPROCESS_INJECTION_SCENARIO = {
                 rawPreview: p.value,
             }));
         }
-        let shellFalseConfirmed = false;
-        let sourceReadable = true;
-        try {
-            const source = fs.readFileSync(commandRunnerSourcePath(), "utf8");
-            shellFalseConfirmed = /shell:\s*false/.test(source);
-        }
-        catch {
-            sourceReadable = false;
-        }
+        const proof = proveShellDisabled(deps, ctx.toolRoot);
         evidence.push(makeEvidence({
             kind: "observation",
-            source: "static check: src/securityValidation/commandRunner.ts",
-            confidence: sourceReadable ? "high" : "low",
+            source: "runtime check: buildSecurityCommandSpawnOptions() used by runSecurityCommand()",
+            confidence: proof.state === "unavailable" ? "low" : "high",
             expectedBehavior: "Subprocess calls should be constructed with shell:false.",
-            observedBehavior: sourceReadable
-                ? shellFalseConfirmed
-                    ? "shell:false found in commandRunner.ts."
-                    : "shell:false NOT found in commandRunner.ts."
-                : "Could not read commandRunner.ts source for static confirmation.",
+            observedBehavior: proof.state === "safe"
+                ? "Command runner spawn options declare shell:false."
+                : proof.state === "unsafe"
+                    ? proof.detail
+                    : `Could not verify shell:false: ${proof.detail}`,
         }));
-        if (notLiteral.length > 0 || (sourceReadable && !shellFalseConfirmed)) {
+        if (notLiteral.length > 0 || proof.state === "unsafe") {
             return {
                 status: "failed",
                 confidence: "high",
                 evidence,
                 recommendation: notLiteral.length > 0
                     ? `Investigate argument construction — payload(s) did not round-trip literally: ${notLiteral.join(", ")}.`
-                    : "commandRunner.ts must construct child_process.spawn calls with shell:false.",
+                    : "The command runner must construct child_process.spawn calls with shell:false.",
             };
         }
-        if (!sourceReadable) {
+        if (proof.state === "unavailable") {
             return {
                 status: "blocked",
                 confidence: "low",
                 evidence,
-                recommendation: "Could not statically confirm shell:false; argv round-trip evidence is still valid.",
+                recommendation: "Could not verify shell:false for the command runner; argv round-trip evidence alone is not sufficient.",
             };
         }
         return {
@@ -113,5 +131,5 @@ export const SUBPROCESS_INJECTION_SCENARIO = {
             confidence: "high",
             evidence,
         };
-    },
-};
+    }
+}

package/dist/src/securityValidation/commandRunner.js

@@ -5,6 +5,18 @@ import { resolveCommand } from "../core/resolveCommand.js";
 export function resolveNpmCommand() {
     return "npm";
 }
+// The single place the spawn options for security commands are constructed.
+// runSecurityCommand() spawns with exactly this result, so the subprocess
+// injection scenario can verify `shell: false` at runtime from the compiled
+// module instead of reading unshipped TypeScript source.
+export function buildSecurityCommandSpawnOptions(options) {
+    return {
+        cwd: options.cwd,
+        shell: false,
+        stdio: ["ignore", "pipe", "pipe"],
+        env: options.env,
+    };
+}
 // Runs a security check command with an argument array (no shell interpolation).
 // npm audit returns exit code 1 on vulnerabilities; that is not treated as a
 // spawn failure here — the caller inspects the output and exit code separately.
@@ -52,12 +64,7 @@ export async function runSecurityCommand(options) {
         let timeout;
         let child;
         try {
-            child = spawn(command, args, {
-                cwd: options.cwd,
-                shell: false,
-                stdio: ["ignore", "pipe", "pipe"],
-                env,
-            });
+            child = spawn(command, args, buildSecurityCommandSpawnOptions({ cwd: options.cwd, env }));
         }
         catch (spawnErr) {
             resolve({

package/docs/METRICS.md

@@ -1,6 +1,6 @@
 # Metrics
 
-This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, `incremental-change-staleness`, `context-window-scaling`, `retrieval-precision-recall`, `retrieval-query-strategy-comparison`, and `context-pack-generation`).
+This document is the canonical metric glossary for my-dev-kit-lab. It defines every metric that appears in benchmark profiles, prompt variants, controlled experiment artifacts, and rendered reports across the registered experiment plugins (`context-strategy-comparison`, `warm-index-reuse`, `incremental-change-staleness`, `context-window-scaling`, `retrieval-precision-recall`, `retrieval-query-strategy-comparison`, `context-pack-generation`, and `agent-success-rate`).
 
 Related documentation:
 - [ARCHITECTURE.md](ARCHITECTURE.md) — how metrics flow through the pipeline
@@ -947,3 +947,13 @@ Token evidence levels are `provider-reported`, `cli-reported`, `parsed-from-outp
 The plugin-specific progress projection is `my-dev-kit-lab-agent-success-rate-campaign-progress-v1`; the writer publishes immutable revision-specific JSON, HTML, and text snapshots. It accounts for selected cases and treatment units, while execution epochs remain recovery history and are excluded from selected-unit counts. Paused provider-limit and uncertain states remain distinct. Partial progress is not a complete scientific analysis artifact.
 
 Evidence quality is metric-specific: task outcomes, token measurements, and duration can have different strong, partial, or unavailable classifications. Strong describes the completeness and provenance of the available evidence; it is not statistical significance. Partial evidence does not mean task failure. Missing token evidence is unavailable, never zero; estimated tokens are not provider usage. Duration is Lab-measured wall-clock time, not provider-native time. Comparisons use matched completed cases and state their denominator and exclusions; the existing paired direction is preserved. These descriptive comparisons establish no causal effect, winner, p-value, or general provider performance.
+
+## v0.9.3 report metric evidence and prompt comparison
+
+The plugin report projects recorded metrics into value, unit, availability, reason, provenance, reliability, variant/case identity, and an optional glossary anchor. It adds deterministic caveats for unavailable/not-applicable values, estimates, unknown token provenance/reliability, partial campaign telemetry, and missing glossary metadata. These are presentation projections; the existing JSON report and scientific formulas are unchanged, and the text report presents the same caveats from the same projection. Unknown provenance remains unknown and is never promoted to high reliability.
+
+The descriptor registry in `src/report/experiments/metricGlossary.ts` covers every metric id emitted by the registered experiment plugins. Static ids are listed explicitly; bounded dynamic families are matched by closed patterns: context-window per-budget ids (`budget-<tokens>-utilization/correctness/success` and `context-budget-<tokens>-fit/too-large/successful/success-rate/mean-correctness/mean-utilization`), per task-locality scope ids (`overall|localized|cross-module|broad-change`-prefixed comparison counts and means for retrieval-query-strategy-comparison and context-pack-generation) and per-comparison `-token-savings-percent` / `-correctness-delta` ids of the context-strategy comparison. Definitions are condensed from the calculation owners (and this document); an id outside the static and family sets is rendered without a definition or link and produces a missing-metadata caveat. Agent-success attempt, repair, optional-fact, `patchApplied` and per-attempt provider duration/token metrics are defined from `analysis.ts`/`repairAnalysis.ts`; incremental-change count, upstream-refresh and run-level metrics are defined from the plugin's own metric descriptions. Per-attempt token provenance and reliability are preserved where the execution evidence records them, an aggregate (`totalProviderTokens`, cumulative warm-index tokens) is attributed a provenance only when every contributing measurement is known and agrees, and an aggregate reliability only when all contributors share it.
+
+The standalone prompt comparison reports matched first-attempt invalid-output rates and reports persisted task-success verdicts separately. The numerator requires persisted proof of a completed provider response that violated the prompt/response contract (`OUTPUT_CONTRACT_*` error, verified malformed diff, or a named empty completed answer); a bare provider `invalid-output` status, provider timeouts, unavailability, limits, transport/infrastructure failures, policy rejections and failed task checks are excluded or valid, never counted as invalid. The denominator is eligible completed first-attempt responses on units matched on both sides. Evidence is verified through the campaign store's complete checkpoint validation against trusted task definitions authenticated by task digest. A V1 identity does not bind a template, and a self-validating sidecar plus intact prompt hash is insufficient to prove historical template identity. Without independent trustworthy run-bound proof, V1 identity is `UNVERIFIED` and the comparison is `INCOMPARABLE`.
+
+No matched paid Codex/Claude smoke was authorized. Observed invalid-output-rate improvement remains **NOT_VERIFIED** and must not be claimed. The v0.9.3 release gate covers reproducible prompt-contract classification, trustworthy run-bound identity, matched comparison eligibility/denominators, and report correctness; real-provider effectiveness is a separate, opt-in empirical objective, not a readiness dependency. Prompt comparison capability alone does not demonstrate improvement.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.9.2",
+  "version": "0.9.3",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {