@dailephd/my-dev-kit-lab 0.9.0 → 0.9.2

package/dist/src/securityValidation/packageChecks/runPackageChecks.js

@@ -3,7 +3,9 @@ import { runSecurityCommand, resolveNpmCommand } from "../commandRunner.js";
 import { writeCheckResult } from "../artifacts.js";
 import { parseNpmPackDryRun } from "./parseNpmPackDryRun.js";
 import { detectForbiddenContents } from "./forbiddenPackageContents.js";
-import { detectMissingRequiredContents } from "./requiredPackageContents.js";
+import { detectMissingRequiredContents, requiredContentsForTarget } from "./requiredPackageContents.js";
+import { resolveValidationTarget } from "../validate/resolveTarget.js";
+import { discoverPackageRoot } from "../../runtime/packageRoot.js";
 // Run npm pack --dry-run and inspect the resulting file list for forbidden contents.
 // Does not publish anything.
 export async function runPackageChecks(options) {
@@ -12,20 +14,23 @@ export async function runPackageChecks(options) {
     const allFindings = [];
     const checks = [];
     const npm = resolveNpmCommand();
+    const target = options.target ?? resolveValidationTarget(path.resolve(cwd), discoverPackageRoot());
+    if (path.resolve(target.targetRoot) !== path.resolve(cwd)) {
+        throw new Error("Package check cwd does not match the resolved target root");
+    }
+    const policy = await requiredContentsForTarget(target);
     // npm pack --dry-run to get the tarball file list
     const startedAt = new Date().toISOString();
     const cmd = await runSecurityCommand({
         command: npm,
-        args: ["pack", "--dry-run"],
+        args: ["pack", "--dry-run", "--json", "--ignore-scripts", "--cache", path.resolve(rawOutputDir, "npm-cache")],
         cwd,
         timeoutMs: commandTimeoutMs,
     });
     const finishedAt = new Date().toISOString();
-    // npm pack --dry-run writes the tarball filename to stdout on some npm
-    // versions and the detailed file list to stderr. Prefer whichever stream
-    // contains the tarball contents section, then fall back to combined output.
-    const packOutput = [cmd.stdout, cmd.stderr].find((stream) => /tarball contents/i.test(stream)) ??
-        [cmd.stdout, cmd.stderr].filter(Boolean).join("\n");
+    // Supported npm versions emit JSON on stdout. Lifecycle scripts are disabled
+    // so inspection cannot build, install, or rewrite the target.
+    const packOutput = cmd.stdout;
     const parsed = parseNpmPackDryRun(packOutput);
     const { findings: contentFindings } = detectForbiddenContents({
         files: parsed.files,
@@ -36,20 +41,28 @@ export async function runPackageChecks(options) {
     const { findings: missingRequiredFindings } = detectMissingRequiredContents({
         files: parsed.files,
         checkId: "npm-pack",
+        requiredFiles: policy.requiredFiles,
     });
-    const packageFindings = [...contentFindings, ...missingRequiredFindings];
+    const evidenceFindings = [];
+    if (cmd.exitCode !== 0 || cmd.timedOut || cmd.skipped || parsed.parseError || parsed.files.length === 0) {
+        evidenceFindings.push({
+            id: "npm-pack-evidence-invalid",
+            title: "npm package inventory could not be validated",
+            severity: "blocker",
+            category: "package-content",
+            description: "A successful npm pack operation and a nonempty, parseable inventory are required.",
+            evidence: `exitCode=${String(cmd.exitCode)}; timedOut=${cmd.timedOut}; skipped=${cmd.skipped}; ${parsed.parseError ?? ""}; ${cmd.stderr.slice(0, 500)}`,
+            recommendation: "Resolve the package command or inventory failure and rerun validation.",
+            releaseImpact: "Blocker: required package evidence is unavailable",
+        });
+    }
+    const packageFindings = [...policy.findings, ...evidenceFindings, ...contentFindings, ...missingRequiredFindings];
     allFindings.push(...packageFindings);
     const packCheck = {
         id: "npm-pack-dry-run",
         name: "npm pack --dry-run (tarball file list)",
         category: "package-content",
-        status: cmd.timedOut
-            ? "failed"
-            : parsed.parseError && parsed.files.length === 0
-                ? "warning"
-                : packageFindings.length > 0
-                    ? "failed"
-                    : "passed",
+        status: packageFindings.length > 0 ? "failed" : "passed",
         severity: packageFindings.length > 0
             ? packageFindings.some((f) => f.severity === "blocker")
                 ? "blocker"
@@ -60,7 +73,9 @@ export async function runPackageChecks(options) {
         durationMs: cmd.durationMs,
         findings: packageFindings,
         skippedReason: undefined,
-        command: "npm pack --dry-run",
+        command: "npm pack --dry-run --json --ignore-scripts",
+        commandCwd: cwd,
+        exitCode: cmd.exitCode,
     };
     await writeCheckResult({
         result: packCheck,

package/dist/src/securityValidation/staticScans/semgrep.js

@@ -5,16 +5,35 @@ import { resolveCommand } from "../../core/resolveCommand.js";
 import { skippedCheck } from "../cliAdversarial/runAdversarialCheck.js";
 export function parseSemgrepJson(raw) {
     if (!raw.trim()) {
-        return { findings: [], rawOutput: raw };
+        return { findings: [], parseError: "Semgrep produced no JSON scan evidence", rawOutput: raw };
     }
     try {
         const parsed = JSON.parse(raw);
-        const results = Array.isArray(parsed["results"]) ? parsed["results"] : [];
-        const findings = results.map((r) => {
+        if (!isRecord(parsed) || !Array.isArray(parsed.results)) {
+            return { findings: [], parseError: "Semgrep JSON requires a results array", rawOutput: raw };
+        }
+        const findings = [];
+        let parseError = !Array.isArray(parsed.errors) || !isRecord(parsed.paths) ||
+            !Array.isArray(parsed.paths.scanned) || parsed.paths.scanned.some((file) => typeof file !== "string")
+            ? "Semgrep JSON requires errors and paths.scanned arrays" : undefined;
+        for (const [index, r] of parsed.results.entries()) {
+            if (!isRecord(r) || typeof r.check_id !== "string" || !r.check_id ||
+                typeof r.path !== "string" || !r.path || !isRecord(r.start) ||
+                !Number.isInteger(r.start.line) || Number(r.start.line) < 1 || !isRecord(r.extra) ||
+                (r.extra.message !== undefined && typeof r.extra.message !== "string") ||
+                (r.extra.metadata !== undefined && !isRecord(r.extra.metadata))) {
+                parseError ??= `Malformed Semgrep result at index ${index}`;
+                continue;
+            }
             const item = r;
             const checkId = typeof item["check_id"] === "string" ? item["check_id"] : "unknown";
             const extra = item["extra"] ?? {};
             const metadata = extra["metadata"] ?? {};
+            if ((extra.severity !== undefined && (typeof extra.severity !== "string" || !extra.severity.trim())) ||
+                (extra.severity === undefined && (typeof metadata.severity !== "string" || !metadata.severity.trim()))) {
+                parseError ??= `Missing or malformed Semgrep severity at index ${index}`;
+                continue;
+            }
             const severity = typeof extra["severity"] === "string"
                 ? extra["severity"]
                 : typeof metadata["severity"] === "string"
@@ -24,9 +43,14 @@ export function parseSemgrepJson(raw) {
             const filePath = typeof item["path"] === "string" ? item["path"] : "";
             const startObj = item["start"] ?? {};
             const line = typeof startObj["line"] === "number" ? startObj["line"] : 0;
-            return { ruleId: checkId, severity, message, path: filePath, line };
-        });
-        return { findings, rawOutput: raw };
+            findings.push({ ruleId: checkId, severity, message, path: filePath, line });
+        }
+        // Semgrep's CLI contract reports partial/fatal analysis in errors even
+        // when the process exits zero. Retain matches, but never call this clean.
+        const analysisError = Array.isArray(parsed.errors) && parsed.errors.length > 0
+            ? `Semgrep reported ${parsed.errors.length} analysis error(s): ${JSON.stringify(parsed.errors).slice(0, 500)}`
+            : undefined;
+        return { findings, parseError, analysisError, rawOutput: raw };
     }
     catch (err) {
         return {
@@ -36,6 +60,9 @@ export function parseSemgrepJson(raw) {
         };
     }
 }
+function isRecord(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
 function semgrepSeverityToSecurity(semgrepSeverity) {
     switch (semgrepSeverity.toUpperCase()) {
         case "ERROR":
@@ -88,67 +115,25 @@ export async function runSemgrepCheck(options) {
         "--quiet",
         scanTarget,
     ];
-    const cmd = await runSecurityCommand({
-        command,
-        args: scanArgs,
-        cwd: targetRoot,
-        timeoutMs,
-    });
+    let cmd;
+    try {
+        cmd = await runSecurityCommand({ command, args: scanArgs, cwd: targetRoot, timeoutMs });
+    }
+    catch (error) {
+        cmd = { command, args: scanArgs, cwd: targetRoot, exitCode: null, durationMs: 0,
+            stdout: "", stderr: error instanceof Error ? error.message : String(error), timedOut: false, skipped: false };
+    }
     const finishedAt = new Date().toISOString();
-    // Semgrep exits 0 = no findings, 1 = findings found, other = error.
-    const isError = cmd.exitCode !== 0 && cmd.exitCode !== 1;
-    const isTimedOut = cmd.timedOut;
-    if (isTimedOut || (isError && !cmd.stdout.trim())) {
-        return {
-            id: "semgrep-scan",
-            name: "Semgrep static analysis",
-            category: "static-scan",
-            status: "failed",
-            severity: "major",
-            startedAt,
-            finishedAt,
-            durationMs: cmd.durationMs,
-            findings: [
-                {
-                    id: "semgrep-execution-error",
-                    title: "Semgrep execution failed",
-                    severity: "major",
-                    category: "static-scan",
-                    description: isTimedOut
-                        ? "Semgrep timed out."
-                        : `Semgrep exited with code ${String(cmd.exitCode)}. stderr: ${cmd.stderr.slice(0, 500)}`,
-                    recommendation: "Check semgrep installation and .semgrep.yml config.",
-                    releaseImpact: "Review before release",
-                },
-            ],
-            command: [command, ...scanArgs].join(" "),
-        };
+    // Distinguish positively established absence from a failed attempted scan.
+    // Network, permission, generic npm, or native spawn errors are not absence.
+    const unavailable = !cmd.timedOut && cmd.exitCode !== 0 && !cmd.stdout.trim() && ((cmd.skipped && cmd.exitCode === null && Boolean(cmd.skippedReason?.match(/\b(semgrep|scanner|cli)\b.*(unavailable|not installed|not found|not on PATH)/i))) ||
+        (useNpx && !cmd.skipped && cmd.exitCode === 1 && /could not determine executable to run/i.test(cmd.stderr)));
+    if (unavailable) {
+        return { ...skippedCheck({ id: "semgrep-scan", name: "Semgrep static analysis", category: "static-scan",
+                reason: cmd.skippedReason ?? "Semgrep CLI is unavailable; the npx fallback package provides no executable." }),
+            command: [command, ...scanArgs].join(" "), commandCwd: targetRoot, exitCode: cmd.exitCode };
     }
     const parsed = parseSemgrepJson(cmd.stdout);
-    if (parsed.parseError) {
-        return {
-            id: "semgrep-scan",
-            name: "Semgrep static analysis",
-            category: "static-scan",
-            status: "warning",
-            severity: "minor",
-            startedAt,
-            finishedAt,
-            durationMs: cmd.durationMs,
-            findings: [
-                {
-                    id: "semgrep-parse-error",
-                    title: "Could not parse Semgrep output",
-                    severity: "minor",
-                    category: "static-scan",
-                    description: `JSON parse error: ${parsed.parseError}`,
-                    recommendation: "Verify semgrep version produces valid JSON output.",
-                    releaseImpact: "Review before release",
-                },
-            ],
-            command: [command, ...scanArgs].join(" "),
-        };
-    }
     const securityFindings = parsed.findings.map((f) => ({
         id: `semgrep-${f.ruleId.replace(/[^a-z0-9-]/gi, "-")}`,
         title: f.ruleId,
@@ -160,6 +145,23 @@ export async function runSemgrepCheck(options) {
             ? "Should fix before release"
             : "Review before release",
     }));
+    const executionFailed = cmd.timedOut || cmd.skipped || (cmd.exitCode !== 0 && cmd.exitCode !== 1) ||
+        (cmd.exitCode === 1 && parsed.findings.length === 0);
+    const evidenceError = executionFailed
+        ? `Semgrep execution failed: exitCode=${String(cmd.exitCode)}, timedOut=${cmd.timedOut}, skipped=${cmd.skipped}. ${cmd.stderr.slice(0, 500)}`
+        : parsed.parseError ?? parsed.analysisError;
+    if (evidenceError) {
+        securityFindings.push({
+            id: executionFailed ? "semgrep-execution-error" : parsed.parseError ? "semgrep-parse-error" : "semgrep-analysis-error",
+            title: "Semgrep scan evidence is incomplete or unsuccessful",
+            severity: "major",
+            category: "static-scan",
+            description: evidenceError,
+            evidence: `exitCode=${String(cmd.exitCode)}; ${parsed.parseError ?? parsed.analysisError ?? ""}`,
+            recommendation: "Resolve the scanner execution or analysis error and rerun with valid JSON scan evidence.",
+            releaseImpact: "Review before release",
+        });
+    }
     const hasBlockerOrMajor = securityFindings.some((f) => f.severity === "blocker" || f.severity === "major");
     return {
         id: "semgrep-scan",
@@ -176,5 +178,7 @@ export async function runSemgrepCheck(options) {
         durationMs: cmd.durationMs,
         findings: securityFindings,
         command: [command, ...scanArgs].join(" "),
+        commandCwd: targetRoot,
+        exitCode: cmd.exitCode,
     };
 }

package/dist/src/securityValidation/types.js

@@ -1,7 +1,3 @@
-// Core types for the my-dev-kit-lab security-validation framework.
-// These types describe the vocabulary for checks, findings, results, and verdicts
-// used by dependency checks (Prompt 3), CLI adversarial tests (Prompts 4–5),
-// static scans (Prompt 6), fuzz smoke tests (Prompt 7), and the release report (Prompt 8).
 export const SECURITY_SEVERITIES = [
     "blocker",
     "major",

package/dist/src/securityValidation/validate/runSecurityValidation.js

@@ -80,7 +80,7 @@ export async function runSecurityValidation(options) {
     // missing package.json gracefully.
     if (selectedChecks.has("package")) {
         try {
-            const pkgOutput = await runPackageChecks({ cwd: target.targetRoot, config });
+            const pkgOutput = await runPackageChecks({ cwd: target.targetRoot, config, target });
             collectMany(pkgOutput.checks);
         }
         catch (err) {

package/docs/METRICS.md

@@ -938,3 +938,12 @@ Cumulative versus final: durations and tokens accumulate over all provider attem
 - Paired differences report the mean of the `raw-full-file` values, the mean of the `context-pack` values, and the mean of `context-pack` minus `raw-full-file` over matched cases.
 - These are **descriptive**. No weighted composite score, ranking, automatic winner, causal claim, or statistical-significance test is produced (`statisticalEffect` is `not-assessed`). The corpus is finite and controlled, so results do not generalize beyond it, and a single run carries no variance estimate.
 - Provider telemetry can be unavailable, and in deterministic-fixture mode both treatments receive the same reference patch, so those results measure the pipeline only.
+
+
+## Campaign progress evidence (v0.9.1 released)
+
+Token evidence levels are `provider-reported`, `cli-reported`, `parsed-from-output`, `unavailable`, and `estimated`. The original telemetry source is retained. Numeric availability is recorded per token field, so partial measurements remain partial and normalization never fills missing fields or invents totals; unattributed OpenTelemetry evidence is unavailable. Duration provenance is `lab-wall-clock` for valid observed elapsed time and `unavailable` for missing or invalid values; campaign comparison time is Lab-measured wall-clock duration, not provider-native time.
+
+The plugin-specific progress projection is `my-dev-kit-lab-agent-success-rate-campaign-progress-v1`; the writer publishes immutable revision-specific JSON, HTML, and text snapshots. It accounts for selected cases and treatment units, while execution epochs remain recovery history and are excluded from selected-unit counts. Paused provider-limit and uncertain states remain distinct. Partial progress is not a complete scientific analysis artifact.
+
+Evidence quality is metric-specific: task outcomes, token measurements, and duration can have different strong, partial, or unavailable classifications. Strong describes the completeness and provenance of the available evidence; it is not statistical significance. Partial evidence does not mean task failure. Missing token evidence is unavailable, never zero; estimated tokens are not provider usage. Duration is Lab-measured wall-clock time, not provider-native time. Comparisons use matched completed cases and state their denominator and exclusions; the existing paired direction is preserved. These descriptive comparisons establish no causal effect, winner, p-value, or general provider performance.

package/examples/tutorial-browser/prepare.mjs

@@ -1,14 +1,14 @@
-import fs from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-const sourceRoot = path.dirname(fileURLToPath(import.meta.url));
-const targetRoot = process.argv[2];
-if (!targetRoot) {
-  console.error("prepare requires a target root argument");
-  process.exit(2);
-}
-fs.mkdirSync(path.join(targetRoot, "app"), { recursive: true });
-fs.mkdirSync(path.join(targetRoot, "out"), { recursive: true });
-for (const file of ["index.html", "server.mjs"]) fs.copyFileSync(path.join(sourceRoot, file), path.join(targetRoot, "app", file));
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const sourceRoot = path.dirname(fileURLToPath(import.meta.url));
+const targetRoot = process.argv[2];
+if (!targetRoot) {
+  console.error("prepare requires a target root argument");
+  process.exit(2);
+}
+fs.mkdirSync(path.join(targetRoot, "app"), { recursive: true });
+fs.mkdirSync(path.join(targetRoot, "out"), { recursive: true });
+for (const file of ["index.html", "server.mjs"]) fs.copyFileSync(path.join(sourceRoot, file), path.join(targetRoot, "app", file));
 fs.writeFileSync(path.join(targetRoot, "out", "report.json"), JSON.stringify({ prepared: true, counts: { widgets: 3 } }, null, 2), "utf8");

package/examples/tutorial-browser/server.mjs

@@ -1,14 +1,14 @@
-import http from "node:http";
-import fs from "node:fs";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-
-const root = path.dirname(fileURLToPath(import.meta.url));
-const port = Number(process.env.TUTORIAL_FIXTURE_PORT || 0);
-const server = http.createServer((req, res) => {
-  const url = new URL(req.url ?? "/", "http://127.0.0.1");
-  if (url.pathname === "/api/status") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true, widgets: 3 })); return; }
-  const filePath = url.pathname === "/" ? path.join(root, "index.html") : path.join(root, url.pathname);
-  fs.readFile(filePath, (error, contents) => { if (error) { res.writeHead(404); res.end("not found"); return; } res.writeHead(200, { "content-type": filePath.endsWith(".html") ? "text/html" : "application/octet-stream" }); res.end(contents); });
-});
+import http from "node:http";
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = path.dirname(fileURLToPath(import.meta.url));
+const port = Number(process.env.TUTORIAL_FIXTURE_PORT || 0);
+const server = http.createServer((req, res) => {
+  const url = new URL(req.url ?? "/", "http://127.0.0.1");
+  if (url.pathname === "/api/status") { res.writeHead(200, { "content-type": "application/json" }); res.end(JSON.stringify({ ok: true, widgets: 3 })); return; }
+  const filePath = url.pathname === "/" ? path.join(root, "index.html") : path.join(root, url.pathname);
+  fs.readFile(filePath, (error, contents) => { if (error) { res.writeHead(404); res.end("not found"); return; } res.writeHead(200, { "content-type": filePath.endsWith(".html") ? "text/html" : "application/octet-stream" }); res.end(contents); });
+});
 server.listen(port, "127.0.0.1", () => console.log("tutorial fixture listening on " + server.address().port));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {