@dailephd/my-dev-kit-lab 0.2.1 → 0.2.2

package/dist/src/securityValidation/validate/runSecurityValidation.js

@@ -6,10 +6,30 @@ import { runCodeqlCheck } from "../staticScans/codeql.js";
 import { runSemgrepCheck } from "../staticScans/semgrep.js";
 import { runAllFuzzTargets } from "../fuzz/fuzzHarness.js";
 import { ALL_FUZZ_TARGETS } from "../fuzz/fuzzTargets.js";
-import { calculateVerdict } from "./verdict.js";
+import { calculateVerdict, summarizeVerdictReasoning } from "./verdict.js";
 import { resolveValidationTarget, targetDescription as describeTarget } from "./resolveTarget.js";
 import { DEFAULT_SECURITY_CONFIG } from "../config.js";
 import { runCliSecuritySuiteCheck } from "./runCliSecuritySuiteCheck.js";
+import { DEFAULT_SECURITY_CHECKS, DEFAULT_SECURITY_PROFILE, IMPLEMENTED_SECURITY_CHECK_IDS, PLANNED_SECURITY_CHECK_IDS, } from "./cliOptions.js";
+import { runAttackScenarios } from "../attackScenarios/attackRunner.js";
+import { toSecurityCheckResult } from "../attackScenarios/attackResult.js";
+import { captureTargetSnapshot } from "../attackScenarios/targetSnapshot.js";
+const NOT_SELECTED_REASON = "Not selected via --checks.";
+function skippedCheck(id, name, category, skippedReason) {
+    const now = new Date().toISOString();
+    return {
+        id,
+        name,
+        category,
+        status: "skipped",
+        severity: "skipped",
+        startedAt: now,
+        finishedAt: now,
+        durationMs: 0,
+        findings: [],
+        skippedReason,
+    };
+}
 export async function runSecurityValidation(options) {
     const toolRoot = options.cwd;
     const config = {
@@ -20,9 +40,15 @@ export async function runSecurityValidation(options) {
     };
     // Resolve and validate the target project
     const target = resolveValidationTarget(options.targetPath, toolRoot);
+    // v0.2.2 Batch 3: read-only git-status snapshot taken before any checks run,
+    // so the target-sandbox attack scenario (which runs last, among the
+    // attack-scenario checks) can distinguish pre-existing target dirtiness
+    // from changes caused by this validation run.
+    const targetSnapshotBefore = captureTargetSnapshot(target.targetRoot, target.hasGit);
     const startedAt = new Date().toISOString();
     const allChecks = [];
     const allFindings = [];
+    const selectedChecks = new Set(options.selectedChecks ?? DEFAULT_SECURITY_CHECKS);
     function collect(check) {
         allChecks.push(check);
         allFindings.push(...check.findings);
@@ -34,57 +60,122 @@ export async function runSecurityValidation(options) {
     // --- Dependency checks (runs in target root) ---
     // If target has no package.json/lockfile, runDependencyChecks will surface that
     // as a structured finding rather than crashing.
-    try {
-        const depOutput = await runDependencyChecks({ cwd: target.targetRoot, config });
-        collectMany(depOutput.checks);
+    if (selectedChecks.has("deps")) {
+        try {
+            const depOutput = await runDependencyChecks({ cwd: target.targetRoot, config });
+            collectMany(depOutput.checks);
+        }
+        catch (err) {
+            collect(errorCheck("dependency-checks", "Dependency checks", "dependency-audit", err));
+        }
     }
-    catch (err) {
-        collect(errorCheck("dependency-checks", "Dependency checks", "dependency-audit", err));
+    else {
+        // Preserve the mandatory-check IDs verdict.ts relies on so deselecting this
+        // group still shows up as a skipped mandatory check, not a silent gap.
+        collect(skippedCheck("npm-audit-full", "npm audit (full)", "dependency-audit", NOT_SELECTED_REASON));
+        collect(skippedCheck("npm-audit-runtime", "npm audit (runtime)", "dependency-audit", NOT_SELECTED_REASON));
     }
     // --- Package content checks (runs in target root) ---
     // Not all projects are publishable npm packages; runPackageChecks handles
     // missing package.json gracefully.
-    try {
-        const pkgOutput = await runPackageChecks({ cwd: target.targetRoot, config });
-        collectMany(pkgOutput.checks);
+    if (selectedChecks.has("package")) {
+        try {
+            const pkgOutput = await runPackageChecks({ cwd: target.targetRoot, config });
+            collectMany(pkgOutput.checks);
+        }
+        catch (err) {
+            collect(errorCheck("package-content", "Package content checks", "package-content", err));
+        }
     }
-    catch (err) {
-        collect(errorCheck("package-content", "Package content checks", "package-content", err));
+    else {
+        collect(skippedCheck("npm-pack-dry-run", "npm pack --dry-run", "package-content", NOT_SELECTED_REASON));
     }
     // --- Static scans (scan target root; config from tool root) ---
-    try {
-        collect(await runCodeqlCheck({ cwd: toolRoot, targetRoot: target.targetRoot, timeoutMs: config.commandTimeoutMs }));
+    if (selectedChecks.has("static")) {
+        try {
+            collect(await runCodeqlCheck({ cwd: toolRoot, targetRoot: target.targetRoot, timeoutMs: config.commandTimeoutMs }));
+        }
+        catch (err) {
+            collect(errorCheck("codeql-scan", "CodeQL static analysis", "static-scan", err));
+        }
+        try {
+            collect(await runSemgrepCheck({
+                targetRoot: target.targetRoot,
+                toolRoot,
+                timeoutMs: config.commandTimeoutMs,
+            }));
+        }
+        catch (err) {
+            collect(errorCheck("semgrep-scan", "Semgrep static analysis", "static-scan", err));
+        }
     }
-    catch (err) {
-        collect(errorCheck("codeql-scan", "CodeQL static analysis", "static-scan", err));
+    else {
+        collect(skippedCheck("codeql-scan", "CodeQL static analysis", "static-scan", NOT_SELECTED_REASON));
+        collect(skippedCheck("semgrep-scan", "Semgrep static analysis", "static-scan", NOT_SELECTED_REASON));
     }
-    try {
-        collect(await runSemgrepCheck({
-            targetRoot: target.targetRoot,
+    // --- Security suite check ---
+    // Self-validation runs the lab's own test:security suite.
+    // External validation runs the target project's test:security script when present.
+    if (selectedChecks.has("cli-adversarial")) {
+        collect(await runCliSecuritySuiteCheck({
             toolRoot,
+            target,
             timeoutMs: config.commandTimeoutMs,
         }));
     }
-    catch (err) {
-        collect(errorCheck("semgrep-scan", "Semgrep static analysis", "static-scan", err));
+    else {
+        collect(skippedCheck("cli-adversarial-suite", "Target security test suite", "cli-adversarial", NOT_SELECTED_REASON));
     }
-    // --- Security suite check ---
-    // Self-validation runs the lab's own test:security suite.
-    // External validation runs the target project's test:security script when present.
-    collect(await runCliSecuritySuiteCheck({
-        toolRoot,
-        target,
-        timeoutMs: config.commandTimeoutMs,
-    }));
     // --- Fuzz smoke (always tool-internal; tests library parser/helper code) ---
-    try {
-        collect(await runAllFuzzTargets(ALL_FUZZ_TARGETS, {
-            seed: options.fuzzSeed ?? 0xdeadbeef,
-            iterations: options.fuzzIterations ?? 50,
-        }));
+    if (selectedChecks.has("fuzz")) {
+        try {
+            collect(await runAllFuzzTargets(ALL_FUZZ_TARGETS, {
+                seed: options.fuzzSeed ?? 0xdeadbeef,
+                iterations: options.fuzzIterations ?? 50,
+            }));
+        }
+        catch (err) {
+            collect(errorCheck("fuzz-smoke", "Fuzz smoke", "fuzz-smoke", err));
+        }
+    }
+    else {
+        collect(skippedCheck("fuzz-smoke", "Fuzz smoke", "fuzz-smoke", NOT_SELECTED_REASON));
     }
-    catch (err) {
-        collect(errorCheck("fuzz-smoke", "Fuzz smoke", "fuzz-smoke", err));
+    // --- Attack-scenario checks (boundary, subprocess, secrets, network) ---
+    // v0.2.2 Batch 2: routed through the attack-scenario runner. The registry
+    // is empty in Batch 2, so these currently resolve to explicit "no
+    // scenarios registered yet" skipped results — never "passed".
+    const profile = options.profile ?? DEFAULT_SECURITY_PROFILE;
+    const attackScenarioChecks = PLANNED_SECURITY_CHECK_IDS.filter((id) => selectedChecks.has(id));
+    let attackResults = [];
+    if (attackScenarioChecks.length > 0) {
+        try {
+            attackResults = await runAttackScenarios({
+                selectedChecks: attackScenarioChecks,
+                profile,
+                toolRoot,
+                target,
+                config,
+                targetSnapshotBefore,
+            });
+        }
+        catch (err) {
+            attackResults = attackScenarioChecks.map((checkId) => ({
+                scenarioId: `${checkId}-attack-runner-error`,
+                scenarioTitle: `Attack scenario runner error for '${checkId}'`,
+                checkId,
+                profileId: profile,
+                status: "blocked",
+                severity: "major",
+                confidence: "low",
+                evidence: [],
+                category: "cli-adversarial",
+                errorSummary: err instanceof Error ? err.message : String(err),
+            }));
+        }
+        for (const result of attackResults) {
+            collect(toSecurityCheckResult(result));
+        }
     }
     const finishedAt = new Date().toISOString();
     // --- Tool root metadata ---
@@ -102,6 +193,11 @@ export async function runSecurityValidation(options) {
         // Use defaults.
     }
     const { verdict, recommendedNextStep } = calculateVerdict(allChecks, allFindings);
+    const verdictReasonSummary = summarizeVerdictReasoning(allChecks, allFindings);
+    // "Full release gate" = every classic pre-v0.2.2 check group was selected.
+    // Attack-scenario checks (boundary/subprocess/secrets/network) are
+    // additional coverage, not part of that traditional baseline gate.
+    const isFullReleaseGate = IMPLEMENTED_SECURITY_CHECK_IDS.every((id) => selectedChecks.has(id));
     return {
         toolRoot,
         toolPackageName,
@@ -119,6 +215,9 @@ export async function runSecurityValidation(options) {
         findings: allFindings,
         verdict,
         recommendedNextStep,
+        attackResults,
+        verdictReasonSummary,
+        isFullReleaseGate,
     };
 }
 function errorCheck(id, name, category, err) {

package/dist/src/securityValidation/validate/verdict.js

@@ -1,3 +1,4 @@
+import { VERDICT_IMPACTS } from "../types.js";
 // ---------------------------------------------------------------------------
 // Verdict calculation
 //
@@ -71,3 +72,149 @@ export function verdictToHumanLabel(verdict) {
             return "inconclusive: audit environment incomplete";
     }
 }
+// ---------------------------------------------------------------------------
+// v0.2.2 Batch 5 — fail-on severity threshold
+//
+// --fail-on uses a separate vocabulary (blocker/high/medium/low) from
+// SecuritySeverity (blocker/major/minor/informational/skipped) — Batch 1
+// deliberately kept these distinct. This maps one onto the other so a
+// --fail-on breach can be detected without renaming any existing severity.
+// Mapping (conservative — never understates a blocker-level finding):
+//   blocker fail-on -> only "blocker" severity findings breach it
+//   high    fail-on -> "blocker" or "major" ("major" is treated as "high")
+//   medium  fail-on -> "blocker", "major", or "minor" ("minor" -> "medium")
+//   low     fail-on -> "blocker", "major", "minor", or "informational"
+// "skipped" severity never breaches any threshold — a skipped optional tool
+// is not a finding.
+// ---------------------------------------------------------------------------
+const SEVERITY_THRESHOLD_RANK = {
+    blocker: 4,
+    major: 3,
+    minor: 2,
+    informational: 1,
+    skipped: 0,
+};
+const FAIL_ON_THRESHOLD_RANK = {
+    blocker: 4,
+    high: 3,
+    medium: 2,
+    low: 1,
+};
+export function severityMeetsFailOnThreshold(severity, threshold) {
+    return SEVERITY_THRESHOLD_RANK[severity] >= FAIL_ON_THRESHOLD_RANK[threshold];
+}
+// True when at least one finding's severity breaches the given --fail-on
+// threshold. Callers should OR this with the existing verdict-based exit
+// decision, never use it to replace/weaken that decision — this only adds
+// stricter exit-1 triggers for --fail-on medium/low (which the default
+// verdict policy doesn't already cover via its blocker/major rules); for
+// --fail-on blocker/high it is always a no-op given the existing policy.
+export function findingsBreachFailOnThreshold(findings, threshold) {
+    return findings.some((f) => severityMeetsFailOnThreshold(f.severity, threshold));
+}
+export const VERDICT_REASON_CATEGORIES = [
+    "release-blocker",
+    "target-project-blocker",
+    "tool-framework-blocker",
+    "environment-inconclusive",
+    "optional-skipped-tool",
+    "adversarial-scenario-skipped",
+    "scanner-finding",
+    "adversarial-scenario-failure",
+    "required-evidence-missing",
+    "informational-evidence",
+];
+// v0.2.2 Batch 6: scenario blocker-impact classification now comes directly
+// from SecurityCheckResult.verdictImpact (carried through from the
+// originating AttackScenario's static declaration — see attackScenario.ts /
+// attackResult.ts). verdict.ts no longer owns a separate scenario-id map, so
+// there is nothing to keep in sync when a new scenario is registered — as
+// long as the scenario declares verdictImpact, categorization is correct
+// automatically. Checks with no verdictImpact but a scenario-shaped id
+// (placeholder/error results, or a scenario that doesn't declare one) fall
+// back to the generic "adversarial-scenario-failure"/"-skipped" buckets.
+//
+// A check is "scenario-shaped" if it carries a verdictImpact OR its id
+// matches the synthetic placeholder/error patterns produced by
+// attackRunner.ts for unregistered checks or runner crashes (those never
+// carry verdictImpact since there's no real AttackScenario object).
+function isAttackScenarioCheckId(id) {
+    return id.endsWith("-no-scenarios-registered") || id.endsWith("-attack-runner-error");
+}
+// Categorizes a single check for verdict-reasoning purposes. Returns null
+// for passed/warning checks — those don't need a blocker-reasoning bucket.
+export function categorizeCheck(check) {
+    const scenario = isAttackScenarioCheckId(check.id) || check.verdictImpact !== undefined;
+    if (check.status === "skipped") {
+        if (scenario)
+            return "adversarial-scenario-skipped";
+        if (MANDATORY_CHECK_IDS.has(check.id))
+            return "required-evidence-missing";
+        return "optional-skipped-tool";
+    }
+    if (check.status === "failed") {
+        if (check.verdictImpact && VERDICT_IMPACTS.includes(check.verdictImpact)) {
+            return check.verdictImpact;
+        }
+        if (scenario)
+            return "adversarial-scenario-failure";
+        if (check.severity === "informational")
+            return "informational-evidence";
+        return "scanner-finding";
+    }
+    return null;
+}
+// Aggregates categorizeCheck() across all checks, plus a direct informational
+// finding count (informational findings don't get their own check-level
+// bucket since a check can carry multiple findings of mixed severity).
+export function summarizeVerdictReasoning(checks, findings) {
+    const summary = {
+        releaseBlockerCount: 0,
+        targetProjectBlockerCount: 0,
+        toolFrameworkBlockerCount: 0,
+        environmentInconclusiveCount: 0,
+        optionalSkippedToolCount: 0,
+        adversarialScenarioSkippedCount: 0,
+        scannerFindingCount: 0,
+        adversarialScenarioFailureCount: 0,
+        requiredEvidenceMissingCount: 0,
+        informationalEvidenceCount: 0,
+    };
+    for (const check of checks) {
+        const category = categorizeCheck(check);
+        switch (category) {
+            case "release-blocker":
+                summary.releaseBlockerCount++;
+                break;
+            case "target-project-blocker":
+                summary.targetProjectBlockerCount++;
+                break;
+            case "tool-framework-blocker":
+                summary.toolFrameworkBlockerCount++;
+                break;
+            case "environment-inconclusive":
+                summary.environmentInconclusiveCount++;
+                break;
+            case "optional-skipped-tool":
+                summary.optionalSkippedToolCount++;
+                break;
+            case "adversarial-scenario-skipped":
+                summary.adversarialScenarioSkippedCount++;
+                break;
+            case "scanner-finding":
+                summary.scannerFindingCount++;
+                break;
+            case "adversarial-scenario-failure":
+                summary.adversarialScenarioFailureCount++;
+                break;
+            case "required-evidence-missing":
+                summary.requiredEvidenceMissingCount++;
+                break;
+            case "informational-evidence":
+            case null:
+                break;
+        }
+    }
+    summary.informationalEvidenceCount = findings.filter((f) => f.severity === "informational").length;
+    return summary;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {
@@ -38,7 +38,7 @@
     "claude"
   ],
   "author": "Dai Le",
-  "license": "UNLICENSED",
+  "license": "MIT",
   "scripts": {
     "build": "node scripts/clean-dist.mjs && tsc -p tsconfig.json",
     "typecheck": "tsc --noEmit",
@@ -79,17 +79,17 @@
     "test:fuzz:smoke": "tsx scripts/security/runFuzzSmoke.ts",
     "security:validate": "tsx scripts/security/validate.ts"
   },
-  "devDependencies": {
-    "@types/node": "^24.0.0",
-    "playwright": "^1.54.1",
-    "tsx": "^4.20.3",
-    "typescript": "^5.8.3",
-    "vitest": "^3.2.4"
-  },
-  "overrides": {
-    "vite": "7.3.6"
-  },
-  "engines": {
-    "node": ">=20"
-  }
-}
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "playwright": "^1.54.1",
+    "tsx": "^4.20.3",
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.4"
+  },
+  "overrides": {
+    "vite": "7.3.6"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}