@dailephd/my-dev-kit-lab 0.2.0 → 0.2.1

package/README.md

@@ -2,7 +2,7 @@
 
 my-dev-kit-lab is the experiment, evidence, reporting, and demo companion for [my-dev-kit](https://github.com/your-org/my-dev-kit). It runs reproducible experiments that test whether my-dev-kit's graph-guided retrieval helps coding-agent workflows, collects metrics, renders reports, generates plots, captures screenshots, and builds gallery outputs.
 
-As of v0.2.0, my-dev-kit-lab also exposes a generic experiment-plugin framework for validating local developer tools, security checks, codebase workflows, retrieval strategies, and experiment outcomes. The first plugin is `context-strategy-comparison`, which preserves the existing raw-full-file vs my-dev-kit-guided workflow through the plugin runner.
+As of v0.2.1, my-dev-kit-lab also exposes a generic experiment-plugin framework for validating local developer tools, security checks, codebase workflows, retrieval strategies, and experiment outcomes. The first plugin is `context-strategy-comparison`, which preserves the existing raw-full-file vs my-dev-kit-guided workflow through the plugin runner.
 
 **my-dev-kit** is the repo indexing and graph-guided retrieval engine.
 **my-dev-kit-lab** is the separate lab layer that feeds it benchmark inputs and records evaluation outputs.
@@ -205,7 +205,7 @@ See [docs/METRICS.md](docs/METRICS.md) for full metric definitions.
 
 ## Current baseline release positioning
 
-my-dev-kit-lab is at a working baseline. The raw-vs-indexed experiment pipeline is fully implemented and produces reproducible artifacts. Real-agent campaign support exists for Codex and Claude. v0.2.0 adds the generic experiment-plugin framework, keeps `context-strategy-comparison` as the first plugin, and keeps future experiment types on the roadmap.
+my-dev-kit-lab is at a working baseline. The raw-vs-indexed experiment pipeline is fully implemented and produces reproducible artifacts. Real-agent campaign support exists for Codex and Claude. v0.2.1 keeps the generic experiment-plugin framework, preserves `context-strategy-comparison` as the first plugin, and fixes target-aware security validation so installed-package checks run `npm run test:security` in the external target project root.
 
 ---
 
@@ -239,7 +239,9 @@ The release gate is implemented as of v0.1.4. It combines static scans, dependen
 
 CodeQL, Semgrep, and OSV-Scanner are optional. When unavailable locally, they are recorded as `skipped` in the report — not as failures — and the verdict is `ready except optional manual checks` rather than `not ready`.
 
-Each security command can validate my-dev-kit-lab itself or another local project via `--target <path>`. When `--target` is omitted, the framework performs self-validation. Target projects are inspected in place: their source files are not modified, generated artifacts stay under `reports/security/`, and external-target reports identify both the tool root and the target root.
+Each security command can validate my-dev-kit-lab itself or another local project via `--target <path>`. When `--target` is omitted, the framework performs self-validation. Target projects are inspected in place: their source files are not modified, generated artifacts stay under `reports/security/`, and external-target reports identify the tool root, target root, target package metadata, target git metadata, command cwd, exit code, and stdout/stderr summaries.
+
+For external-target validation, `security:validate` reads the target `package.json`, detects `scripts.test:security`, and runs `npm run test:security` in the target project root when that script exists. This behavior is validated both from the source checkout and from an installed packed tarball because published-package execution differs from local source execution.
 
 Generated security reports under `reports/security/` are excluded from git by default. They are produced locally or in CI as release-gate evidence and are not committed to the repository.
 

package/dist/scripts/security/resolveToolRoot.js

@@ -0,0 +1,16 @@
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+export function resolveToolRoot(moduleUrl) {
+    let current = path.dirname(fileURLToPath(moduleUrl));
+    while (true) {
+        if (fs.existsSync(path.join(current, "package.json"))) {
+            return current;
+        }
+        const parent = path.dirname(current);
+        if (parent === current) {
+            return process.cwd();
+        }
+        current = parent;
+    }
+}

package/dist/scripts/security/runCodeql.js

@@ -1,9 +1,10 @@
 #!/usr/bin/env node
 import { runCodeqlCheck } from "../../src/securityValidation/staticScans/codeql.js";
 import { resolveValidationTarget } from "../../src/securityValidation/validate/resolveTarget.js";
+import { resolveToolRoot } from "./resolveToolRoot.js";
 const rawArgs = process.argv.slice(2);
 const args = parseArgs(rawArgs);
-const toolRoot = process.cwd();
+const toolRoot = resolveToolRoot(import.meta.url);
 let targetRoot;
 try {
     const target = resolveValidationTarget(args.target, toolRoot);

package/dist/scripts/security/runDependencyChecks.js

@@ -3,9 +3,10 @@ import path from "node:path";
 import { runDependencyChecks } from "../../src/securityValidation/index.js";
 import { DEFAULT_SECURITY_CONFIG } from "../../src/securityValidation/index.js";
 import { resolveValidationTarget } from "../../src/securityValidation/validate/resolveTarget.js";
+import { resolveToolRoot } from "./resolveToolRoot.js";
 const rawArgs = process.argv.slice(2);
 const args = parseArgs(rawArgs);
-const toolRoot = process.cwd();
+const toolRoot = resolveToolRoot(import.meta.url);
 let targetRoot;
 try {
     const target = resolveValidationTarget(args.target, toolRoot);

package/dist/scripts/security/runPackageChecks.js

@@ -3,9 +3,10 @@ import path from "node:path";
 import { runPackageChecks } from "../../src/securityValidation/index.js";
 import { DEFAULT_SECURITY_CONFIG } from "../../src/securityValidation/index.js";
 import { resolveValidationTarget } from "../../src/securityValidation/validate/resolveTarget.js";
+import { resolveToolRoot } from "./resolveToolRoot.js";
 const rawArgs = process.argv.slice(2);
 const args = parseArgs(rawArgs);
-const toolRoot = process.cwd();
+const toolRoot = resolveToolRoot(import.meta.url);
 let targetRoot;
 try {
     const target = resolveValidationTarget(args.target, toolRoot);

package/dist/scripts/security/runSemgrep.js

@@ -2,9 +2,10 @@
 import path from "node:path";
 import { runSemgrepCheck } from "../../src/securityValidation/staticScans/semgrep.js";
 import { resolveValidationTarget } from "../../src/securityValidation/validate/resolveTarget.js";
+import { resolveToolRoot } from "./resolveToolRoot.js";
 const rawArgs = process.argv.slice(2);
 const args = parseArgs(rawArgs);
-const toolRoot = process.cwd();
+const toolRoot = resolveToolRoot(import.meta.url);
 let targetRoot;
 try {
     const target = resolveValidationTarget(args.target, toolRoot);

package/dist/scripts/security/validate.js

@@ -4,10 +4,11 @@ import fs from "node:fs";
 import { runSecurityValidation } from "../../src/securityValidation/validate/runSecurityValidation.js";
 import { resolveValidationTarget, reportFilenamePrefix } from "../../src/securityValidation/validate/resolveTarget.js";
 import { renderTextReport, renderJsonReport } from "../../src/securityValidation/report/renderSecurityReport.js";
+import { resolveToolRoot } from "./resolveToolRoot.js";
 // Parse CLI arguments from process.argv (after the node/tsx and script path).
 const rawArgs = process.argv.slice(2);
 const args = parseArgs(rawArgs);
-const toolRoot = process.cwd();
+const toolRoot = resolveToolRoot(import.meta.url);
 // Resolve and validate target early so we can fail fast with a clean error.
 let target;
 try {

package/dist/src/core/localProjectTarget.js

@@ -27,6 +27,7 @@ export function resolveLocalProjectTarget(targetPathArg, toolRoot) {
         packageName: packageMetadata.packageName,
         packageVersion: packageMetadata.packageVersion,
         hasPackageJson: packageMetadata.hasPackageJson,
+        hasSecurityTestScript: packageMetadata.hasSecurityTestScript,
         hasLockfile: hasLockfile(resolvedTargetRoot),
         ...readGitMetadata(resolvedTargetRoot),
         isSelf: resolvedTargetRoot === resolvedToolRoot,
@@ -36,10 +37,12 @@ function readPackageMetadata(targetRoot) {
     try {
         const pkgRaw = fs.readFileSync(path.join(targetRoot, "package.json"), "utf8");
         const pkg = JSON.parse(pkgRaw);
+        const testSecurityScript = pkg.scripts && typeof pkg.scripts["test:security"] === "string";
         return {
             hasPackageJson: true,
             packageName: typeof pkg.name === "string" ? pkg.name : null,
             packageVersion: typeof pkg.version === "string" ? pkg.version : null,
+            hasSecurityTestScript: Boolean(testSecurityScript),
         };
     }
     catch {
@@ -47,6 +50,7 @@ function readPackageMetadata(targetRoot) {
             hasPackageJson: false,
             packageName: null,
             packageVersion: null,
+            hasSecurityTestScript: false,
         };
     }
 }

package/dist/src/core/runMeasuredCommand.js

@@ -70,9 +70,9 @@ export async function runMeasuredCommand(options) {
                 resolvedCommand: resolution
             };
             void Promise.all([
-                writeFile(stdoutPath, stdout, "utf8"),
-                writeFile(stderrPath, stderr, "utf8"),
-                writeFile(telemetryPath, JSON.stringify(measured, null, 2), "utf8")
+                writeArtifact(stdoutPath, stdout),
+                writeArtifact(stderrPath, stderr),
+                writeArtifact(telemetryPath, JSON.stringify(measured, null, 2))
             ]).then(() => resolve(measured));
             return;
         }
@@ -98,8 +98,8 @@ export async function runMeasuredCommand(options) {
             }
             const endedAt = new Date().toISOString();
             const durationMs = Date.now() - started;
-            await writeFile(stdoutPath, stdout, "utf8");
-            await writeFile(stderrPath, stderr, "utf8");
+            await writeArtifact(stdoutPath, stdout);
+            await writeArtifact(stderrPath, stderr);
             const measured = {
                 commandId: options.commandId,
                 commandString: options.commandString,
@@ -119,12 +119,16 @@ export async function runMeasuredCommand(options) {
                 error: spawnError,
                 resolvedCommand: resolution
             };
-            await writeFile(telemetryPath, JSON.stringify(measured, null, 2), "utf8");
+            await writeArtifact(telemetryPath, JSON.stringify(measured, null, 2));
             resolve(measured);
         });
     });
     return result;
 }
+async function writeArtifact(filePath, value) {
+    await mkdir(path.dirname(filePath), { recursive: true });
+    await writeFile(filePath, value, "utf8");
+}
 function killProcessTree(pid) {
     if (!pid) {
         return;

package/dist/src/securityValidation/report/renderSecurityReport.js

@@ -72,7 +72,7 @@ export function renderTextReport(report) {
         { num: 7, title: "npm audit Result", ids: ["npm-audit-full", "npm-audit-runtime"] },
         { num: 8, title: "OSV-Scanner Result", ids: ["osv-scanner"] },
         { num: 9, title: "Package Tarball Inspection", ids: ["npm-pack-dry-run"] },
-        { num: 10, title: "File-System / Path Traversal Tests", ids: ["path-traversal-root", "path-traversal-out", "path-traversal-index", "absolute-path-escape", "path-harness-escape-detection"] },
+        { num: 10, title: "CLI / File-System / Path Traversal Tests", ids: ["cli-adversarial-suite", "path-traversal-root", "path-traversal-out", "path-traversal-index", "absolute-path-escape", "path-harness-escape-detection"] },
         { num: 11, title: "Source Read-Only Boundary Tests", ids: ["source-files-not-modified", "writes-limited-to-output", "index-write-containment", "artifact-cleanup-safe"] },
         { num: 12, title: "Graphviz / Subprocess Safety Tests", ids: ["graphviz-label-escaping", "subprocess-no-shell-interpolation"] },
         { num: 13, title: "JSON stdout/stderr Safety Tests", ids: ["json-mode-parseable-output", "stderr-not-in-stdout", "json-failure-error-object", "progress-not-in-json-stdout"] },
@@ -144,6 +144,21 @@ export function renderTextReport(report) {
         }
         for (const check of sectionChecks) {
             lines.push(`[${statusIcon(check.status)}] ${pad(check.name)} ${formatDuration(check.durationMs)}`);
+            if (check.command) {
+                lines.push(`       Command: ${check.command}`);
+            }
+            if (check.commandCwd) {
+                lines.push(`       Cwd: ${check.commandCwd}`);
+            }
+            if (check.exitCode !== undefined) {
+                lines.push(`       Exit code: ${String(check.exitCode)}`);
+            }
+            if (check.stdoutSummary) {
+                lines.push(`       Stdout: ${check.stdoutSummary.slice(0, 200)}`);
+            }
+            if (check.stderrSummary) {
+                lines.push(`       Stderr: ${check.stderrSummary.slice(0, 200)}`);
+            }
             if (check.status === "skipped" && check.skippedReason) {
                 lines.push(`       Reason: ${check.skippedReason}`);
             }
@@ -212,6 +227,10 @@ export function renderJsonReport(report) {
         findingCount: c.findings.length,
         skippedReason: c.skippedReason ?? null,
         command: c.command ?? null,
+        commandCwd: c.commandCwd ?? null,
+        exitCode: c.exitCode ?? null,
+        stdoutSummary: c.stdoutSummary ?? null,
+        stderrSummary: c.stderrSummary ?? null,
     }));
     const sanitizedFindings = allFindings.map((f) => ({
         id: f.id,

package/dist/src/securityValidation/validate/runCliSecuritySuiteCheck.js

@@ -0,0 +1,124 @@
+import { runSecurityCommand, resolveNpmCommand } from "../commandRunner.js";
+export async function runCliSecuritySuiteCheck(options) {
+    const startedAt = new Date().toISOString();
+    if (!options.target.isSelf && !options.target.hasPackageJson) {
+        const finishedAt = new Date().toISOString();
+        return {
+            id: "cli-adversarial-suite",
+            name: "Target security test suite",
+            category: "cli-adversarial",
+            status: "failed",
+            severity: "major",
+            startedAt,
+            finishedAt,
+            durationMs: 0,
+            command: "npm run test:security",
+            commandCwd: options.target.targetRoot,
+            exitCode: null,
+            findings: [
+                {
+                    id: "target-security-suite-missing-package-json",
+                    title: "Target package.json is missing",
+                    severity: "major",
+                    category: "cli-adversarial",
+                    description: "Target-aware security suite validation requires a package.json at the target root.",
+                    recommendation: "Add a package.json with a test:security script or validate a package-based target.",
+                    releaseImpact: "Should fix before release",
+                },
+            ],
+        };
+    }
+    if (!options.target.isSelf && !options.target.hasSecurityTestScript) {
+        const finishedAt = new Date().toISOString();
+        return {
+            id: "cli-adversarial-suite",
+            name: "Target security test suite",
+            category: "cli-adversarial",
+            status: "failed",
+            severity: "major",
+            startedAt,
+            finishedAt,
+            durationMs: 0,
+            command: "npm run test:security",
+            commandCwd: options.target.targetRoot,
+            exitCode: null,
+            findings: [
+                {
+                    id: "target-security-suite-missing-script",
+                    title: "Target test:security script is missing",
+                    severity: "major",
+                    category: "cli-adversarial",
+                    description: "The target package.json does not define scripts.test:security, so the target security suite could not be validated.",
+                    recommendation: "Add a target test:security script and rerun security:validate.",
+                    releaseImpact: "Should fix before release",
+                },
+            ],
+        };
+    }
+    const npm = resolveNpmCommand();
+    const isSelf = options.target.isSelf;
+    const commandCwd = isSelf ? options.toolRoot : options.target.targetRoot;
+    const cmd = await runSecurityCommand({
+        command: npm,
+        args: ["run", "test:security"],
+        cwd: commandCwd,
+        timeoutMs: Math.min(options.timeoutMs * 3, 180_000),
+    });
+    const finishedAt = new Date().toISOString();
+    const passed = cmd.exitCode === 0;
+    const name = isSelf
+        ? "CLI adversarial test suite (tool self-tests)"
+        : "Target security test suite";
+    return {
+        id: "cli-adversarial-suite",
+        name,
+        category: "cli-adversarial",
+        status: passed ? "passed" : "failed",
+        severity: "major",
+        startedAt,
+        finishedAt,
+        durationMs: cmd.durationMs,
+        findings: passed
+            ? []
+            : [
+                {
+                    id: "cli-adversarial-suite-failed",
+                    title: isSelf ? "CLI adversarial test suite had failures" : "Target test:security script failed",
+                    severity: "major",
+                    category: "cli-adversarial",
+                    description: isSelf
+                        ? `Tool-root test:security exited with code ${String(cmd.exitCode)}.`
+                        : `Target-root test:security exited with code ${String(cmd.exitCode)}.`,
+                    evidence: buildEvidence(cmd),
+                    recommendation: isSelf
+                        ? "Fix failing tool security tests before release."
+                        : "Fix the target test:security failure and rerun security:validate.",
+                    releaseImpact: "Should fix before release",
+                },
+            ],
+        command: `${npm} run test:security`,
+        commandCwd,
+        exitCode: cmd.exitCode,
+        stdoutSummary: summarizeOutput(cmd.stdout),
+        stderrSummary: summarizeOutput(cmd.stderr),
+    };
+}
+function summarizeOutput(value) {
+    const normalized = value.replace(/\r\n/g, "\n").trim();
+    if (!normalized) {
+        return undefined;
+    }
+    return normalized.slice(0, 500);
+}
+function buildEvidence(cmd) {
+    const parts = [`cwd=${cmd.cwd}`, `exitCode=${String(cmd.exitCode)}`];
+    const stdoutSummary = summarizeOutput(cmd.stdout);
+    const stderrSummary = summarizeOutput(cmd.stderr);
+    if (stdoutSummary) {
+        parts.push(`stdout=${stdoutSummary}`);
+    }
+    if (stderrSummary) {
+        parts.push(`stderr=${stderrSummary}`);
+    }
+    return parts.join(" | ");
+}

package/dist/src/securityValidation/validate/runSecurityValidation.js

@@ -6,10 +6,10 @@ import { runCodeqlCheck } from "../staticScans/codeql.js";
 import { runSemgrepCheck } from "../staticScans/semgrep.js";
 import { runAllFuzzTargets } from "../fuzz/fuzzHarness.js";
 import { ALL_FUZZ_TARGETS } from "../fuzz/fuzzTargets.js";
-import { runSecurityCommand, resolveNpmCommand } from "../commandRunner.js";
 import { calculateVerdict } from "./verdict.js";
 import { resolveValidationTarget, targetDescription as describeTarget } from "./resolveTarget.js";
 import { DEFAULT_SECURITY_CONFIG } from "../config.js";
+import { runCliSecuritySuiteCheck } from "./runCliSecuritySuiteCheck.js";
 export async function runSecurityValidation(options) {
     const toolRoot = options.cwd;
     const config = {
@@ -68,45 +68,14 @@ export async function runSecurityValidation(options) {
     catch (err) {
         collect(errorCheck("semgrep-scan", "Semgrep static analysis", "static-scan", err));
     }
-    // --- CLI adversarial tests (always tool-internal; tests my-dev-kit-lab harness) ---
-    // These run in the tool root because they use fake-adversarial-cli.js fixtures.
-    // Labeled clearly so the report shows they are not target-specific checks.
-    {
-        const startedAt2 = new Date().toISOString();
-        const npm = resolveNpmCommand();
-        const cmd = await runSecurityCommand({
-            command: npm,
-            args: ["run", "test:security", "--", "--reporter=json"],
-            cwd: toolRoot,
-            timeoutMs: Math.min(config.commandTimeoutMs * 3, 180_000),
-        });
-        const finishedAt2 = new Date().toISOString();
-        const passed = cmd.exitCode === 0;
-        collect({
-            id: "cli-adversarial-suite",
-            name: "CLI adversarial test suite (tool self-tests, not target-specific)",
-            category: "cli-adversarial",
-            status: passed ? "passed" : "failed",
-            severity: "major",
-            startedAt: startedAt2,
-            finishedAt: finishedAt2,
-            durationMs: cmd.durationMs,
-            findings: passed
-                ? []
-                : [
-                    {
-                        id: "cli-adversarial-suite-failed",
-                        title: "CLI adversarial test suite had failures",
-                        severity: "major",
-                        category: "cli-adversarial",
-                        description: `test:security exited with code ${String(cmd.exitCode)}. Run 'npm run test:security' for details.`,
-                        recommendation: "Fix failing adversarial tests before release.",
-                        releaseImpact: "Should fix before release",
-                    },
-                ],
-            command: `${npm} run test:security`,
-        });
-    }
+    // --- Security suite check ---
+    // Self-validation runs the lab's own test:security suite.
+    // External validation runs the target project's test:security script when present.
+    collect(await runCliSecuritySuiteCheck({
+        toolRoot,
+        target,
+        timeoutMs: config.commandTimeoutMs,
+    }));
     // --- Fuzz smoke (always tool-internal; tests library parser/helper code) ---
     try {
         collect(await runAllFuzzTargets(ALL_FUZZ_TARGETS, {

package/package.json

@@ -1,6 +1,6 @@
 {
-  "name": "@dailephd/my-dev-kit-lab",
-  "version": "0.2.0",
+  "name": "@dailephd/my-dev-kit-lab",
+  "version": "0.2.1",
   "type": "module",
   "description": "Evidence, benchmark, and evaluation companion for my-dev-kit.",
   "bin": {
@@ -19,13 +19,13 @@
     "url": "git+https://github.com/dailephd/my-dev-kit-lab.git"
   },
   "homepage": "https://github.com/dailephd/my-dev-kit-lab#readme",
-  "bugs": {
-    "url": "https://github.com/dailephd/my-dev-kit-lab/issues"
-  },
-  "publishConfig": {
-    "access": "public"
-  },
-  "funding": [
+  "bugs": {
+    "url": "https://github.com/dailephd/my-dev-kit-lab/issues"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "funding": [
     "https://github.com/sponsors/dailephd",
     "https://paypal.me/daile88"
   ],
@@ -41,6 +41,7 @@
   "license": "UNLICENSED",
   "scripts": {
     "build": "node scripts/clean-dist.mjs && tsc -p tsconfig.json",
+    "typecheck": "tsc --noEmit",
     "test": "vitest run",
     "test:benchmarks": "vitest run tests/benchmarks tests/scripts",
     "test:report": "vitest run tests/report tests/commands",
@@ -78,14 +79,17 @@
     "test:fuzz:smoke": "tsx scripts/security/runFuzzSmoke.ts",
     "security:validate": "tsx scripts/security/validate.ts"
   },
-  "devDependencies": {
-    "@types/node": "^24.0.0",
-    "playwright": "^1.54.1",
-    "tsx": "^4.20.3",
-    "typescript": "^5.8.3",
-    "vitest": "^3.2.4"
-  },
-  "engines": {
-    "node": ">=20"
-  }
-}
+  "devDependencies": {
+    "@types/node": "^24.0.0",
+    "playwright": "^1.54.1",
+    "tsx": "^4.20.3",
+    "typescript": "^5.8.3",
+    "vitest": "^3.2.4"
+  },
+  "overrides": {
+    "vite": "7.3.6"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}