@dailephd/my-dev-kit-lab 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +19 -2
  2. package/README.md +10 -2
  3. package/dist/src/agents/adapters/codexAdapter.js +4 -1
  4. package/dist/src/cli/help.js +38 -2
  5. package/dist/src/cli/runLabCli.js +9 -1
  6. package/dist/src/commands/buildGalleryCommand.js +16 -3
  7. package/dist/src/commands/runComparePromptsCommand.js +109 -0
  8. package/dist/src/commands/runExperimentRunCommand.js +51 -5
  9. package/dist/src/core/processTree.js +41 -17
  10. package/dist/src/core/runMeasuredCommand.js +14 -8
  11. package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +24 -2
  12. package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +7 -2
  13. package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +29 -4
  14. package/dist/src/experiments/plugins/agentSuccessRate/campaignProgress.js +4 -2
  15. package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +1 -1
  16. package/dist/src/experiments/plugins/agentSuccessRate/config.js +47 -2
  17. package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +4 -2
  18. package/dist/src/experiments/plugins/agentSuccessRate/promptContract.js +121 -0
  19. package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +32 -2
  20. package/dist/src/experiments/plugins/agentSuccessRate/realAgentPrompt.js +105 -11
  21. package/dist/src/experiments/plugins/agentSuccessRate/strictDiffResponse.js +312 -0
  22. package/dist/src/gallery/galleryPathSafety.js +111 -0
  23. package/dist/src/gallery/readGallerySources.js +542 -0
  24. package/dist/src/gallery/writeGalleryManifest.js +57 -3
  25. package/dist/src/report/experiments/buildPluginExperimentReport.js +4 -16
  26. package/dist/src/report/experiments/htmlSafety.js +32 -0
  27. package/dist/src/report/experiments/metricEvidence.js +389 -0
  28. package/dist/src/report/experiments/metricGlossary.js +227 -0
  29. package/dist/src/report/experiments/metricGlossaryCatalog.js +199 -0
  30. package/dist/src/report/experiments/promptTemplateComparisonEvidence.js +377 -0
  31. package/dist/src/report/experiments/promptTemplateComparisonModel.js +436 -0
  32. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +158 -64
  33. package/dist/src/report/experiments/renderPluginExperimentReportText.js +22 -677
  34. package/dist/src/report/experiments/renderPromptTemplateComparison.js +192 -0
  35. package/dist/src/report/experiments/renderSpecializedTextSections.js +633 -0
  36. package/dist/src/report/experiments/reportSectionRegistry.js +79 -0
  37. package/dist/src/report/experiments/specializedReportSections.js +142 -0
  38. package/dist/src/report/experiments/writeAgentSuccessCampaignProgressReport.js +6 -2
  39. package/dist/src/report/experiments/writePluginExperimentReports.js +4 -2
  40. package/dist/src/securityValidation/attackScenarios/payloadCorpus.js +0 -0
  41. package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +146 -133
  42. package/dist/src/securityValidation/attackScenarios/scenarios/subprocessInjectionScenario.js +54 -36
  43. package/dist/src/securityValidation/commandRunner.js +13 -6
  44. package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +34 -4
  45. package/dist/src/securityValidation/packageChecks/requiredPackageContents.js +106 -2
  46. package/dist/src/securityValidation/packageChecks/runPackageChecks.js +31 -16
  47. package/dist/src/securityValidation/staticScans/semgrep.js +68 -64
  48. package/dist/src/securityValidation/types.js +0 -4
  49. package/dist/src/securityValidation/validate/runSecurityValidation.js +1 -1
  50. package/docs/METRICS.md +12 -2
  51. package/examples/tutorial-browser/prepare.mjs +13 -13
  52. package/examples/tutorial-browser/server.mjs +13 -13
  53. package/package.json +1 -1
@@ -1,9 +1,6 @@
1
- import fs from "node:fs";
2
- import path from "node:path";
3
- import { fileURLToPath } from "node:url";
4
1
  import { makeEvidence } from "../exploitEvidence.js";
5
2
  import { getPayloadsForGroup } from "../payloadCorpus.js";
6
- import { runSecurityCommand } from "../../commandRunner.js";
3
+ import { buildSecurityCommandSpawnOptions, runSecurityCommand } from "../../commandRunner.js";
7
4
  // ---------------------------------------------------------------------------
8
5
  // v0.2.2 Batch 3 — subprocess injection scenario.
9
6
  //
@@ -15,31 +12,60 @@ import { runSecurityCommand } from "../../commandRunner.js";
15
12
  // that no shell re-interpreted it (splitting it into multiple commands via
16
13
  // `;`, `&&`, backticks, `$()`, or `|`).
17
14
  //
18
- // Also does a bounded static check that the command-execution helpers this
19
- // scenario exercises are built with `shell: false` (source inspection of a
20
- // known-small file, not a general code scanner).
15
+ // Also verifies at runtime, through the compiled command runner's own
16
+ // spawn-options builder (the exact value runSecurityCommand() spawns with),
17
+ // that the runner does not enable a shell. No TypeScript source is read, so
18
+ // the proof works from an installed package.
21
19
  // ---------------------------------------------------------------------------
22
20
  const ECHO_ARGV_SCRIPT = "process.stdout.write(JSON.stringify(process.argv.slice(1)))";
23
- function commandRunnerSourcePath() {
24
- const here = path.dirname(fileURLToPath(import.meta.url));
25
- return path.resolve(here, "..", "..", "commandRunner.ts");
21
+ const DEFAULT_DEPS = {
22
+ runCommand: runSecurityCommand,
23
+ readSpawnOptions: (cwd) => buildSecurityCommandSpawnOptions({ cwd, env: process.env }),
24
+ };
25
+ // Runtime proof that the applicable runner's spawn path does not use a shell.
26
+ // A strict `shell === false` is the only accepted safe value; a truthy shell
27
+ // is demonstrated unsafe; a missing/malformed/throwing proof is unavailable.
28
+ function proveShellDisabled(deps, cwd) {
29
+ let options;
30
+ try {
31
+ options = deps.readSpawnOptions(cwd);
32
+ }
33
+ catch (error) {
34
+ return {
35
+ state: "unavailable",
36
+ detail: `Spawn options could not be obtained from the command runner: ${error instanceof Error ? error.message : String(error)}`,
37
+ };
38
+ }
39
+ if (typeof options !== "object" || options === null || !("shell" in options)) {
40
+ return { state: "unavailable", detail: "Command runner spawn options did not declare a shell setting." };
41
+ }
42
+ const shell = options.shell;
43
+ if (shell === false)
44
+ return { state: "safe" };
45
+ if (shell === true || typeof shell === "string") {
46
+ return { state: "unsafe", detail: "Command runner spawn options enable a shell." };
47
+ }
48
+ return { state: "unavailable", detail: "Command runner spawn options declared a malformed shell setting." };
26
49
  }
27
50
  export const SUBPROCESS_INJECTION_SCENARIO = {
28
51
  id: "subprocess-injection-safety",
29
52
  title: "Subprocess injection: shell metacharacter payloads are passed as literal arguments",
30
- description: "Runs a benign argv-echoing Node script with each subprocess-injection payload as a single spawn argument and confirms it round-trips as exactly one literal string, proving no shell re-interpretation. Also checks that commandRunner.ts constructs subprocess calls with shell:false.",
53
+ description: "Runs a benign argv-echoing Node script with each subprocess-injection payload as a single spawn argument and confirms it round-trips as exactly one literal string, proving no shell re-interpretation. Also verifies at runtime that the command runner constructs subprocess calls with shell:false.",
31
54
  checkId: "subprocess",
32
55
  applicableProfiles: [],
33
56
  severityBaseline: "blocker",
34
57
  verdictImpact: "tool-framework-blocker",
35
- expectedSafeBehavior: "Every subprocess-injection payload round-trips as a single literal argv element; commandRunner.ts uses shell:false.",
58
+ expectedSafeBehavior: "Every subprocess-injection payload round-trips as a single literal argv element; the command runner uses shell:false.",
36
59
  evidenceRequirements: ["command", "observation"],
37
- run: async (ctx) => {
60
+ run: (ctx) => runSubprocessInjectionScenario(ctx, DEFAULT_DEPS),
61
+ };
62
+ export async function runSubprocessInjectionScenario(ctx, deps) {
63
+ {
38
64
  const evidence = [];
39
65
  const notLiteral = [];
40
66
  const payloads = getPayloadsForGroup("subprocess-injection");
41
67
  for (const p of payloads) {
42
- const cmd = await runSecurityCommand({
68
+ const cmd = await deps.runCommand({
43
69
  command: process.execPath,
44
70
  args: ["-e", ECHO_ARGV_SCRIPT, p.value],
45
71
  cwd: ctx.toolRoot,
@@ -70,42 +96,34 @@ export const SUBPROCESS_INJECTION_SCENARIO = {
70
96
  rawPreview: p.value,
71
97
  }));
72
98
  }
73
- let shellFalseConfirmed = false;
74
- let sourceReadable = true;
75
- try {
76
- const source = fs.readFileSync(commandRunnerSourcePath(), "utf8");
77
- shellFalseConfirmed = /shell:\s*false/.test(source);
78
- }
79
- catch {
80
- sourceReadable = false;
81
- }
99
+ const proof = proveShellDisabled(deps, ctx.toolRoot);
82
100
  evidence.push(makeEvidence({
83
101
  kind: "observation",
84
- source: "static check: src/securityValidation/commandRunner.ts",
85
- confidence: sourceReadable ? "high" : "low",
102
+ source: "runtime check: buildSecurityCommandSpawnOptions() used by runSecurityCommand()",
103
+ confidence: proof.state === "unavailable" ? "low" : "high",
86
104
  expectedBehavior: "Subprocess calls should be constructed with shell:false.",
87
- observedBehavior: sourceReadable
88
- ? shellFalseConfirmed
89
- ? "shell:false found in commandRunner.ts."
90
- : "shell:false NOT found in commandRunner.ts."
91
- : "Could not read commandRunner.ts source for static confirmation.",
105
+ observedBehavior: proof.state === "safe"
106
+ ? "Command runner spawn options declare shell:false."
107
+ : proof.state === "unsafe"
108
+ ? proof.detail
109
+ : `Could not verify shell:false: ${proof.detail}`,
92
110
  }));
93
- if (notLiteral.length > 0 || (sourceReadable && !shellFalseConfirmed)) {
111
+ if (notLiteral.length > 0 || proof.state === "unsafe") {
94
112
  return {
95
113
  status: "failed",
96
114
  confidence: "high",
97
115
  evidence,
98
116
  recommendation: notLiteral.length > 0
99
117
  ? `Investigate argument construction — payload(s) did not round-trip literally: ${notLiteral.join(", ")}.`
100
- : "commandRunner.ts must construct child_process.spawn calls with shell:false.",
118
+ : "The command runner must construct child_process.spawn calls with shell:false.",
101
119
  };
102
120
  }
103
- if (!sourceReadable) {
121
+ if (proof.state === "unavailable") {
104
122
  return {
105
123
  status: "blocked",
106
124
  confidence: "low",
107
125
  evidence,
108
- recommendation: "Could not statically confirm shell:false; argv round-trip evidence is still valid.",
126
+ recommendation: "Could not verify shell:false for the command runner; argv round-trip evidence alone is not sufficient.",
109
127
  };
110
128
  }
111
129
  return {
@@ -113,5 +131,5 @@ export const SUBPROCESS_INJECTION_SCENARIO = {
113
131
  confidence: "high",
114
132
  evidence,
115
133
  };
116
- },
117
- };
134
+ }
135
+ }
@@ -5,6 +5,18 @@ import { resolveCommand } from "../core/resolveCommand.js";
5
5
  export function resolveNpmCommand() {
6
6
  return "npm";
7
7
  }
8
+ // The single place the spawn options for security commands are constructed.
9
+ // runSecurityCommand() spawns with exactly this result, so the subprocess
10
+ // injection scenario can verify `shell: false` at runtime from the compiled
11
+ // module instead of reading unshipped TypeScript source.
12
+ export function buildSecurityCommandSpawnOptions(options) {
13
+ return {
14
+ cwd: options.cwd,
15
+ shell: false,
16
+ stdio: ["ignore", "pipe", "pipe"],
17
+ env: options.env,
18
+ };
19
+ }
8
20
  // Runs a security check command with an argument array (no shell interpolation).
9
21
  // npm audit returns exit code 1 on vulnerabilities; that is not treated as a
10
22
  // spawn failure here — the caller inspects the output and exit code separately.
@@ -52,12 +64,7 @@ export async function runSecurityCommand(options) {
52
64
  let timeout;
53
65
  let child;
54
66
  try {
55
- child = spawn(command, args, {
56
- cwd: options.cwd,
57
- shell: false,
58
- stdio: ["ignore", "pipe", "pipe"],
59
- env,
60
- });
67
+ child = spawn(command, args, buildSecurityCommandSpawnOptions({ cwd: options.cwd, env }));
61
68
  }
62
69
  catch (spawnErr) {
63
70
  resolve({
@@ -13,6 +13,29 @@ export function parseNpmPackDryRun(stdout) {
13
13
  if (!stdout.trim()) {
14
14
  return { files: [], parseError: "npm pack --dry-run produced no output" };
15
15
  }
16
+ if (/^[\[{]/.test(stdout.trim())) {
17
+ try {
18
+ const inventory = JSON.parse(stdout);
19
+ if (!Array.isArray(inventory) || inventory.length !== 1 ||
20
+ !inventory[0] || !Array.isArray(inventory[0].files) || !inventory[0].files.length) {
21
+ return { files: [], parseError: "npm pack JSON must contain exactly one nonempty package inventory" };
22
+ }
23
+ const files = [];
24
+ for (const file of inventory[0].files) {
25
+ if (!file || typeof file.path !== "string" || !safeInventoryPath(file.path)) {
26
+ return { files: [], parseError: "npm pack JSON contains a malformed or unsafe file path" };
27
+ }
28
+ // JSON paths are package-relative, unlike text tarball paths. Add the
29
+ // archive root exactly once so a real nested package/ directory cannot
30
+ // masquerade as a root entrypoint in the existing detectors.
31
+ files.push(`package/${file.path.replace(/\\/g, "/").replace(/^(\.\/)+/, "")}`);
32
+ }
33
+ return { files };
34
+ }
35
+ catch {
36
+ return { files: [], parseError: "npm pack JSON is unparseable" };
37
+ }
38
+ }
16
39
  const files = [];
17
40
  let inContents = false;
18
41
  let totalSize;
@@ -35,10 +58,8 @@ export function parseNpmPackDryRun(stdout) {
35
58
  if (match?.[1]) {
36
59
  files.push(match[1].trim());
37
60
  }
38
- else {
39
- // Fallback: take the whole line if size pattern doesn't match
40
- files.push(line);
41
- }
61
+ else
62
+ return { files: [], parseError: "Malformed npm tarball contents line" };
42
63
  }
43
64
  const totalMatch = line.match(/total files\s*:\s*(\d+)/i) ?? line.match(/unpacked size\s*:\s*([\d.]+\s*[kKmMgGbB]+)/i);
44
65
  if (totalMatch?.[1]) {
@@ -52,5 +73,14 @@ export function parseNpmPackDryRun(stdout) {
52
73
  parseError: "npm pack --dry-run: no files detected in tarball contents section",
53
74
  };
54
75
  }
76
+ if (files.some((file) => !safeInventoryPath(file))) {
77
+ return { files: [], parseError: "npm pack contains an unsafe file path" };
78
+ }
55
79
  return { files, totalSize };
56
80
  }
81
+ function safeInventoryPath(file) {
82
+ const normalized = file.replace(/\\/g, "/").replace(/^(\.\/)+/, "");
83
+ return Boolean(normalized) && !normalized.startsWith("/") &&
84
+ !/[\x00-\x1f\x7f:]/.test(normalized) &&
85
+ normalized.split("/").every((part) => Boolean(part) && part !== "." && part !== "..");
86
+ }
@@ -1,3 +1,5 @@
1
+ import fs from "node:fs/promises";
2
+ import path from "node:path";
1
3
  export const REQUIRED_PACKAGE_CONTENTS = [
2
4
  "CHANGELOG.md",
3
5
  "LICENSE",
@@ -20,15 +22,117 @@ export const REQUIRED_PACKAGE_CONTENTS = [
20
22
  "examples/token-savings-cases.json",
21
23
  ];
22
24
  function normalizeTarballPath(file) {
23
- const normalized = file.replace(/\\/g, "/");
25
+ const normalized = file.replace(/\\/g, "/").replace(/^(\.\/)+/, "");
24
26
  return normalized.startsWith("package/")
25
27
  ? normalized.slice("package/".length)
26
28
  : normalized;
27
29
  }
30
+ // Policy belongs beside the existing required-file detector. Identity is
31
+ // supplied by the existing target resolver, never inferred from package name.
32
+ export async function requiredContentsForTarget(target) {
33
+ const required = new Set(target.isSelf ? REQUIRED_PACKAGE_CONTENTS : ["package.json"]);
34
+ const findings = [];
35
+ function invalid(field, reason) {
36
+ findings.push({
37
+ id: `npm-pack-metadata-${findings.length}`,
38
+ title: `Package metadata cannot be validated: ${field}`,
39
+ severity: "blocker",
40
+ category: "package-content",
41
+ description: reason,
42
+ evidence: `package.json field: ${field}; ${reason}`,
43
+ affectedFiles: ["package.json"],
44
+ recommendation: "Provide readable package metadata with supported, safe package-relative entrypoint paths and rerun validation.",
45
+ releaseImpact: "Blocker: required package evidence is incomplete",
46
+ });
47
+ }
48
+ function entry(value, field, exportsPath = false) {
49
+ if (typeof value !== "string") {
50
+ invalid(field, "Malformed entrypoint: expected a nonempty string.");
51
+ return;
52
+ }
53
+ const normalized = value.replace(/\\/g, "/");
54
+ const relative = normalized.replace(/^(\.\/)+/, "");
55
+ if (!relative || normalized !== normalized.trim() ||
56
+ /[\x00-\x1f\x7f:*?#%]/.test(normalized) || normalized.startsWith("/") ||
57
+ relative.split("/").some((part) => !part || part === ".." || part === "." || part === "node_modules") ||
58
+ (exportsPath && !normalized.startsWith("./"))) {
59
+ invalid(field, "Unsafe or unsupported entrypoint path (absolute, escaping, pattern, URL, or ambiguous path).");
60
+ return;
61
+ }
62
+ required.add(relative);
63
+ }
64
+ try {
65
+ const metadata = JSON.parse(await fs.readFile(path.join(target.targetRoot, "package.json"), "utf8"));
66
+ if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
67
+ invalid("package.json", "Malformed package metadata: expected an object.");
68
+ }
69
+ else if (!target.isSelf) {
70
+ const pkg = metadata;
71
+ if ("bin" in pkg) {
72
+ if (typeof pkg.bin === "string")
73
+ entry(pkg.bin, "bin");
74
+ else if (pkg.bin && typeof pkg.bin === "object" && !Array.isArray(pkg.bin)) {
75
+ for (const key of Object.keys(pkg.bin).sort()) {
76
+ if (!key || key === "." || key === ".." || /[\\/:\x00-\x1f\x7f]/.test(key)) {
77
+ invalid("bin", "Unsafe or unsupported executable name in bin map.");
78
+ }
79
+ else
80
+ entry(pkg.bin[key], `bin.${key}`);
81
+ }
82
+ }
83
+ else
84
+ invalid("bin", "Malformed bin: expected a string or object map.");
85
+ }
86
+ if ("main" in pkg) {
87
+ if (typeof pkg.main === "string" && pkg.main && !path.posix.extname(pkg.main.replace(/\\/g, "/"))) {
88
+ invalid("main", "Unsupported or ambiguous main: extensionless or directory resolution is not validated; declare an exact runtime file.");
89
+ }
90
+ else
91
+ entry(pkg.main, "main");
92
+ }
93
+ if ("exports" in pkg && pkg.exports !== null) {
94
+ if (typeof pkg.exports === "string")
95
+ entry(pkg.exports, "exports", true);
96
+ else if (pkg.exports && typeof pkg.exports === "object" && !Array.isArray(pkg.exports)) {
97
+ const exports = pkg.exports;
98
+ const keys = Object.keys(exports).sort();
99
+ if (!keys.length || keys.some((key) => key !== "." && (!key.startsWith("./") || /[\\:*?#%\x00-\x1f\x7f]/.test(key) ||
100
+ key.slice(2).split("/").some((part) => !part || part === "." || part === "..")))) {
101
+ invalid("exports", "Unsupported or ambiguous exports: conditional or pattern mappings require additional evidence.");
102
+ }
103
+ else
104
+ for (const key of keys) {
105
+ if (exports[key] === null)
106
+ continue; // Explicitly excluded subpath.
107
+ if (typeof exports[key] !== "string") {
108
+ invalid(`exports.${key}`, "Unsupported exports shape: conditional objects and fallback arrays are not validated.");
109
+ }
110
+ else
111
+ entry(exports[key], `exports.${key}`, true);
112
+ }
113
+ }
114
+ else
115
+ invalid("exports", "Unsupported exports shape: expected an exact string or subpath map.");
116
+ }
117
+ // npm automatically ships existing root README and license files. Do not
118
+ // invent a documentation requirement for packages without those files.
119
+ for (const file of (await fs.readdir(target.targetRoot, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
120
+ if (file.isFile() && /^(readme|licen[cs]e)(\..*)?$/i.test(file.name))
121
+ required.add(file.name);
122
+ }
123
+ }
124
+ }
125
+ catch {
126
+ invalid("package.json", "Package metadata is missing, unreadable, or unparseable.");
127
+ }
128
+ return { requiredFiles: [...required].sort(), findings };
129
+ }
28
130
  export function detectMissingRequiredContents(options) {
29
131
  const present = new Set(options.files.map(normalizeTarballPath));
30
132
  const required = [...(options.requiredFiles ?? REQUIRED_PACKAGE_CONTENTS)]
31
- .map(normalizeTarballPath)
133
+ // Requirements are package-relative declarations, not archive paths. A
134
+ // literal package/ directory in an entrypoint must not be stripped.
135
+ .map((file) => file.replace(/\\/g, "/").replace(/^(\.\/)+/, ""))
32
136
  .sort((left, right) => left.localeCompare(right));
33
137
  const missing = required.filter((file) => !present.has(file));
34
138
  const findings = missing.map((file, index) => ({
@@ -3,7 +3,9 @@ import { runSecurityCommand, resolveNpmCommand } from "../commandRunner.js";
3
3
  import { writeCheckResult } from "../artifacts.js";
4
4
  import { parseNpmPackDryRun } from "./parseNpmPackDryRun.js";
5
5
  import { detectForbiddenContents } from "./forbiddenPackageContents.js";
6
- import { detectMissingRequiredContents } from "./requiredPackageContents.js";
6
+ import { detectMissingRequiredContents, requiredContentsForTarget } from "./requiredPackageContents.js";
7
+ import { resolveValidationTarget } from "../validate/resolveTarget.js";
8
+ import { discoverPackageRoot } from "../../runtime/packageRoot.js";
7
9
  // Run npm pack --dry-run and inspect the resulting file list for forbidden contents.
8
10
  // Does not publish anything.
9
11
  export async function runPackageChecks(options) {
@@ -12,20 +14,23 @@ export async function runPackageChecks(options) {
12
14
  const allFindings = [];
13
15
  const checks = [];
14
16
  const npm = resolveNpmCommand();
17
+ const target = options.target ?? resolveValidationTarget(path.resolve(cwd), discoverPackageRoot());
18
+ if (path.resolve(target.targetRoot) !== path.resolve(cwd)) {
19
+ throw new Error("Package check cwd does not match the resolved target root");
20
+ }
21
+ const policy = await requiredContentsForTarget(target);
15
22
  // npm pack --dry-run to get the tarball file list
16
23
  const startedAt = new Date().toISOString();
17
24
  const cmd = await runSecurityCommand({
18
25
  command: npm,
19
- args: ["pack", "--dry-run"],
26
+ args: ["pack", "--dry-run", "--json", "--ignore-scripts", "--cache", path.resolve(rawOutputDir, "npm-cache")],
20
27
  cwd,
21
28
  timeoutMs: commandTimeoutMs,
22
29
  });
23
30
  const finishedAt = new Date().toISOString();
24
- // npm pack --dry-run writes the tarball filename to stdout on some npm
25
- // versions and the detailed file list to stderr. Prefer whichever stream
26
- // contains the tarball contents section, then fall back to combined output.
27
- const packOutput = [cmd.stdout, cmd.stderr].find((stream) => /tarball contents/i.test(stream)) ??
28
- [cmd.stdout, cmd.stderr].filter(Boolean).join("\n");
31
+ // Supported npm versions emit JSON on stdout. Lifecycle scripts are disabled
32
+ // so inspection cannot build, install, or rewrite the target.
33
+ const packOutput = cmd.stdout;
29
34
  const parsed = parseNpmPackDryRun(packOutput);
30
35
  const { findings: contentFindings } = detectForbiddenContents({
31
36
  files: parsed.files,
@@ -36,20 +41,28 @@ export async function runPackageChecks(options) {
36
41
  const { findings: missingRequiredFindings } = detectMissingRequiredContents({
37
42
  files: parsed.files,
38
43
  checkId: "npm-pack",
44
+ requiredFiles: policy.requiredFiles,
39
45
  });
40
- const packageFindings = [...contentFindings, ...missingRequiredFindings];
46
+ const evidenceFindings = [];
47
+ if (cmd.exitCode !== 0 || cmd.timedOut || cmd.skipped || parsed.parseError || parsed.files.length === 0) {
48
+ evidenceFindings.push({
49
+ id: "npm-pack-evidence-invalid",
50
+ title: "npm package inventory could not be validated",
51
+ severity: "blocker",
52
+ category: "package-content",
53
+ description: "A successful npm pack operation and a nonempty, parseable inventory are required.",
54
+ evidence: `exitCode=${String(cmd.exitCode)}; timedOut=${cmd.timedOut}; skipped=${cmd.skipped}; ${parsed.parseError ?? ""}; ${cmd.stderr.slice(0, 500)}`,
55
+ recommendation: "Resolve the package command or inventory failure and rerun validation.",
56
+ releaseImpact: "Blocker: required package evidence is unavailable",
57
+ });
58
+ }
59
+ const packageFindings = [...policy.findings, ...evidenceFindings, ...contentFindings, ...missingRequiredFindings];
41
60
  allFindings.push(...packageFindings);
42
61
  const packCheck = {
43
62
  id: "npm-pack-dry-run",
44
63
  name: "npm pack --dry-run (tarball file list)",
45
64
  category: "package-content",
46
- status: cmd.timedOut
47
- ? "failed"
48
- : parsed.parseError && parsed.files.length === 0
49
- ? "warning"
50
- : packageFindings.length > 0
51
- ? "failed"
52
- : "passed",
65
+ status: packageFindings.length > 0 ? "failed" : "passed",
53
66
  severity: packageFindings.length > 0
54
67
  ? packageFindings.some((f) => f.severity === "blocker")
55
68
  ? "blocker"
@@ -60,7 +73,9 @@ export async function runPackageChecks(options) {
60
73
  durationMs: cmd.durationMs,
61
74
  findings: packageFindings,
62
75
  skippedReason: undefined,
63
- command: "npm pack --dry-run",
76
+ command: "npm pack --dry-run --json --ignore-scripts",
77
+ commandCwd: cwd,
78
+ exitCode: cmd.exitCode,
64
79
  };
65
80
  await writeCheckResult({
66
81
  result: packCheck,
@@ -5,16 +5,35 @@ import { resolveCommand } from "../../core/resolveCommand.js";
5
5
  import { skippedCheck } from "../cliAdversarial/runAdversarialCheck.js";
6
6
  export function parseSemgrepJson(raw) {
7
7
  if (!raw.trim()) {
8
- return { findings: [], rawOutput: raw };
8
+ return { findings: [], parseError: "Semgrep produced no JSON scan evidence", rawOutput: raw };
9
9
  }
10
10
  try {
11
11
  const parsed = JSON.parse(raw);
12
- const results = Array.isArray(parsed["results"]) ? parsed["results"] : [];
13
- const findings = results.map((r) => {
12
+ if (!isRecord(parsed) || !Array.isArray(parsed.results)) {
13
+ return { findings: [], parseError: "Semgrep JSON requires a results array", rawOutput: raw };
14
+ }
15
+ const findings = [];
16
+ let parseError = !Array.isArray(parsed.errors) || !isRecord(parsed.paths) ||
17
+ !Array.isArray(parsed.paths.scanned) || parsed.paths.scanned.some((file) => typeof file !== "string")
18
+ ? "Semgrep JSON requires errors and paths.scanned arrays" : undefined;
19
+ for (const [index, r] of parsed.results.entries()) {
20
+ if (!isRecord(r) || typeof r.check_id !== "string" || !r.check_id ||
21
+ typeof r.path !== "string" || !r.path || !isRecord(r.start) ||
22
+ !Number.isInteger(r.start.line) || Number(r.start.line) < 1 || !isRecord(r.extra) ||
23
+ (r.extra.message !== undefined && typeof r.extra.message !== "string") ||
24
+ (r.extra.metadata !== undefined && !isRecord(r.extra.metadata))) {
25
+ parseError ??= `Malformed Semgrep result at index ${index}`;
26
+ continue;
27
+ }
14
28
  const item = r;
15
29
  const checkId = typeof item["check_id"] === "string" ? item["check_id"] : "unknown";
16
30
  const extra = item["extra"] ?? {};
17
31
  const metadata = extra["metadata"] ?? {};
32
+ if ((extra.severity !== undefined && (typeof extra.severity !== "string" || !extra.severity.trim())) ||
33
+ (extra.severity === undefined && (typeof metadata.severity !== "string" || !metadata.severity.trim()))) {
34
+ parseError ??= `Missing or malformed Semgrep severity at index ${index}`;
35
+ continue;
36
+ }
18
37
  const severity = typeof extra["severity"] === "string"
19
38
  ? extra["severity"]
20
39
  : typeof metadata["severity"] === "string"
@@ -24,9 +43,14 @@ export function parseSemgrepJson(raw) {
24
43
  const filePath = typeof item["path"] === "string" ? item["path"] : "";
25
44
  const startObj = item["start"] ?? {};
26
45
  const line = typeof startObj["line"] === "number" ? startObj["line"] : 0;
27
- return { ruleId: checkId, severity, message, path: filePath, line };
28
- });
29
- return { findings, rawOutput: raw };
46
+ findings.push({ ruleId: checkId, severity, message, path: filePath, line });
47
+ }
48
+ // Semgrep's CLI contract reports partial/fatal analysis in errors even
49
+ // when the process exits zero. Retain matches, but never call this clean.
50
+ const analysisError = Array.isArray(parsed.errors) && parsed.errors.length > 0
51
+ ? `Semgrep reported ${parsed.errors.length} analysis error(s): ${JSON.stringify(parsed.errors).slice(0, 500)}`
52
+ : undefined;
53
+ return { findings, parseError, analysisError, rawOutput: raw };
30
54
  }
31
55
  catch (err) {
32
56
  return {
@@ -36,6 +60,9 @@ export function parseSemgrepJson(raw) {
36
60
  };
37
61
  }
38
62
  }
63
+ function isRecord(value) {
64
+ return value !== null && typeof value === "object" && !Array.isArray(value);
65
+ }
39
66
  function semgrepSeverityToSecurity(semgrepSeverity) {
40
67
  switch (semgrepSeverity.toUpperCase()) {
41
68
  case "ERROR":
@@ -88,67 +115,25 @@ export async function runSemgrepCheck(options) {
88
115
  "--quiet",
89
116
  scanTarget,
90
117
  ];
91
- const cmd = await runSecurityCommand({
92
- command,
93
- args: scanArgs,
94
- cwd: targetRoot,
95
- timeoutMs,
96
- });
118
+ let cmd;
119
+ try {
120
+ cmd = await runSecurityCommand({ command, args: scanArgs, cwd: targetRoot, timeoutMs });
121
+ }
122
+ catch (error) {
123
+ cmd = { command, args: scanArgs, cwd: targetRoot, exitCode: null, durationMs: 0,
124
+ stdout: "", stderr: error instanceof Error ? error.message : String(error), timedOut: false, skipped: false };
125
+ }
97
126
  const finishedAt = new Date().toISOString();
98
- // Semgrep exits 0 = no findings, 1 = findings found, other = error.
99
- const isError = cmd.exitCode !== 0 && cmd.exitCode !== 1;
100
- const isTimedOut = cmd.timedOut;
101
- if (isTimedOut || (isError && !cmd.stdout.trim())) {
102
- return {
103
- id: "semgrep-scan",
104
- name: "Semgrep static analysis",
105
- category: "static-scan",
106
- status: "failed",
107
- severity: "major",
108
- startedAt,
109
- finishedAt,
110
- durationMs: cmd.durationMs,
111
- findings: [
112
- {
113
- id: "semgrep-execution-error",
114
- title: "Semgrep execution failed",
115
- severity: "major",
116
- category: "static-scan",
117
- description: isTimedOut
118
- ? "Semgrep timed out."
119
- : `Semgrep exited with code ${String(cmd.exitCode)}. stderr: ${cmd.stderr.slice(0, 500)}`,
120
- recommendation: "Check semgrep installation and .semgrep.yml config.",
121
- releaseImpact: "Review before release",
122
- },
123
- ],
124
- command: [command, ...scanArgs].join(" "),
125
- };
127
+ // Distinguish positively established absence from a failed attempted scan.
128
+ // Network, permission, generic npm, or native spawn errors are not absence.
129
+ const unavailable = !cmd.timedOut && cmd.exitCode !== 0 && !cmd.stdout.trim() && ((cmd.skipped && cmd.exitCode === null && Boolean(cmd.skippedReason?.match(/\b(semgrep|scanner|cli)\b.*(unavailable|not installed|not found|not on PATH)/i))) ||
130
+ (useNpx && !cmd.skipped && cmd.exitCode === 1 && /could not determine executable to run/i.test(cmd.stderr)));
131
+ if (unavailable) {
132
+ return { ...skippedCheck({ id: "semgrep-scan", name: "Semgrep static analysis", category: "static-scan",
133
+ reason: cmd.skippedReason ?? "Semgrep CLI is unavailable; the npx fallback package provides no executable." }),
134
+ command: [command, ...scanArgs].join(" "), commandCwd: targetRoot, exitCode: cmd.exitCode };
126
135
  }
127
136
  const parsed = parseSemgrepJson(cmd.stdout);
128
- if (parsed.parseError) {
129
- return {
130
- id: "semgrep-scan",
131
- name: "Semgrep static analysis",
132
- category: "static-scan",
133
- status: "warning",
134
- severity: "minor",
135
- startedAt,
136
- finishedAt,
137
- durationMs: cmd.durationMs,
138
- findings: [
139
- {
140
- id: "semgrep-parse-error",
141
- title: "Could not parse Semgrep output",
142
- severity: "minor",
143
- category: "static-scan",
144
- description: `JSON parse error: ${parsed.parseError}`,
145
- recommendation: "Verify semgrep version produces valid JSON output.",
146
- releaseImpact: "Review before release",
147
- },
148
- ],
149
- command: [command, ...scanArgs].join(" "),
150
- };
151
- }
152
137
  const securityFindings = parsed.findings.map((f) => ({
153
138
  id: `semgrep-${f.ruleId.replace(/[^a-z0-9-]/gi, "-")}`,
154
139
  title: f.ruleId,
@@ -160,6 +145,23 @@ export async function runSemgrepCheck(options) {
160
145
  ? "Should fix before release"
161
146
  : "Review before release",
162
147
  }));
148
+ const executionFailed = cmd.timedOut || cmd.skipped || (cmd.exitCode !== 0 && cmd.exitCode !== 1) ||
149
+ (cmd.exitCode === 1 && parsed.findings.length === 0);
150
+ const evidenceError = executionFailed
151
+ ? `Semgrep execution failed: exitCode=${String(cmd.exitCode)}, timedOut=${cmd.timedOut}, skipped=${cmd.skipped}. ${cmd.stderr.slice(0, 500)}`
152
+ : parsed.parseError ?? parsed.analysisError;
153
+ if (evidenceError) {
154
+ securityFindings.push({
155
+ id: executionFailed ? "semgrep-execution-error" : parsed.parseError ? "semgrep-parse-error" : "semgrep-analysis-error",
156
+ title: "Semgrep scan evidence is incomplete or unsuccessful",
157
+ severity: "major",
158
+ category: "static-scan",
159
+ description: evidenceError,
160
+ evidence: `exitCode=${String(cmd.exitCode)}; ${parsed.parseError ?? parsed.analysisError ?? ""}`,
161
+ recommendation: "Resolve the scanner execution or analysis error and rerun with valid JSON scan evidence.",
162
+ releaseImpact: "Review before release",
163
+ });
164
+ }
163
165
  const hasBlockerOrMajor = securityFindings.some((f) => f.severity === "blocker" || f.severity === "major");
164
166
  return {
165
167
  id: "semgrep-scan",
@@ -176,5 +178,7 @@ export async function runSemgrepCheck(options) {
176
178
  durationMs: cmd.durationMs,
177
179
  findings: securityFindings,
178
180
  command: [command, ...scanArgs].join(" "),
181
+ commandCwd: targetRoot,
182
+ exitCode: cmd.exitCode,
179
183
  };
180
184
  }
@@ -1,7 +1,3 @@
1
- // Core types for the my-dev-kit-lab security-validation framework.
2
- // These types describe the vocabulary for checks, findings, results, and verdicts
3
- // used by dependency checks (Prompt 3), CLI adversarial tests (Prompts 4–5),
4
- // static scans (Prompt 6), fuzz smoke tests (Prompt 7), and the release report (Prompt 8).
5
1
  export const SECURITY_SEVERITIES = [
6
2
  "blocker",
7
3
  "major",