@dailephd/my-dev-kit-lab 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -2
- package/README.md +10 -2
- package/dist/src/agents/adapters/codexAdapter.js +4 -1
- package/dist/src/cli/help.js +38 -2
- package/dist/src/cli/runLabCli.js +9 -1
- package/dist/src/commands/buildGalleryCommand.js +16 -3
- package/dist/src/commands/runComparePromptsCommand.js +109 -0
- package/dist/src/commands/runExperimentRunCommand.js +51 -5
- package/dist/src/core/processTree.js +41 -17
- package/dist/src/core/runMeasuredCommand.js +14 -8
- package/dist/src/experiments/plugins/agentSuccessRate/campaignCheckpointStore.js +24 -2
- package/dist/src/experiments/plugins/agentSuccessRate/campaignExecution.js +7 -2
- package/dist/src/experiments/plugins/agentSuccessRate/campaignIdentity.js +29 -4
- package/dist/src/experiments/plugins/agentSuccessRate/campaignProgress.js +4 -2
- package/dist/src/experiments/plugins/agentSuccessRate/campaignResumableExecution.js +1 -1
- package/dist/src/experiments/plugins/agentSuccessRate/config.js +47 -2
- package/dist/src/experiments/plugins/agentSuccessRate/plugin.js +4 -2
- package/dist/src/experiments/plugins/agentSuccessRate/promptContract.js +121 -0
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentExecution.js +32 -2
- package/dist/src/experiments/plugins/agentSuccessRate/realAgentPrompt.js +105 -11
- package/dist/src/experiments/plugins/agentSuccessRate/strictDiffResponse.js +312 -0
- package/dist/src/gallery/galleryPathSafety.js +111 -0
- package/dist/src/gallery/readGallerySources.js +542 -0
- package/dist/src/gallery/writeGalleryManifest.js +57 -3
- package/dist/src/report/experiments/buildPluginExperimentReport.js +4 -16
- package/dist/src/report/experiments/htmlSafety.js +32 -0
- package/dist/src/report/experiments/metricEvidence.js +389 -0
- package/dist/src/report/experiments/metricGlossary.js +227 -0
- package/dist/src/report/experiments/metricGlossaryCatalog.js +199 -0
- package/dist/src/report/experiments/promptTemplateComparisonEvidence.js +377 -0
- package/dist/src/report/experiments/promptTemplateComparisonModel.js +436 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +158 -64
- package/dist/src/report/experiments/renderPluginExperimentReportText.js +22 -677
- package/dist/src/report/experiments/renderPromptTemplateComparison.js +192 -0
- package/dist/src/report/experiments/renderSpecializedTextSections.js +633 -0
- package/dist/src/report/experiments/reportSectionRegistry.js +79 -0
- package/dist/src/report/experiments/specializedReportSections.js +142 -0
- package/dist/src/report/experiments/writeAgentSuccessCampaignProgressReport.js +6 -2
- package/dist/src/report/experiments/writePluginExperimentReports.js +4 -2
- package/dist/src/securityValidation/attackScenarios/payloadCorpus.js +0 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +146 -133
- package/dist/src/securityValidation/attackScenarios/scenarios/subprocessInjectionScenario.js +54 -36
- package/dist/src/securityValidation/commandRunner.js +13 -6
- package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +34 -4
- package/dist/src/securityValidation/packageChecks/requiredPackageContents.js +106 -2
- package/dist/src/securityValidation/packageChecks/runPackageChecks.js +31 -16
- package/dist/src/securityValidation/staticScans/semgrep.js +68 -64
- package/dist/src/securityValidation/types.js +0 -4
- package/dist/src/securityValidation/validate/runSecurityValidation.js +1 -1
- package/docs/METRICS.md +12 -2
- package/examples/tutorial-browser/prepare.mjs +13 -13
- package/examples/tutorial-browser/server.mjs +13 -13
- package/package.json +1 -1
package/dist/src/securityValidation/attackScenarios/scenarios/subprocessInjectionScenario.js
CHANGED
|
@@ -1,9 +1,6 @@
|
|
|
1
|
-
import fs from "node:fs";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { fileURLToPath } from "node:url";
|
|
4
1
|
import { makeEvidence } from "../exploitEvidence.js";
|
|
5
2
|
import { getPayloadsForGroup } from "../payloadCorpus.js";
|
|
6
|
-
import { runSecurityCommand } from "../../commandRunner.js";
|
|
3
|
+
import { buildSecurityCommandSpawnOptions, runSecurityCommand } from "../../commandRunner.js";
|
|
7
4
|
// ---------------------------------------------------------------------------
|
|
8
5
|
// v0.2.2 Batch 3 — subprocess injection scenario.
|
|
9
6
|
//
|
|
@@ -15,31 +12,60 @@ import { runSecurityCommand } from "../../commandRunner.js";
|
|
|
15
12
|
// that no shell re-interpreted it (splitting it into multiple commands via
|
|
16
13
|
// `;`, `&&`, backticks, `$()`, or `|`).
|
|
17
14
|
//
|
|
18
|
-
// Also
|
|
19
|
-
//
|
|
20
|
-
//
|
|
15
|
+
// Also verifies at runtime, through the compiled command runner's own
|
|
16
|
+
// spawn-options builder (the exact value runSecurityCommand() spawns with),
|
|
17
|
+
// that the runner does not enable a shell. No TypeScript source is read, so
|
|
18
|
+
// the proof works from an installed package.
|
|
21
19
|
// ---------------------------------------------------------------------------
|
|
22
20
|
const ECHO_ARGV_SCRIPT = "process.stdout.write(JSON.stringify(process.argv.slice(1)))";
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
21
|
+
const DEFAULT_DEPS = {
|
|
22
|
+
runCommand: runSecurityCommand,
|
|
23
|
+
readSpawnOptions: (cwd) => buildSecurityCommandSpawnOptions({ cwd, env: process.env }),
|
|
24
|
+
};
|
|
25
|
+
// Runtime proof that the applicable runner's spawn path does not use a shell.
|
|
26
|
+
// A strict `shell === false` is the only accepted safe value; a truthy shell
|
|
27
|
+
// is demonstrated unsafe; a missing/malformed/throwing proof is unavailable.
|
|
28
|
+
function proveShellDisabled(deps, cwd) {
|
|
29
|
+
let options;
|
|
30
|
+
try {
|
|
31
|
+
options = deps.readSpawnOptions(cwd);
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
return {
|
|
35
|
+
state: "unavailable",
|
|
36
|
+
detail: `Spawn options could not be obtained from the command runner: ${error instanceof Error ? error.message : String(error)}`,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
if (typeof options !== "object" || options === null || !("shell" in options)) {
|
|
40
|
+
return { state: "unavailable", detail: "Command runner spawn options did not declare a shell setting." };
|
|
41
|
+
}
|
|
42
|
+
const shell = options.shell;
|
|
43
|
+
if (shell === false)
|
|
44
|
+
return { state: "safe" };
|
|
45
|
+
if (shell === true || typeof shell === "string") {
|
|
46
|
+
return { state: "unsafe", detail: "Command runner spawn options enable a shell." };
|
|
47
|
+
}
|
|
48
|
+
return { state: "unavailable", detail: "Command runner spawn options declared a malformed shell setting." };
|
|
26
49
|
}
|
|
27
50
|
export const SUBPROCESS_INJECTION_SCENARIO = {
|
|
28
51
|
id: "subprocess-injection-safety",
|
|
29
52
|
title: "Subprocess injection: shell metacharacter payloads are passed as literal arguments",
|
|
30
|
-
description: "Runs a benign argv-echoing Node script with each subprocess-injection payload as a single spawn argument and confirms it round-trips as exactly one literal string, proving no shell re-interpretation. Also
|
|
53
|
+
description: "Runs a benign argv-echoing Node script with each subprocess-injection payload as a single spawn argument and confirms it round-trips as exactly one literal string, proving no shell re-interpretation. Also verifies at runtime that the command runner constructs subprocess calls with shell:false.",
|
|
31
54
|
checkId: "subprocess",
|
|
32
55
|
applicableProfiles: [],
|
|
33
56
|
severityBaseline: "blocker",
|
|
34
57
|
verdictImpact: "tool-framework-blocker",
|
|
35
|
-
expectedSafeBehavior: "Every subprocess-injection payload round-trips as a single literal argv element;
|
|
58
|
+
expectedSafeBehavior: "Every subprocess-injection payload round-trips as a single literal argv element; the command runner uses shell:false.",
|
|
36
59
|
evidenceRequirements: ["command", "observation"],
|
|
37
|
-
run:
|
|
60
|
+
run: (ctx) => runSubprocessInjectionScenario(ctx, DEFAULT_DEPS),
|
|
61
|
+
};
|
|
62
|
+
export async function runSubprocessInjectionScenario(ctx, deps) {
|
|
63
|
+
{
|
|
38
64
|
const evidence = [];
|
|
39
65
|
const notLiteral = [];
|
|
40
66
|
const payloads = getPayloadsForGroup("subprocess-injection");
|
|
41
67
|
for (const p of payloads) {
|
|
42
|
-
const cmd = await
|
|
68
|
+
const cmd = await deps.runCommand({
|
|
43
69
|
command: process.execPath,
|
|
44
70
|
args: ["-e", ECHO_ARGV_SCRIPT, p.value],
|
|
45
71
|
cwd: ctx.toolRoot,
|
|
@@ -70,42 +96,34 @@ export const SUBPROCESS_INJECTION_SCENARIO = {
|
|
|
70
96
|
rawPreview: p.value,
|
|
71
97
|
}));
|
|
72
98
|
}
|
|
73
|
-
|
|
74
|
-
let sourceReadable = true;
|
|
75
|
-
try {
|
|
76
|
-
const source = fs.readFileSync(commandRunnerSourcePath(), "utf8");
|
|
77
|
-
shellFalseConfirmed = /shell:\s*false/.test(source);
|
|
78
|
-
}
|
|
79
|
-
catch {
|
|
80
|
-
sourceReadable = false;
|
|
81
|
-
}
|
|
99
|
+
const proof = proveShellDisabled(deps, ctx.toolRoot);
|
|
82
100
|
evidence.push(makeEvidence({
|
|
83
101
|
kind: "observation",
|
|
84
|
-
source: "
|
|
85
|
-
confidence:
|
|
102
|
+
source: "runtime check: buildSecurityCommandSpawnOptions() used by runSecurityCommand()",
|
|
103
|
+
confidence: proof.state === "unavailable" ? "low" : "high",
|
|
86
104
|
expectedBehavior: "Subprocess calls should be constructed with shell:false.",
|
|
87
|
-
observedBehavior:
|
|
88
|
-
?
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
105
|
+
observedBehavior: proof.state === "safe"
|
|
106
|
+
? "Command runner spawn options declare shell:false."
|
|
107
|
+
: proof.state === "unsafe"
|
|
108
|
+
? proof.detail
|
|
109
|
+
: `Could not verify shell:false: ${proof.detail}`,
|
|
92
110
|
}));
|
|
93
|
-
if (notLiteral.length > 0 ||
|
|
111
|
+
if (notLiteral.length > 0 || proof.state === "unsafe") {
|
|
94
112
|
return {
|
|
95
113
|
status: "failed",
|
|
96
114
|
confidence: "high",
|
|
97
115
|
evidence,
|
|
98
116
|
recommendation: notLiteral.length > 0
|
|
99
117
|
? `Investigate argument construction — payload(s) did not round-trip literally: ${notLiteral.join(", ")}.`
|
|
100
|
-
: "
|
|
118
|
+
: "The command runner must construct child_process.spawn calls with shell:false.",
|
|
101
119
|
};
|
|
102
120
|
}
|
|
103
|
-
if (
|
|
121
|
+
if (proof.state === "unavailable") {
|
|
104
122
|
return {
|
|
105
123
|
status: "blocked",
|
|
106
124
|
confidence: "low",
|
|
107
125
|
evidence,
|
|
108
|
-
recommendation: "Could not
|
|
126
|
+
recommendation: "Could not verify shell:false for the command runner; argv round-trip evidence alone is not sufficient.",
|
|
109
127
|
};
|
|
110
128
|
}
|
|
111
129
|
return {
|
|
@@ -113,5 +131,5 @@ export const SUBPROCESS_INJECTION_SCENARIO = {
|
|
|
113
131
|
confidence: "high",
|
|
114
132
|
evidence,
|
|
115
133
|
};
|
|
116
|
-
}
|
|
117
|
-
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
@@ -5,6 +5,18 @@ import { resolveCommand } from "../core/resolveCommand.js";
|
|
|
5
5
|
export function resolveNpmCommand() {
|
|
6
6
|
return "npm";
|
|
7
7
|
}
|
|
8
|
+
// The single place the spawn options for security commands are constructed.
|
|
9
|
+
// runSecurityCommand() spawns with exactly this result, so the subprocess
|
|
10
|
+
// injection scenario can verify `shell: false` at runtime from the compiled
|
|
11
|
+
// module instead of reading unshipped TypeScript source.
|
|
12
|
+
export function buildSecurityCommandSpawnOptions(options) {
|
|
13
|
+
return {
|
|
14
|
+
cwd: options.cwd,
|
|
15
|
+
shell: false,
|
|
16
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
17
|
+
env: options.env,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
8
20
|
// Runs a security check command with an argument array (no shell interpolation).
|
|
9
21
|
// npm audit returns exit code 1 on vulnerabilities; that is not treated as a
|
|
10
22
|
// spawn failure here — the caller inspects the output and exit code separately.
|
|
@@ -52,12 +64,7 @@ export async function runSecurityCommand(options) {
|
|
|
52
64
|
let timeout;
|
|
53
65
|
let child;
|
|
54
66
|
try {
|
|
55
|
-
child = spawn(command, args, {
|
|
56
|
-
cwd: options.cwd,
|
|
57
|
-
shell: false,
|
|
58
|
-
stdio: ["ignore", "pipe", "pipe"],
|
|
59
|
-
env,
|
|
60
|
-
});
|
|
67
|
+
child = spawn(command, args, buildSecurityCommandSpawnOptions({ cwd: options.cwd, env }));
|
|
61
68
|
}
|
|
62
69
|
catch (spawnErr) {
|
|
63
70
|
resolve({
|
|
@@ -13,6 +13,29 @@ export function parseNpmPackDryRun(stdout) {
|
|
|
13
13
|
if (!stdout.trim()) {
|
|
14
14
|
return { files: [], parseError: "npm pack --dry-run produced no output" };
|
|
15
15
|
}
|
|
16
|
+
if (/^[\[{]/.test(stdout.trim())) {
|
|
17
|
+
try {
|
|
18
|
+
const inventory = JSON.parse(stdout);
|
|
19
|
+
if (!Array.isArray(inventory) || inventory.length !== 1 ||
|
|
20
|
+
!inventory[0] || !Array.isArray(inventory[0].files) || !inventory[0].files.length) {
|
|
21
|
+
return { files: [], parseError: "npm pack JSON must contain exactly one nonempty package inventory" };
|
|
22
|
+
}
|
|
23
|
+
const files = [];
|
|
24
|
+
for (const file of inventory[0].files) {
|
|
25
|
+
if (!file || typeof file.path !== "string" || !safeInventoryPath(file.path)) {
|
|
26
|
+
return { files: [], parseError: "npm pack JSON contains a malformed or unsafe file path" };
|
|
27
|
+
}
|
|
28
|
+
// JSON paths are package-relative, unlike text tarball paths. Add the
|
|
29
|
+
// archive root exactly once so a real nested package/ directory cannot
|
|
30
|
+
// masquerade as a root entrypoint in the existing detectors.
|
|
31
|
+
files.push(`package/${file.path.replace(/\\/g, "/").replace(/^(\.\/)+/, "")}`);
|
|
32
|
+
}
|
|
33
|
+
return { files };
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return { files: [], parseError: "npm pack JSON is unparseable" };
|
|
37
|
+
}
|
|
38
|
+
}
|
|
16
39
|
const files = [];
|
|
17
40
|
let inContents = false;
|
|
18
41
|
let totalSize;
|
|
@@ -35,10 +58,8 @@ export function parseNpmPackDryRun(stdout) {
|
|
|
35
58
|
if (match?.[1]) {
|
|
36
59
|
files.push(match[1].trim());
|
|
37
60
|
}
|
|
38
|
-
else
|
|
39
|
-
|
|
40
|
-
files.push(line);
|
|
41
|
-
}
|
|
61
|
+
else
|
|
62
|
+
return { files: [], parseError: "Malformed npm tarball contents line" };
|
|
42
63
|
}
|
|
43
64
|
const totalMatch = line.match(/total files\s*:\s*(\d+)/i) ?? line.match(/unpacked size\s*:\s*([\d.]+\s*[kKmMgGbB]+)/i);
|
|
44
65
|
if (totalMatch?.[1]) {
|
|
@@ -52,5 +73,14 @@ export function parseNpmPackDryRun(stdout) {
|
|
|
52
73
|
parseError: "npm pack --dry-run: no files detected in tarball contents section",
|
|
53
74
|
};
|
|
54
75
|
}
|
|
76
|
+
if (files.some((file) => !safeInventoryPath(file))) {
|
|
77
|
+
return { files: [], parseError: "npm pack contains an unsafe file path" };
|
|
78
|
+
}
|
|
55
79
|
return { files, totalSize };
|
|
56
80
|
}
|
|
81
|
+
function safeInventoryPath(file) {
|
|
82
|
+
const normalized = file.replace(/\\/g, "/").replace(/^(\.\/)+/, "");
|
|
83
|
+
return Boolean(normalized) && !normalized.startsWith("/") &&
|
|
84
|
+
!/[\x00-\x1f\x7f:]/.test(normalized) &&
|
|
85
|
+
normalized.split("/").every((part) => Boolean(part) && part !== "." && part !== "..");
|
|
86
|
+
}
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
1
3
|
export const REQUIRED_PACKAGE_CONTENTS = [
|
|
2
4
|
"CHANGELOG.md",
|
|
3
5
|
"LICENSE",
|
|
@@ -20,15 +22,117 @@ export const REQUIRED_PACKAGE_CONTENTS = [
|
|
|
20
22
|
"examples/token-savings-cases.json",
|
|
21
23
|
];
|
|
22
24
|
function normalizeTarballPath(file) {
|
|
23
|
-
const normalized = file.replace(/\\/g, "/");
|
|
25
|
+
const normalized = file.replace(/\\/g, "/").replace(/^(\.\/)+/, "");
|
|
24
26
|
return normalized.startsWith("package/")
|
|
25
27
|
? normalized.slice("package/".length)
|
|
26
28
|
: normalized;
|
|
27
29
|
}
|
|
30
|
+
// Policy belongs beside the existing required-file detector. Identity is
|
|
31
|
+
// supplied by the existing target resolver, never inferred from package name.
|
|
32
|
+
export async function requiredContentsForTarget(target) {
|
|
33
|
+
const required = new Set(target.isSelf ? REQUIRED_PACKAGE_CONTENTS : ["package.json"]);
|
|
34
|
+
const findings = [];
|
|
35
|
+
function invalid(field, reason) {
|
|
36
|
+
findings.push({
|
|
37
|
+
id: `npm-pack-metadata-${findings.length}`,
|
|
38
|
+
title: `Package metadata cannot be validated: ${field}`,
|
|
39
|
+
severity: "blocker",
|
|
40
|
+
category: "package-content",
|
|
41
|
+
description: reason,
|
|
42
|
+
evidence: `package.json field: ${field}; ${reason}`,
|
|
43
|
+
affectedFiles: ["package.json"],
|
|
44
|
+
recommendation: "Provide readable package metadata with supported, safe package-relative entrypoint paths and rerun validation.",
|
|
45
|
+
releaseImpact: "Blocker: required package evidence is incomplete",
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
function entry(value, field, exportsPath = false) {
|
|
49
|
+
if (typeof value !== "string") {
|
|
50
|
+
invalid(field, "Malformed entrypoint: expected a nonempty string.");
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
const normalized = value.replace(/\\/g, "/");
|
|
54
|
+
const relative = normalized.replace(/^(\.\/)+/, "");
|
|
55
|
+
if (!relative || normalized !== normalized.trim() ||
|
|
56
|
+
/[\x00-\x1f\x7f:*?#%]/.test(normalized) || normalized.startsWith("/") ||
|
|
57
|
+
relative.split("/").some((part) => !part || part === ".." || part === "." || part === "node_modules") ||
|
|
58
|
+
(exportsPath && !normalized.startsWith("./"))) {
|
|
59
|
+
invalid(field, "Unsafe or unsupported entrypoint path (absolute, escaping, pattern, URL, or ambiguous path).");
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
required.add(relative);
|
|
63
|
+
}
|
|
64
|
+
try {
|
|
65
|
+
const metadata = JSON.parse(await fs.readFile(path.join(target.targetRoot, "package.json"), "utf8"));
|
|
66
|
+
if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
|
|
67
|
+
invalid("package.json", "Malformed package metadata: expected an object.");
|
|
68
|
+
}
|
|
69
|
+
else if (!target.isSelf) {
|
|
70
|
+
const pkg = metadata;
|
|
71
|
+
if ("bin" in pkg) {
|
|
72
|
+
if (typeof pkg.bin === "string")
|
|
73
|
+
entry(pkg.bin, "bin");
|
|
74
|
+
else if (pkg.bin && typeof pkg.bin === "object" && !Array.isArray(pkg.bin)) {
|
|
75
|
+
for (const key of Object.keys(pkg.bin).sort()) {
|
|
76
|
+
if (!key || key === "." || key === ".." || /[\\/:\x00-\x1f\x7f]/.test(key)) {
|
|
77
|
+
invalid("bin", "Unsafe or unsupported executable name in bin map.");
|
|
78
|
+
}
|
|
79
|
+
else
|
|
80
|
+
entry(pkg.bin[key], `bin.${key}`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
else
|
|
84
|
+
invalid("bin", "Malformed bin: expected a string or object map.");
|
|
85
|
+
}
|
|
86
|
+
if ("main" in pkg) {
|
|
87
|
+
if (typeof pkg.main === "string" && pkg.main && !path.posix.extname(pkg.main.replace(/\\/g, "/"))) {
|
|
88
|
+
invalid("main", "Unsupported or ambiguous main: extensionless or directory resolution is not validated; declare an exact runtime file.");
|
|
89
|
+
}
|
|
90
|
+
else
|
|
91
|
+
entry(pkg.main, "main");
|
|
92
|
+
}
|
|
93
|
+
if ("exports" in pkg && pkg.exports !== null) {
|
|
94
|
+
if (typeof pkg.exports === "string")
|
|
95
|
+
entry(pkg.exports, "exports", true);
|
|
96
|
+
else if (pkg.exports && typeof pkg.exports === "object" && !Array.isArray(pkg.exports)) {
|
|
97
|
+
const exports = pkg.exports;
|
|
98
|
+
const keys = Object.keys(exports).sort();
|
|
99
|
+
if (!keys.length || keys.some((key) => key !== "." && (!key.startsWith("./") || /[\\:*?#%\x00-\x1f\x7f]/.test(key) ||
|
|
100
|
+
key.slice(2).split("/").some((part) => !part || part === "." || part === "..")))) {
|
|
101
|
+
invalid("exports", "Unsupported or ambiguous exports: conditional or pattern mappings require additional evidence.");
|
|
102
|
+
}
|
|
103
|
+
else
|
|
104
|
+
for (const key of keys) {
|
|
105
|
+
if (exports[key] === null)
|
|
106
|
+
continue; // Explicitly excluded subpath.
|
|
107
|
+
if (typeof exports[key] !== "string") {
|
|
108
|
+
invalid(`exports.${key}`, "Unsupported exports shape: conditional objects and fallback arrays are not validated.");
|
|
109
|
+
}
|
|
110
|
+
else
|
|
111
|
+
entry(exports[key], `exports.${key}`, true);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
else
|
|
115
|
+
invalid("exports", "Unsupported exports shape: expected an exact string or subpath map.");
|
|
116
|
+
}
|
|
117
|
+
// npm automatically ships existing root README and license files. Do not
|
|
118
|
+
// invent a documentation requirement for packages without those files.
|
|
119
|
+
for (const file of (await fs.readdir(target.targetRoot, { withFileTypes: true })).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
120
|
+
if (file.isFile() && /^(readme|licen[cs]e)(\..*)?$/i.test(file.name))
|
|
121
|
+
required.add(file.name);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
invalid("package.json", "Package metadata is missing, unreadable, or unparseable.");
|
|
127
|
+
}
|
|
128
|
+
return { requiredFiles: [...required].sort(), findings };
|
|
129
|
+
}
|
|
28
130
|
export function detectMissingRequiredContents(options) {
|
|
29
131
|
const present = new Set(options.files.map(normalizeTarballPath));
|
|
30
132
|
const required = [...(options.requiredFiles ?? REQUIRED_PACKAGE_CONTENTS)]
|
|
31
|
-
.
|
|
133
|
+
// Requirements are package-relative declarations, not archive paths. A
|
|
134
|
+
// literal package/ directory in an entrypoint must not be stripped.
|
|
135
|
+
.map((file) => file.replace(/\\/g, "/").replace(/^(\.\/)+/, ""))
|
|
32
136
|
.sort((left, right) => left.localeCompare(right));
|
|
33
137
|
const missing = required.filter((file) => !present.has(file));
|
|
34
138
|
const findings = missing.map((file, index) => ({
|
|
@@ -3,7 +3,9 @@ import { runSecurityCommand, resolveNpmCommand } from "../commandRunner.js";
|
|
|
3
3
|
import { writeCheckResult } from "../artifacts.js";
|
|
4
4
|
import { parseNpmPackDryRun } from "./parseNpmPackDryRun.js";
|
|
5
5
|
import { detectForbiddenContents } from "./forbiddenPackageContents.js";
|
|
6
|
-
import { detectMissingRequiredContents } from "./requiredPackageContents.js";
|
|
6
|
+
import { detectMissingRequiredContents, requiredContentsForTarget } from "./requiredPackageContents.js";
|
|
7
|
+
import { resolveValidationTarget } from "../validate/resolveTarget.js";
|
|
8
|
+
import { discoverPackageRoot } from "../../runtime/packageRoot.js";
|
|
7
9
|
// Run npm pack --dry-run and inspect the resulting file list for forbidden contents.
|
|
8
10
|
// Does not publish anything.
|
|
9
11
|
export async function runPackageChecks(options) {
|
|
@@ -12,20 +14,23 @@ export async function runPackageChecks(options) {
|
|
|
12
14
|
const allFindings = [];
|
|
13
15
|
const checks = [];
|
|
14
16
|
const npm = resolveNpmCommand();
|
|
17
|
+
const target = options.target ?? resolveValidationTarget(path.resolve(cwd), discoverPackageRoot());
|
|
18
|
+
if (path.resolve(target.targetRoot) !== path.resolve(cwd)) {
|
|
19
|
+
throw new Error("Package check cwd does not match the resolved target root");
|
|
20
|
+
}
|
|
21
|
+
const policy = await requiredContentsForTarget(target);
|
|
15
22
|
// npm pack --dry-run to get the tarball file list
|
|
16
23
|
const startedAt = new Date().toISOString();
|
|
17
24
|
const cmd = await runSecurityCommand({
|
|
18
25
|
command: npm,
|
|
19
|
-
args: ["pack", "--dry-run"],
|
|
26
|
+
args: ["pack", "--dry-run", "--json", "--ignore-scripts", "--cache", path.resolve(rawOutputDir, "npm-cache")],
|
|
20
27
|
cwd,
|
|
21
28
|
timeoutMs: commandTimeoutMs,
|
|
22
29
|
});
|
|
23
30
|
const finishedAt = new Date().toISOString();
|
|
24
|
-
// npm
|
|
25
|
-
//
|
|
26
|
-
|
|
27
|
-
const packOutput = [cmd.stdout, cmd.stderr].find((stream) => /tarball contents/i.test(stream)) ??
|
|
28
|
-
[cmd.stdout, cmd.stderr].filter(Boolean).join("\n");
|
|
31
|
+
// Supported npm versions emit JSON on stdout. Lifecycle scripts are disabled
|
|
32
|
+
// so inspection cannot build, install, or rewrite the target.
|
|
33
|
+
const packOutput = cmd.stdout;
|
|
29
34
|
const parsed = parseNpmPackDryRun(packOutput);
|
|
30
35
|
const { findings: contentFindings } = detectForbiddenContents({
|
|
31
36
|
files: parsed.files,
|
|
@@ -36,20 +41,28 @@ export async function runPackageChecks(options) {
|
|
|
36
41
|
const { findings: missingRequiredFindings } = detectMissingRequiredContents({
|
|
37
42
|
files: parsed.files,
|
|
38
43
|
checkId: "npm-pack",
|
|
44
|
+
requiredFiles: policy.requiredFiles,
|
|
39
45
|
});
|
|
40
|
-
const
|
|
46
|
+
const evidenceFindings = [];
|
|
47
|
+
if (cmd.exitCode !== 0 || cmd.timedOut || cmd.skipped || parsed.parseError || parsed.files.length === 0) {
|
|
48
|
+
evidenceFindings.push({
|
|
49
|
+
id: "npm-pack-evidence-invalid",
|
|
50
|
+
title: "npm package inventory could not be validated",
|
|
51
|
+
severity: "blocker",
|
|
52
|
+
category: "package-content",
|
|
53
|
+
description: "A successful npm pack operation and a nonempty, parseable inventory are required.",
|
|
54
|
+
evidence: `exitCode=${String(cmd.exitCode)}; timedOut=${cmd.timedOut}; skipped=${cmd.skipped}; ${parsed.parseError ?? ""}; ${cmd.stderr.slice(0, 500)}`,
|
|
55
|
+
recommendation: "Resolve the package command or inventory failure and rerun validation.",
|
|
56
|
+
releaseImpact: "Blocker: required package evidence is unavailable",
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
const packageFindings = [...policy.findings, ...evidenceFindings, ...contentFindings, ...missingRequiredFindings];
|
|
41
60
|
allFindings.push(...packageFindings);
|
|
42
61
|
const packCheck = {
|
|
43
62
|
id: "npm-pack-dry-run",
|
|
44
63
|
name: "npm pack --dry-run (tarball file list)",
|
|
45
64
|
category: "package-content",
|
|
46
|
-
status:
|
|
47
|
-
? "failed"
|
|
48
|
-
: parsed.parseError && parsed.files.length === 0
|
|
49
|
-
? "warning"
|
|
50
|
-
: packageFindings.length > 0
|
|
51
|
-
? "failed"
|
|
52
|
-
: "passed",
|
|
65
|
+
status: packageFindings.length > 0 ? "failed" : "passed",
|
|
53
66
|
severity: packageFindings.length > 0
|
|
54
67
|
? packageFindings.some((f) => f.severity === "blocker")
|
|
55
68
|
? "blocker"
|
|
@@ -60,7 +73,9 @@ export async function runPackageChecks(options) {
|
|
|
60
73
|
durationMs: cmd.durationMs,
|
|
61
74
|
findings: packageFindings,
|
|
62
75
|
skippedReason: undefined,
|
|
63
|
-
command: "npm pack --dry-run",
|
|
76
|
+
command: "npm pack --dry-run --json --ignore-scripts",
|
|
77
|
+
commandCwd: cwd,
|
|
78
|
+
exitCode: cmd.exitCode,
|
|
64
79
|
};
|
|
65
80
|
await writeCheckResult({
|
|
66
81
|
result: packCheck,
|
|
@@ -5,16 +5,35 @@ import { resolveCommand } from "../../core/resolveCommand.js";
|
|
|
5
5
|
import { skippedCheck } from "../cliAdversarial/runAdversarialCheck.js";
|
|
6
6
|
export function parseSemgrepJson(raw) {
|
|
7
7
|
if (!raw.trim()) {
|
|
8
|
-
return { findings: [], rawOutput: raw };
|
|
8
|
+
return { findings: [], parseError: "Semgrep produced no JSON scan evidence", rawOutput: raw };
|
|
9
9
|
}
|
|
10
10
|
try {
|
|
11
11
|
const parsed = JSON.parse(raw);
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
if (!isRecord(parsed) || !Array.isArray(parsed.results)) {
|
|
13
|
+
return { findings: [], parseError: "Semgrep JSON requires a results array", rawOutput: raw };
|
|
14
|
+
}
|
|
15
|
+
const findings = [];
|
|
16
|
+
let parseError = !Array.isArray(parsed.errors) || !isRecord(parsed.paths) ||
|
|
17
|
+
!Array.isArray(parsed.paths.scanned) || parsed.paths.scanned.some((file) => typeof file !== "string")
|
|
18
|
+
? "Semgrep JSON requires errors and paths.scanned arrays" : undefined;
|
|
19
|
+
for (const [index, r] of parsed.results.entries()) {
|
|
20
|
+
if (!isRecord(r) || typeof r.check_id !== "string" || !r.check_id ||
|
|
21
|
+
typeof r.path !== "string" || !r.path || !isRecord(r.start) ||
|
|
22
|
+
!Number.isInteger(r.start.line) || Number(r.start.line) < 1 || !isRecord(r.extra) ||
|
|
23
|
+
(r.extra.message !== undefined && typeof r.extra.message !== "string") ||
|
|
24
|
+
(r.extra.metadata !== undefined && !isRecord(r.extra.metadata))) {
|
|
25
|
+
parseError ??= `Malformed Semgrep result at index ${index}`;
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
14
28
|
const item = r;
|
|
15
29
|
const checkId = typeof item["check_id"] === "string" ? item["check_id"] : "unknown";
|
|
16
30
|
const extra = item["extra"] ?? {};
|
|
17
31
|
const metadata = extra["metadata"] ?? {};
|
|
32
|
+
if ((extra.severity !== undefined && (typeof extra.severity !== "string" || !extra.severity.trim())) ||
|
|
33
|
+
(extra.severity === undefined && (typeof metadata.severity !== "string" || !metadata.severity.trim()))) {
|
|
34
|
+
parseError ??= `Missing or malformed Semgrep severity at index ${index}`;
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
18
37
|
const severity = typeof extra["severity"] === "string"
|
|
19
38
|
? extra["severity"]
|
|
20
39
|
: typeof metadata["severity"] === "string"
|
|
@@ -24,9 +43,14 @@ export function parseSemgrepJson(raw) {
|
|
|
24
43
|
const filePath = typeof item["path"] === "string" ? item["path"] : "";
|
|
25
44
|
const startObj = item["start"] ?? {};
|
|
26
45
|
const line = typeof startObj["line"] === "number" ? startObj["line"] : 0;
|
|
27
|
-
|
|
28
|
-
}
|
|
29
|
-
|
|
46
|
+
findings.push({ ruleId: checkId, severity, message, path: filePath, line });
|
|
47
|
+
}
|
|
48
|
+
// Semgrep's CLI contract reports partial/fatal analysis in errors even
|
|
49
|
+
// when the process exits zero. Retain matches, but never call this clean.
|
|
50
|
+
const analysisError = Array.isArray(parsed.errors) && parsed.errors.length > 0
|
|
51
|
+
? `Semgrep reported ${parsed.errors.length} analysis error(s): ${JSON.stringify(parsed.errors).slice(0, 500)}`
|
|
52
|
+
: undefined;
|
|
53
|
+
return { findings, parseError, analysisError, rawOutput: raw };
|
|
30
54
|
}
|
|
31
55
|
catch (err) {
|
|
32
56
|
return {
|
|
@@ -36,6 +60,9 @@ export function parseSemgrepJson(raw) {
|
|
|
36
60
|
};
|
|
37
61
|
}
|
|
38
62
|
}
|
|
63
|
+
function isRecord(value) {
|
|
64
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
65
|
+
}
|
|
39
66
|
function semgrepSeverityToSecurity(semgrepSeverity) {
|
|
40
67
|
switch (semgrepSeverity.toUpperCase()) {
|
|
41
68
|
case "ERROR":
|
|
@@ -88,67 +115,25 @@ export async function runSemgrepCheck(options) {
|
|
|
88
115
|
"--quiet",
|
|
89
116
|
scanTarget,
|
|
90
117
|
];
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
args: scanArgs,
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
118
|
+
let cmd;
|
|
119
|
+
try {
|
|
120
|
+
cmd = await runSecurityCommand({ command, args: scanArgs, cwd: targetRoot, timeoutMs });
|
|
121
|
+
}
|
|
122
|
+
catch (error) {
|
|
123
|
+
cmd = { command, args: scanArgs, cwd: targetRoot, exitCode: null, durationMs: 0,
|
|
124
|
+
stdout: "", stderr: error instanceof Error ? error.message : String(error), timedOut: false, skipped: false };
|
|
125
|
+
}
|
|
97
126
|
const finishedAt = new Date().toISOString();
|
|
98
|
-
//
|
|
99
|
-
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
status: "failed",
|
|
107
|
-
severity: "major",
|
|
108
|
-
startedAt,
|
|
109
|
-
finishedAt,
|
|
110
|
-
durationMs: cmd.durationMs,
|
|
111
|
-
findings: [
|
|
112
|
-
{
|
|
113
|
-
id: "semgrep-execution-error",
|
|
114
|
-
title: "Semgrep execution failed",
|
|
115
|
-
severity: "major",
|
|
116
|
-
category: "static-scan",
|
|
117
|
-
description: isTimedOut
|
|
118
|
-
? "Semgrep timed out."
|
|
119
|
-
: `Semgrep exited with code ${String(cmd.exitCode)}. stderr: ${cmd.stderr.slice(0, 500)}`,
|
|
120
|
-
recommendation: "Check semgrep installation and .semgrep.yml config.",
|
|
121
|
-
releaseImpact: "Review before release",
|
|
122
|
-
},
|
|
123
|
-
],
|
|
124
|
-
command: [command, ...scanArgs].join(" "),
|
|
125
|
-
};
|
|
127
|
+
// Distinguish positively established absence from a failed attempted scan.
|
|
128
|
+
// Network, permission, generic npm, or native spawn errors are not absence.
|
|
129
|
+
const unavailable = !cmd.timedOut && cmd.exitCode !== 0 && !cmd.stdout.trim() && ((cmd.skipped && cmd.exitCode === null && Boolean(cmd.skippedReason?.match(/\b(semgrep|scanner|cli)\b.*(unavailable|not installed|not found|not on PATH)/i))) ||
|
|
130
|
+
(useNpx && !cmd.skipped && cmd.exitCode === 1 && /could not determine executable to run/i.test(cmd.stderr)));
|
|
131
|
+
if (unavailable) {
|
|
132
|
+
return { ...skippedCheck({ id: "semgrep-scan", name: "Semgrep static analysis", category: "static-scan",
|
|
133
|
+
reason: cmd.skippedReason ?? "Semgrep CLI is unavailable; the npx fallback package provides no executable." }),
|
|
134
|
+
command: [command, ...scanArgs].join(" "), commandCwd: targetRoot, exitCode: cmd.exitCode };
|
|
126
135
|
}
|
|
127
136
|
const parsed = parseSemgrepJson(cmd.stdout);
|
|
128
|
-
if (parsed.parseError) {
|
|
129
|
-
return {
|
|
130
|
-
id: "semgrep-scan",
|
|
131
|
-
name: "Semgrep static analysis",
|
|
132
|
-
category: "static-scan",
|
|
133
|
-
status: "warning",
|
|
134
|
-
severity: "minor",
|
|
135
|
-
startedAt,
|
|
136
|
-
finishedAt,
|
|
137
|
-
durationMs: cmd.durationMs,
|
|
138
|
-
findings: [
|
|
139
|
-
{
|
|
140
|
-
id: "semgrep-parse-error",
|
|
141
|
-
title: "Could not parse Semgrep output",
|
|
142
|
-
severity: "minor",
|
|
143
|
-
category: "static-scan",
|
|
144
|
-
description: `JSON parse error: ${parsed.parseError}`,
|
|
145
|
-
recommendation: "Verify semgrep version produces valid JSON output.",
|
|
146
|
-
releaseImpact: "Review before release",
|
|
147
|
-
},
|
|
148
|
-
],
|
|
149
|
-
command: [command, ...scanArgs].join(" "),
|
|
150
|
-
};
|
|
151
|
-
}
|
|
152
137
|
const securityFindings = parsed.findings.map((f) => ({
|
|
153
138
|
id: `semgrep-${f.ruleId.replace(/[^a-z0-9-]/gi, "-")}`,
|
|
154
139
|
title: f.ruleId,
|
|
@@ -160,6 +145,23 @@ export async function runSemgrepCheck(options) {
|
|
|
160
145
|
? "Should fix before release"
|
|
161
146
|
: "Review before release",
|
|
162
147
|
}));
|
|
148
|
+
const executionFailed = cmd.timedOut || cmd.skipped || (cmd.exitCode !== 0 && cmd.exitCode !== 1) ||
|
|
149
|
+
(cmd.exitCode === 1 && parsed.findings.length === 0);
|
|
150
|
+
const evidenceError = executionFailed
|
|
151
|
+
? `Semgrep execution failed: exitCode=${String(cmd.exitCode)}, timedOut=${cmd.timedOut}, skipped=${cmd.skipped}. ${cmd.stderr.slice(0, 500)}`
|
|
152
|
+
: parsed.parseError ?? parsed.analysisError;
|
|
153
|
+
if (evidenceError) {
|
|
154
|
+
securityFindings.push({
|
|
155
|
+
id: executionFailed ? "semgrep-execution-error" : parsed.parseError ? "semgrep-parse-error" : "semgrep-analysis-error",
|
|
156
|
+
title: "Semgrep scan evidence is incomplete or unsuccessful",
|
|
157
|
+
severity: "major",
|
|
158
|
+
category: "static-scan",
|
|
159
|
+
description: evidenceError,
|
|
160
|
+
evidence: `exitCode=${String(cmd.exitCode)}; ${parsed.parseError ?? parsed.analysisError ?? ""}`,
|
|
161
|
+
recommendation: "Resolve the scanner execution or analysis error and rerun with valid JSON scan evidence.",
|
|
162
|
+
releaseImpact: "Review before release",
|
|
163
|
+
});
|
|
164
|
+
}
|
|
163
165
|
const hasBlockerOrMajor = securityFindings.some((f) => f.severity === "blocker" || f.severity === "major");
|
|
164
166
|
return {
|
|
165
167
|
id: "semgrep-scan",
|
|
@@ -176,5 +178,7 @@ export async function runSemgrepCheck(options) {
|
|
|
176
178
|
durationMs: cmd.durationMs,
|
|
177
179
|
findings: securityFindings,
|
|
178
180
|
command: [command, ...scanArgs].join(" "),
|
|
181
|
+
commandCwd: targetRoot,
|
|
182
|
+
exitCode: cmd.exitCode,
|
|
179
183
|
};
|
|
180
184
|
}
|
|
@@ -1,7 +1,3 @@
|
|
|
1
|
-
// Core types for the my-dev-kit-lab security-validation framework.
|
|
2
|
-
// These types describe the vocabulary for checks, findings, results, and verdicts
|
|
3
|
-
// used by dependency checks (Prompt 3), CLI adversarial tests (Prompts 4–5),
|
|
4
|
-
// static scans (Prompt 6), fuzz smoke tests (Prompt 7), and the release report (Prompt 8).
|
|
5
1
|
export const SECURITY_SEVERITIES = [
|
|
6
2
|
"blocker",
|
|
7
3
|
"major",
|