@dailephd/my-dev-kit-lab 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +35 -24
- package/dist/scripts/security/validate.js +68 -29
- package/dist/src/securityValidation/attackScenarios/attackProfile.js +20 -0
- package/dist/src/securityValidation/attackScenarios/attackResult.js +37 -0
- package/dist/src/securityValidation/attackScenarios/attackRunner.js +153 -0
- package/dist/src/securityValidation/attackScenarios/attackScenario.js +21 -0
- package/dist/src/securityValidation/attackScenarios/boundedSourceScan.js +100 -0
- package/dist/src/securityValidation/attackScenarios/exploitEvidence.js +69 -0
- package/dist/src/securityValidation/attackScenarios/payloadCorpus.js +0 -0
- package/dist/src/securityValidation/attackScenarios/profiles/localToolProfile.js +10 -0
- package/dist/src/securityValidation/attackScenarios/profiles/nodeCliPackageProfile.js +20 -0
- package/dist/src/securityValidation/attackScenarios/profiles/npmPackageProfile.js +10 -0
- package/dist/src/securityValidation/attackScenarios/reportSchemaGuard.js +41 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/configInjectionScenario.js +144 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/networkAssumptionScenario.js +129 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/outputBoundaryScenario.js +69 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/packageBoundaryScenario.js +118 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/pathTraversalScenario.js +119 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/reportPoisoningScenario.js +176 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/secretLeakageScenario.js +143 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/subprocessInjectionScenario.js +117 -0
- package/dist/src/securityValidation/attackScenarios/scenarios/targetSandboxScenario.js +118 -0
- package/dist/src/securityValidation/attackScenarios/targetSnapshot.js +70 -0
- package/dist/src/securityValidation/packageChecks/forbiddenPackageContents.js +4 -1
- package/dist/src/securityValidation/report/renderSecurityReport.js +135 -13
- package/dist/src/securityValidation/types.js +7 -0
- package/dist/src/securityValidation/validate/cliOptions.js +182 -0
- package/dist/src/securityValidation/validate/runSecurityValidation.js +134 -35
- package/dist/src/securityValidation/validate/verdict.js +147 -0
- package/package.json +16 -16
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 dailephd LLC
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# my-dev-kit-lab
|
|
2
2
|
|
|
3
|
-
my-dev-kit-lab is the experiment, evidence, reporting, and
|
|
3
|
+
my-dev-kit-lab is the experiment, evidence, reporting, security-validation, and future audit companion for my-dev-kit. It runs reproducible experiments that test whether my-dev-kit's graph-guided retrieval helps coding-agent workflows, collects metrics, renders reports, generates plots, captures screenshots, builds gallery outputs, and performs automated CLI/package security validation.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
The current package is v0.2.2, release-prepared but not yet published to npm. Its generic experiment-plugin framework was introduced in v0.2.0. The first plugin is `context-strategy-comparison`, which preserves the existing raw-full-file vs my-dev-kit-guided workflow through the plugin runner.
|
|
6
|
+
|
|
7
|
+
my-dev-kit is most useful when the repository is larger than the task. It helps coding agents work with large codebases through reusable structural indexing, graph-guided retrieval, targeted source slices, and auditable context selection. Results are scoped evidence; the project does not claim that my-dev-kit always saves tokens.
|
|
6
8
|
|
|
7
9
|
**my-dev-kit** is the repo indexing and graph-guided retrieval engine.
|
|
8
10
|
**my-dev-kit-lab** is the separate lab layer that feeds it benchmark inputs and records evaluation outputs.
|
|
@@ -29,7 +31,7 @@ As of v0.2.1, my-dev-kit-lab also exposes a generic experiment-plugin framework
|
|
|
29
31
|
- First experiment plugin: `context-strategy-comparison`
|
|
30
32
|
- Target-aware experiment execution for local projects via `experiment:run -- --target <path>`
|
|
31
33
|
- Plugin-aware JSON and HTML reports with plugin, target, variant, metric, artifact, warning, skip, and failure metadata
|
|
32
|
-
- Security validation framework: dependency audit, package tarball inspection, CLI adversarial tests, static scans (CodeQL/Semgrep), bounded fuzz smoke, and
|
|
34
|
+
- Security validation framework: dependency audit, package tarball inspection, CLI adversarial tests, static scans (CodeQL/Semgrep), bounded fuzz smoke, attack-scenario checks, and structured verdict/report output — runnable against any local project via `security:validate --target <path>`
|
|
33
35
|
|
|
34
36
|
---
|
|
35
37
|
|
|
@@ -37,19 +39,14 @@ As of v0.2.1, my-dev-kit-lab also exposes a generic experiment-plugin framework
|
|
|
37
39
|
|
|
38
40
|
```mermaid
|
|
39
41
|
flowchart TD
|
|
40
|
-
A[
|
|
41
|
-
B --> C[
|
|
42
|
-
C --> D[Controlled
|
|
43
|
-
D --> E[
|
|
44
|
-
E --> F[
|
|
45
|
-
E --> G[
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
G --> J[SVG Charts]
|
|
49
|
-
H --> K[Demo Artifacts]
|
|
50
|
-
I --> L[Gallery]
|
|
51
|
-
J --> L
|
|
52
|
-
K --> L
|
|
42
|
+
A[scripts/experiments] --> B[Experiment plugin registry + runner]
|
|
43
|
+
B --> C[context-strategy-comparison]
|
|
44
|
+
C --> D[Controlled experiment foundations]
|
|
45
|
+
D --> E[Plugin + legacy artifacts]
|
|
46
|
+
E --> F[Plugin-aware reports]
|
|
47
|
+
E --> G[Plots / screenshots / gallery]
|
|
48
|
+
H[scripts/security] --> I[Automated security validation]
|
|
49
|
+
I --> J[Security reports + verdict]
|
|
53
50
|
```
|
|
54
51
|
|
|
55
52
|
---
|
|
@@ -205,7 +202,7 @@ See [docs/METRICS.md](docs/METRICS.md) for full metric definitions.
|
|
|
205
202
|
|
|
206
203
|
## Current baseline release positioning
|
|
207
204
|
|
|
208
|
-
my-dev-kit-lab is at a working baseline. The raw-vs-indexed experiment pipeline is fully implemented and produces reproducible artifacts. Real-agent campaign support exists for Codex and Claude. v0.2.
|
|
205
|
+
my-dev-kit-lab is at a working baseline. The raw-vs-indexed experiment pipeline is fully implemented and produces reproducible artifacts. Real-agent campaign support exists for Codex and Claude. The current package version is `v0.2.2`, which implements the automated security-validation fortification work described below and has passed pre-release readiness and cross-platform validation. `v0.2.2` has not yet been published to npm.
|
|
209
206
|
|
|
210
207
|
---
|
|
211
208
|
|
|
@@ -223,7 +220,7 @@ This is not a web application pentest framework. **my-dev-kit** is a local CLI/p
|
|
|
223
220
|
- database-free
|
|
224
221
|
- safe to run on local repositories
|
|
225
222
|
|
|
226
|
-
The
|
|
223
|
+
The automated validation gate is implemented. It combines static scans, dependency/package checks, adversarial CLI tests, bounded fuzz smoke tests, and attack-scenario checks with structured text/JSON reports and a four-category verdict.
|
|
227
224
|
|
|
228
225
|
### Security commands
|
|
229
226
|
|
|
@@ -233,15 +230,23 @@ The release gate is implemented as of v0.1.4. It combines static scans, dependen
|
|
|
233
230
|
| `npm run security:package` | npm pack --dry-run, forbidden content detection |
|
|
234
231
|
| `npm run security:codeql` | CodeQL CLI availability check; skipped gracefully when absent |
|
|
235
232
|
| `npm run security:semgrep` | Semgrep scan via local binary or npx; skipped gracefully when both absent |
|
|
236
|
-
| `npm run test:security` |
|
|
233
|
+
| `npm run test:security` | Automated security and adversarial CLI tests (path traversal, read-only boundaries, malformed artifacts, JSON safety, and related checks) |
|
|
237
234
|
| `npm run test:fuzz:smoke` | 9 bounded fuzz targets, seeded PRNG, completes in under 1 second |
|
|
238
|
-
| `npm run security:validate` |
|
|
235
|
+
| `npm run security:validate` | Runs selected security checks, applies profile-aware defaults when requested, and writes text/JSON reports according to `--format` |
|
|
236
|
+
|
|
237
|
+
CodeQL, Semgrep, and OSV-Scanner are optional. When unavailable locally, they are recorded as `skipped` in the report, not as passed. That can lead to `ready except optional manual checks`; it does not silently turn missing tooling into a clean pass.
|
|
238
|
+
|
|
239
|
+
Each security command can validate my-dev-kit-lab itself or another local project via `--target <path>`. When `--target` is omitted, the framework performs self-validation. Target projects are inspected in place: their source files are not modified by default, generated artifacts stay under `reports/security/` unless `--out` is used, and external-target reports identify the tool root, target root, target package metadata, target git metadata, command cwd, exit code, and stdout/stderr summaries.
|
|
240
|
+
|
|
241
|
+
`security:validate` supports `--checks`, `--profile`, `--format`, `--fail-on`, `--out`, and `--report-prefix`. The no-flag path remains backward compatible and still runs the classic implemented check groups: `deps`, `package`, `static`, `cli-adversarial`, and `fuzz`. Supplying `--profile` without `--checks` swaps in that profile's default checks; supplying explicit `--checks` always wins over profile defaults.
|
|
242
|
+
|
|
243
|
+
The current attack-scenario checks cover boundary, subprocess, secrets, and network assumptions. They are automated adversarial checks, not a manual pentest. `verdictImpact` metadata from each registered scenario drives blocker categorization in report reasoning, and narrowed `--checks` runs are labeled as scoped rather than described as a full release gate.
|
|
239
244
|
|
|
240
|
-
|
|
245
|
+
JSON report poisoning/config-injection checks use a baseline-diff schema guard so legitimate additive JSON fields do not fail the guard while payload-created trusted top-level fields still do. Text reports are sanitized to strip ANSI/control-byte payloads before rendering.
|
|
241
246
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
For external-target validation, `security:validate` reads the target `package.json`, detects `scripts.test:security`, and runs `npm run test:security` in the target project root when that script exists. This behavior is validated both from the source checkout and from an installed packed tarball because published-package execution differs from local source execution.
|
|
247
|
+
Known limits remain intentionally explicit: secret scanning is bounded rather than exhaustive, the network/local-first check is a bounded static assumption check rather than proof of runtime isolation, package-boundary severity is still result-level rather than per-evidence-item, and profile-specific scenario behavior is limited to profile-based scenario selection/default checks.
|
|
248
|
+
|
|
249
|
+
For external-target validation, `security:validate` reads the target `package.json`, detects `scripts.test:security`, and runs `npm run test:security` in the target project root when that script exists. This behavior is validated both from the source checkout and from an installed packed tarball because published-package execution differs from local source execution.
|
|
245
250
|
|
|
246
251
|
Generated security reports under `reports/security/` are excluded from git by default. They are produced locally or in CI as release-gate evidence and are not committed to the repository.
|
|
247
252
|
|
|
@@ -249,6 +254,12 @@ See [docs/COMMANDS.md](docs/COMMANDS.md) for full command options and [docs/secu
|
|
|
249
254
|
|
|
250
255
|
---
|
|
251
256
|
|
|
257
|
+
## License
|
|
258
|
+
|
|
259
|
+
MIT License. See [LICENSE](LICENSE) for the full text.
|
|
260
|
+
|
|
261
|
+
---
|
|
262
|
+
|
|
252
263
|
## Support
|
|
253
264
|
|
|
254
265
|
my-dev-kit-lab is an independent project by dailephd LLC, developed and maintained by Dai Le.
|
|
@@ -4,11 +4,34 @@ import fs from "node:fs";
|
|
|
4
4
|
import { runSecurityValidation } from "../../src/securityValidation/validate/runSecurityValidation.js";
|
|
5
5
|
import { resolveValidationTarget, reportFilenamePrefix } from "../../src/securityValidation/validate/resolveTarget.js";
|
|
6
6
|
import { renderTextReport, renderJsonReport } from "../../src/securityValidation/report/renderSecurityReport.js";
|
|
7
|
+
import { parseSecurityValidateArgs, normalizeSecurityValidateConfig, applyProfileDefaultChecksIfApplicable, } from "../../src/securityValidation/validate/cliOptions.js";
|
|
8
|
+
import { resolveAttackProfile } from "../../src/securityValidation/attackScenarios/attackProfile.js";
|
|
9
|
+
import { findingsBreachFailOnThreshold } from "../../src/securityValidation/validate/verdict.js";
|
|
7
10
|
import { resolveToolRoot } from "./resolveToolRoot.js";
|
|
11
|
+
const USAGE = "Usage: npm run security:validate -- [--target <path>] [--out <dir>] [--report-prefix <name>] " +
|
|
12
|
+
"[--checks deps,package,static,cli-adversarial,fuzz,boundary,subprocess,secrets,network] " +
|
|
13
|
+
"[--profile node-cli-package|local-tool|npm-package] [--format text,json] [--fail-on blocker|high|medium|low]";
|
|
8
14
|
// Parse CLI arguments from process.argv (after the node/tsx and script path).
|
|
9
15
|
const rawArgs = process.argv.slice(2);
|
|
10
|
-
const args = parseArgs(rawArgs);
|
|
11
16
|
const toolRoot = resolveToolRoot(import.meta.url);
|
|
17
|
+
let args;
|
|
18
|
+
let config;
|
|
19
|
+
try {
|
|
20
|
+
args = parseSecurityValidateArgs(rawArgs);
|
|
21
|
+
config = normalizeSecurityValidateConfig(args, toolRoot);
|
|
22
|
+
// v0.2.2 Batch 5: --profile with no --checks uses that profile's declared
|
|
23
|
+
// default checks. No-op unless the user passed --profile without --checks
|
|
24
|
+
// (see applyProfileDefaultChecksIfApplicable's guard) — the true no-flag
|
|
25
|
+
// case and any explicit --checks case are always unaffected.
|
|
26
|
+
config = applyProfileDefaultChecksIfApplicable(config, resolveAttackProfile(config.profile).defaultCheckIds);
|
|
27
|
+
}
|
|
28
|
+
catch (err) {
|
|
29
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
30
|
+
console.error(`\nERROR: ${msg}`);
|
|
31
|
+
console.error(USAGE);
|
|
32
|
+
process.exitCode = 1;
|
|
33
|
+
process.exit(1);
|
|
34
|
+
}
|
|
12
35
|
// Resolve and validate target early so we can fail fast with a clean error.
|
|
13
36
|
let target;
|
|
14
37
|
try {
|
|
@@ -17,7 +40,7 @@ try {
|
|
|
17
40
|
catch (err) {
|
|
18
41
|
const msg = err instanceof Error ? err.message : String(err);
|
|
19
42
|
console.error(`\nERROR: ${msg}`);
|
|
20
|
-
console.error(
|
|
43
|
+
console.error(USAGE);
|
|
21
44
|
process.exitCode = 1;
|
|
22
45
|
process.exit(1);
|
|
23
46
|
}
|
|
@@ -33,13 +56,30 @@ if (!target.isSelf) {
|
|
|
33
56
|
else {
|
|
34
57
|
console.log(`Mode : self-validation`);
|
|
35
58
|
}
|
|
59
|
+
if (!config.checksWereDefault) {
|
|
60
|
+
console.log(`Checks : ${config.checks.join(", ")}`);
|
|
61
|
+
}
|
|
62
|
+
if (config.plannedChecksRequested.length > 0) {
|
|
63
|
+
console.log(`Note : ${config.plannedChecksRequested.join(", ")} are routed through the attack-scenario framework rather than the classic check-group runners. Review the attack-scenario section in the report for their concrete results.`);
|
|
64
|
+
}
|
|
36
65
|
console.log("");
|
|
37
66
|
const summary = await runSecurityValidation({
|
|
38
67
|
cwd: toolRoot,
|
|
39
68
|
targetPath: args.target,
|
|
40
69
|
fuzzIterations: parseInt(process.env["FUZZ_ITERATIONS"] ?? "50", 10),
|
|
41
70
|
fuzzSeed: parseInt(process.env["FUZZ_SEED"] ?? "0xDEADBEEF", 16),
|
|
71
|
+
// Full selection (implemented + planned/attack-scenario ids) — the runner
|
|
72
|
+
// gates both implemented check groups and attack-scenario checks off the
|
|
73
|
+
// same selectedChecks set.
|
|
74
|
+
selectedChecks: config.checks,
|
|
75
|
+
profile: config.profile,
|
|
42
76
|
});
|
|
77
|
+
// v0.2.2 Batch 5: --fail-on breach is additive to (never a substitute for)
|
|
78
|
+
// the existing verdict-based exit decision below — it can only escalate
|
|
79
|
+
// exit 0 to exit 1 (e.g. --fail-on medium/low catching minor/informational
|
|
80
|
+
// findings the verdict policy doesn't already treat as blocking), never
|
|
81
|
+
// downgrade an existing blocker.
|
|
82
|
+
const failOnBreached = findingsBreachFailOnThreshold(summary.findings, config.failOnThreshold);
|
|
43
83
|
// Build report object
|
|
44
84
|
const report = {
|
|
45
85
|
metadata: {
|
|
@@ -55,19 +95,25 @@ const report = {
|
|
|
55
95
|
isSelf: summary.isSelf,
|
|
56
96
|
generatedAt: summary.finishedAt,
|
|
57
97
|
totalDurationMs: new Date(summary.finishedAt).getTime() - new Date(summary.startedAt).getTime(),
|
|
98
|
+
profile: config.profile,
|
|
99
|
+
selectedChecks: config.checks,
|
|
100
|
+
failOnThreshold: config.failOnThreshold,
|
|
101
|
+
formats: config.formats,
|
|
102
|
+
failOnBreached,
|
|
103
|
+
isFullReleaseGate: summary.isFullReleaseGate,
|
|
58
104
|
},
|
|
59
105
|
sections: [],
|
|
60
106
|
allChecks: summary.checks,
|
|
61
107
|
allFindings: summary.findings,
|
|
62
108
|
verdict: summary.verdict,
|
|
63
109
|
recommendedNextStep: summary.recommendedNextStep,
|
|
110
|
+
attackResults: summary.attackResults,
|
|
111
|
+
verdictReasonSummary: summary.verdictReasonSummary,
|
|
64
112
|
};
|
|
65
113
|
const textReport = renderTextReport(report);
|
|
66
114
|
const jsonReport = renderJsonReport(report);
|
|
67
115
|
// Determine output directory
|
|
68
|
-
const reportsDir =
|
|
69
|
-
? path.resolve(args.out)
|
|
70
|
-
: path.join(toolRoot, "reports", "security");
|
|
116
|
+
const reportsDir = config.out;
|
|
71
117
|
if (!fs.existsSync(reportsDir)) {
|
|
72
118
|
fs.mkdirSync(reportsDir, { recursive: true });
|
|
73
119
|
}
|
|
@@ -75,14 +121,22 @@ if (!fs.existsSync(reportsDir)) {
|
|
|
75
121
|
const prefix = args.reportPrefix ?? reportFilenamePrefix(target);
|
|
76
122
|
const txtPath = path.join(reportsDir, `${prefix}-security-validation.txt`);
|
|
77
123
|
const jsonPath = path.join(reportsDir, `${prefix}-security-validation.json`);
|
|
78
|
-
|
|
79
|
-
|
|
124
|
+
const writtenPaths = [];
|
|
125
|
+
if (config.formats.includes("text")) {
|
|
126
|
+
fs.writeFileSync(txtPath, textReport, "utf8");
|
|
127
|
+
writtenPaths.push(txtPath);
|
|
128
|
+
}
|
|
129
|
+
if (config.formats.includes("json")) {
|
|
130
|
+
fs.writeFileSync(jsonPath, jsonReport, "utf8");
|
|
131
|
+
writtenPaths.push(jsonPath);
|
|
132
|
+
}
|
|
80
133
|
// Print report to stdout
|
|
81
134
|
console.log(textReport);
|
|
82
135
|
console.log(`\nReports written:`);
|
|
83
|
-
|
|
84
|
-
console.log(` ${
|
|
85
|
-
|
|
136
|
+
for (const p of writtenPaths) {
|
|
137
|
+
console.log(` ${p}`);
|
|
138
|
+
}
|
|
139
|
+
// Exit code based on verdict, escalated (never downgraded) by --fail-on.
|
|
86
140
|
const blockerExists = summary.verdict === "not-ready-security-blocker-remains";
|
|
87
141
|
const inconclusive = summary.verdict === "inconclusive-audit-environment-incomplete";
|
|
88
142
|
if (blockerExists) {
|
|
@@ -93,26 +147,11 @@ else if (inconclusive) {
|
|
|
93
147
|
console.warn("\nExit 2 — audit environment incomplete.");
|
|
94
148
|
process.exitCode = 2;
|
|
95
149
|
}
|
|
150
|
+
else if (failOnBreached) {
|
|
151
|
+
console.error(`\nExit 1 — --fail-on ${config.failOnThreshold} threshold breached.`);
|
|
152
|
+
process.exitCode = 1;
|
|
153
|
+
}
|
|
96
154
|
else {
|
|
97
155
|
console.log("\nExit 0 — validation completed.");
|
|
98
156
|
process.exitCode = 0;
|
|
99
157
|
}
|
|
100
|
-
// ---------------------------------------------------------------------------
|
|
101
|
-
// Argument parser
|
|
102
|
-
// ---------------------------------------------------------------------------
|
|
103
|
-
function parseArgs(argv) {
|
|
104
|
-
const result = {};
|
|
105
|
-
for (let i = 0; i < argv.length; i++) {
|
|
106
|
-
const arg = argv[i];
|
|
107
|
-
if ((arg === "--target" || arg === "-t") && i + 1 < argv.length) {
|
|
108
|
-
result.target = argv[++i];
|
|
109
|
-
}
|
|
110
|
-
else if (arg === "--out" && i + 1 < argv.length) {
|
|
111
|
-
result.out = argv[++i];
|
|
112
|
-
}
|
|
113
|
-
else if (arg === "--report-prefix" && i + 1 < argv.length) {
|
|
114
|
-
result.reportPrefix = argv[++i];
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
return result;
|
|
118
|
-
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { SECURITY_PROFILE_IDS } from "../validate/cliOptions.js";
|
|
2
|
+
import { NODE_CLI_PACKAGE_PROFILE } from "./profiles/nodeCliPackageProfile.js";
|
|
3
|
+
import { LOCAL_TOOL_PROFILE } from "./profiles/localToolProfile.js";
|
|
4
|
+
import { NPM_PACKAGE_PROFILE } from "./profiles/npmPackageProfile.js";
|
|
5
|
+
const PROFILE_REGISTRY = {
|
|
6
|
+
"node-cli-package": NODE_CLI_PACKAGE_PROFILE,
|
|
7
|
+
"local-tool": LOCAL_TOOL_PROFILE,
|
|
8
|
+
"npm-package": NPM_PACKAGE_PROFILE,
|
|
9
|
+
};
|
|
10
|
+
// Resolves a profile id to its definition. Throws a clean, descriptive Error
|
|
11
|
+
// for unknown ids (mirrors Batch 1's --profile validation error style).
|
|
12
|
+
export function resolveAttackProfile(id) {
|
|
13
|
+
if (!SECURITY_PROFILE_IDS.includes(id)) {
|
|
14
|
+
throw new Error(`Unknown security profile: "${id}". Valid values are: ${SECURITY_PROFILE_IDS.join(", ")}.`);
|
|
15
|
+
}
|
|
16
|
+
return PROFILE_REGISTRY[id];
|
|
17
|
+
}
|
|
18
|
+
export function listAttackProfiles() {
|
|
19
|
+
return SECURITY_PROFILE_IDS.map((id) => PROFILE_REGISTRY[id]);
|
|
20
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// Bridges an AttackResult into the existing SecurityCheckResult shape so it
|
|
2
|
+
// can flow through the existing verdict/report pipeline without redesigning
|
|
3
|
+
// either. "blocked" (scenario crashed) maps to a failed check with severity
|
|
4
|
+
// bumped to at least "major" so a scenario crash is never silently invisible.
|
|
5
|
+
export function toSecurityCheckResult(result) {
|
|
6
|
+
const now = new Date().toISOString();
|
|
7
|
+
const status = result.status === "blocked" ? "failed" : result.status;
|
|
8
|
+
const severity = result.status === "blocked" && result.severity === "skipped" ? "major" : result.severity;
|
|
9
|
+
return {
|
|
10
|
+
id: result.scenarioId,
|
|
11
|
+
name: result.scenarioTitle,
|
|
12
|
+
category: result.category,
|
|
13
|
+
status,
|
|
14
|
+
severity,
|
|
15
|
+
startedAt: now,
|
|
16
|
+
finishedAt: now,
|
|
17
|
+
durationMs: 0,
|
|
18
|
+
findings: result.status === "failed" || result.status === "blocked"
|
|
19
|
+
? [
|
|
20
|
+
{
|
|
21
|
+
id: `${result.scenarioId}-finding`,
|
|
22
|
+
title: result.scenarioTitle,
|
|
23
|
+
severity,
|
|
24
|
+
category: result.category,
|
|
25
|
+
description: result.status === "blocked"
|
|
26
|
+
? `Scenario execution error: ${result.errorSummary ?? "unknown error"}`
|
|
27
|
+
: `Attack scenario detected unsafe behavior for check group '${result.checkId}'.`,
|
|
28
|
+
evidence: result.evidence.map((e) => e.redactedPreview).filter(Boolean).join(" | ") || undefined,
|
|
29
|
+
recommendation: result.recommendation,
|
|
30
|
+
releaseImpact: severity === "blocker" ? "Must fix before release" : "Should fix before release",
|
|
31
|
+
},
|
|
32
|
+
]
|
|
33
|
+
: [],
|
|
34
|
+
skippedReason: result.skippedReason,
|
|
35
|
+
verdictImpact: result.verdictImpact,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { CHECK_ID_CATEGORY } from "./attackScenario.js";
|
|
2
|
+
import { TARGET_SANDBOX_SCENARIO } from "./scenarios/targetSandboxScenario.js";
|
|
3
|
+
import { PACKAGE_BOUNDARY_SCENARIO } from "./scenarios/packageBoundaryScenario.js";
|
|
4
|
+
import { OUTPUT_BOUNDARY_SCENARIO } from "./scenarios/outputBoundaryScenario.js";
|
|
5
|
+
import { PATH_TRAVERSAL_SCENARIO } from "./scenarios/pathTraversalScenario.js";
|
|
6
|
+
import { CONFIG_INJECTION_SCENARIO } from "./scenarios/configInjectionScenario.js";
|
|
7
|
+
import { SUBPROCESS_INJECTION_SCENARIO } from "./scenarios/subprocessInjectionScenario.js";
|
|
8
|
+
import { SECRET_LEAKAGE_SCENARIO } from "./scenarios/secretLeakageScenario.js";
|
|
9
|
+
import { REPORT_POISONING_SCENARIO } from "./scenarios/reportPoisoningScenario.js";
|
|
10
|
+
import { NETWORK_ASSUMPTION_SCENARIO } from "./scenarios/networkAssumptionScenario.js";
|
|
11
|
+
import { registerScenarioPayloadGroups } from "./payloadCorpus.js";
|
|
12
|
+
registerScenarioPayloadGroups(OUTPUT_BOUNDARY_SCENARIO.id, ["output-boundary"]);
|
|
13
|
+
registerScenarioPayloadGroups(PATH_TRAVERSAL_SCENARIO.id, ["path-traversal"]);
|
|
14
|
+
registerScenarioPayloadGroups(CONFIG_INJECTION_SCENARIO.id, ["malformed-config", "report-poisoning"]);
|
|
15
|
+
registerScenarioPayloadGroups(SUBPROCESS_INJECTION_SCENARIO.id, ["subprocess-injection"]);
|
|
16
|
+
registerScenarioPayloadGroups(REPORT_POISONING_SCENARIO.id, ["report-poisoning"]);
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// v0.2.2 Batch 2/3/4 — attack scenario runner.
|
|
19
|
+
//
|
|
20
|
+
// SCENARIO_REGISTRY was empty in Batch 2. Batch 3 registered the first
|
|
21
|
+
// concrete scenarios (boundary: target sandbox, package boundary, output
|
|
22
|
+
// boundary, path traversal, config injection; subprocess: subprocess
|
|
23
|
+
// injection). Batch 4 adds secret leakage (secrets), report poisoning
|
|
24
|
+
// (boundary — report-rendering safety is a boundary-of-trust concern), and
|
|
25
|
+
// network/local-first assumption (network). All 9 --checks ids now have at
|
|
26
|
+
// least one registered scenario.
|
|
27
|
+
// ---------------------------------------------------------------------------
|
|
28
|
+
export const SCENARIO_REGISTRY = [
|
|
29
|
+
TARGET_SANDBOX_SCENARIO,
|
|
30
|
+
PACKAGE_BOUNDARY_SCENARIO,
|
|
31
|
+
OUTPUT_BOUNDARY_SCENARIO,
|
|
32
|
+
PATH_TRAVERSAL_SCENARIO,
|
|
33
|
+
CONFIG_INJECTION_SCENARIO,
|
|
34
|
+
REPORT_POISONING_SCENARIO,
|
|
35
|
+
SUBPROCESS_INJECTION_SCENARIO,
|
|
36
|
+
SECRET_LEAKAGE_SCENARIO,
|
|
37
|
+
NETWORK_ASSUMPTION_SCENARIO,
|
|
38
|
+
];
|
|
39
|
+
const DEFAULT_SCENARIO_TIMEOUT_MS = 30_000;
|
|
40
|
+
// Selects scenarios whose checkId is in selectedChecks and whose
|
|
41
|
+
// applicableProfiles either is empty (applies to all profiles) or includes
|
|
42
|
+
// the given profile. Preserves registry order (deterministic).
|
|
43
|
+
export function filterScenarios(scenarios, selectedChecks, profile) {
|
|
44
|
+
const checkSet = new Set(selectedChecks);
|
|
45
|
+
return scenarios.filter((s) => checkSet.has(s.checkId) && (s.applicableProfiles.length === 0 || s.applicableProfiles.includes(profile)));
|
|
46
|
+
}
|
|
47
|
+
async function withTimeout(promise, timeoutMs, scenarioId) {
|
|
48
|
+
let timer;
|
|
49
|
+
try {
|
|
50
|
+
return await Promise.race([
|
|
51
|
+
promise,
|
|
52
|
+
new Promise((_resolve, reject) => {
|
|
53
|
+
timer = setTimeout(() => {
|
|
54
|
+
reject(new Error(`Scenario '${scenarioId}' exceeded its ${timeoutMs}ms execution budget.`));
|
|
55
|
+
}, timeoutMs);
|
|
56
|
+
}),
|
|
57
|
+
]);
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
if (timer)
|
|
61
|
+
clearTimeout(timer);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
// Runs a single scenario, converting a skip condition, thrown error, or
|
|
65
|
+
// timeout into a structured AttackResult rather than letting exceptions
|
|
66
|
+
// propagate to the caller.
|
|
67
|
+
export async function runScenario(scenario, ctx) {
|
|
68
|
+
const baseFields = {
|
|
69
|
+
scenarioId: scenario.id,
|
|
70
|
+
scenarioTitle: scenario.title,
|
|
71
|
+
checkId: scenario.checkId,
|
|
72
|
+
profileId: ctx.profile,
|
|
73
|
+
};
|
|
74
|
+
try {
|
|
75
|
+
const skipReason = scenario.skipCondition?.(ctx);
|
|
76
|
+
if (skipReason) {
|
|
77
|
+
return {
|
|
78
|
+
...baseFields,
|
|
79
|
+
status: "skipped",
|
|
80
|
+
severity: "skipped",
|
|
81
|
+
confidence: "low",
|
|
82
|
+
evidence: [],
|
|
83
|
+
category: CHECK_ID_CATEGORY[scenario.checkId],
|
|
84
|
+
skippedReason: skipReason,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
const outcome = await withTimeout(scenario.run(ctx), scenario.timeoutMs ?? DEFAULT_SCENARIO_TIMEOUT_MS, scenario.id);
|
|
88
|
+
return {
|
|
89
|
+
...baseFields,
|
|
90
|
+
status: outcome.status,
|
|
91
|
+
severity: outcome.severity ?? scenario.severityBaseline,
|
|
92
|
+
confidence: outcome.confidence,
|
|
93
|
+
evidence: outcome.evidence,
|
|
94
|
+
category: outcome.category ?? CHECK_ID_CATEGORY[scenario.checkId],
|
|
95
|
+
recommendation: outcome.recommendation,
|
|
96
|
+
verdictImpact: scenario.verdictImpact,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
return {
|
|
101
|
+
...baseFields,
|
|
102
|
+
status: "blocked",
|
|
103
|
+
severity: scenario.severityBaseline,
|
|
104
|
+
confidence: "low",
|
|
105
|
+
evidence: [],
|
|
106
|
+
category: CHECK_ID_CATEGORY[scenario.checkId],
|
|
107
|
+
errorSummary: err instanceof Error ? err.message : String(err),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function unavailableResult(checkId, profile) {
|
|
112
|
+
return {
|
|
113
|
+
scenarioId: `${checkId}-no-scenarios-registered`,
|
|
114
|
+
scenarioTitle: `No attack scenarios registered for '${checkId}'`,
|
|
115
|
+
checkId,
|
|
116
|
+
profileId: profile,
|
|
117
|
+
status: "skipped",
|
|
118
|
+
severity: "skipped",
|
|
119
|
+
confidence: "low",
|
|
120
|
+
evidence: [],
|
|
121
|
+
category: CHECK_ID_CATEGORY[checkId],
|
|
122
|
+
skippedReason: `No attack scenarios are registered yet for check group '${checkId}'. ` +
|
|
123
|
+
`The attack-scenario framework is present (v0.2.2 Batch 2); concrete scenarios are planned for a later v0.2.2 batch. ` +
|
|
124
|
+
`This must not be treated as evidence that '${checkId}' is safe.`,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
// Runs all scenarios matching selectedChecks/profile, then appends one
|
|
128
|
+
// explicit "unavailable" placeholder result for every selected check that
|
|
129
|
+
// matched zero scenarios. Safe when the scenario list is empty. Result order
|
|
130
|
+
// is deterministic: matched scenarios in registry order, then unavailable
|
|
131
|
+
// placeholders in selectedChecks order.
|
|
132
|
+
export async function runAttackScenarios(options) {
|
|
133
|
+
const scenarios = options.scenarios ?? SCENARIO_REGISTRY;
|
|
134
|
+
const matched = filterScenarios(scenarios, options.selectedChecks, options.profile);
|
|
135
|
+
const ctx = {
|
|
136
|
+
toolRoot: options.toolRoot,
|
|
137
|
+
target: options.target,
|
|
138
|
+
profile: options.profile,
|
|
139
|
+
config: options.config,
|
|
140
|
+
targetSnapshotBefore: options.targetSnapshotBefore,
|
|
141
|
+
};
|
|
142
|
+
const results = [];
|
|
143
|
+
for (const scenario of matched) {
|
|
144
|
+
results.push(await runScenario(scenario, ctx));
|
|
145
|
+
}
|
|
146
|
+
const matchedCheckIds = new Set(matched.map((s) => s.checkId));
|
|
147
|
+
for (const checkId of options.selectedChecks) {
|
|
148
|
+
if (!matchedCheckIds.has(checkId)) {
|
|
149
|
+
results.push(unavailableResult(checkId, options.profile));
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return results;
|
|
153
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// v0.2.2 Batch 2 — attack scenario contract.
|
|
3
|
+
//
|
|
4
|
+
// This is the shared interface later v0.2.2 batches implement concrete
|
|
5
|
+
// scenarios against. Batch 2 intentionally ships with an empty scenario
|
|
6
|
+
// registry (see attackRunner.ts) — this file only defines the shape.
|
|
7
|
+
// ---------------------------------------------------------------------------
|
|
8
|
+
// Maps a planned/attack-scenario check id to its default report category.
|
|
9
|
+
// Used both when no scenarios are registered yet (placeholder result) and by
|
|
10
|
+
// scenario authors as a sensible default category.
|
|
11
|
+
export const CHECK_ID_CATEGORY = {
|
|
12
|
+
deps: "dependency-audit",
|
|
13
|
+
package: "package-content",
|
|
14
|
+
static: "static-scan",
|
|
15
|
+
"cli-adversarial": "cli-adversarial",
|
|
16
|
+
fuzz: "fuzz-smoke",
|
|
17
|
+
boundary: "artifact-safety",
|
|
18
|
+
subprocess: "cli-adversarial",
|
|
19
|
+
secrets: "secret-leakage",
|
|
20
|
+
network: "network-boundary",
|
|
21
|
+
};
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import { collectFilesForGlobs } from "../../core/fileGlobs.js";
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// v0.2.2 Batch 4 — shared bounded, deterministic source-file scan helper.
|
|
5
|
+
//
|
|
6
|
+
// Reused by secretLeakageScenario and networkAssumptionScenario. Both need
|
|
7
|
+
// "read a bounded, deterministic set of source-ish files, skip large/binary
|
|
8
|
+
// content" and nothing more — collectFilesForGlobs() already provides
|
|
9
|
+
// deterministic ordering and standard exclusions (node_modules, dist, build,
|
|
10
|
+
// coverage, .git, lab-output, .my-dev-kit); this layer adds a size cap and a
|
|
11
|
+
// small file-content allowlist filter for scenario-specific exclusions.
|
|
12
|
+
// ---------------------------------------------------------------------------
|
|
13
|
+
// Default bounded glob set: project manifests/config plus first-party source
|
|
14
|
+
// and scripts. Deliberately excludes tests/, benchmarks/, examples/ — those
|
|
15
|
+
// commonly contain intentionally fake secret-shaped fixtures (including this
|
|
16
|
+
// project's own security payload corpus test suite) that would otherwise be
|
|
17
|
+
// permanent false positives against this project's own repo.
|
|
18
|
+
export const DEFAULT_SOURCE_SCAN_GLOBS = [
|
|
19
|
+
"package.json",
|
|
20
|
+
"package-lock.json",
|
|
21
|
+
"npm-shrinkwrap.json",
|
|
22
|
+
"tsconfig.json",
|
|
23
|
+
"tsconfig.*.json",
|
|
24
|
+
".env",
|
|
25
|
+
".env.*",
|
|
26
|
+
"src/**/*",
|
|
27
|
+
"scripts/**/*",
|
|
28
|
+
];
|
|
29
|
+
// Files that are deliberately excluded from scan output even though they
|
|
30
|
+
// match a glob above — e.g. the payload corpus itself, which intentionally
|
|
31
|
+
// contains fake-secret-shaped strings for Batch 2's test corpus.
|
|
32
|
+
const KNOWN_SYNTHETIC_FIXTURE_FILES = new Set(["src/securityValidation/attackScenarios/payloadCorpus.ts"]);
|
|
33
|
+
// Skip files larger than this to keep the scan bounded and fast; source
|
|
34
|
+
// files this large are not expected in this project.
|
|
35
|
+
const MAX_SCANNABLE_FILE_BYTES = 1_000_000;
|
|
36
|
+
// A single NUL character, used as a cheap binary-file heuristic below.
|
|
37
|
+
// Built via fromCharCode rather than a "\0" string escape to avoid any
|
|
38
|
+
// literal control byte ending up in this source file.
|
|
39
|
+
const NUL_CHAR = String.fromCharCode(0);
|
|
40
|
+
// Returns file contents for the bounded glob set, skipping known synthetic
|
|
41
|
+
// fixtures, oversized files, and anything unreadable as UTF-8 text (treated
|
|
42
|
+
// as binary and skipped, not an error). Stable, deterministic order.
|
|
43
|
+
export function collectBoundedSourceFiles(targetRoot, globs = DEFAULT_SOURCE_SCAN_GLOBS) {
|
|
44
|
+
// collectFilesForGlobs() throws for the *entire* call if any single glob's
|
|
45
|
+
// base path doesn't exist (e.g. no .env file, no tsconfig.*.json match) —
|
|
46
|
+
// resolve each glob independently so one missing/inapplicable glob doesn't
|
|
47
|
+
// silently zero out results for all the others.
|
|
48
|
+
const candidateMap = new Map();
|
|
49
|
+
for (const glob of globs) {
|
|
50
|
+
let matches;
|
|
51
|
+
try {
|
|
52
|
+
matches = collectFilesForGlobs(targetRoot, [glob]);
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
// This glob's base path doesn't exist for this target — a legitimate
|
|
56
|
+
// "nothing to scan for this pattern" outcome, not an error.
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
for (const m of matches) {
|
|
60
|
+
candidateMap.set(m.relativePath, m.absolutePath);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const candidates = [...candidateMap.entries()]
|
|
64
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
65
|
+
.map(([relativePath, absolutePath]) => ({ relativePath, absolutePath }));
|
|
66
|
+
const files = [];
|
|
67
|
+
for (const candidate of candidates) {
|
|
68
|
+
const normalizedRel = candidate.relativePath.replace(/\\/g, "/");
|
|
69
|
+
if (KNOWN_SYNTHETIC_FIXTURE_FILES.has(normalizedRel))
|
|
70
|
+
continue;
|
|
71
|
+
let stat;
|
|
72
|
+
try {
|
|
73
|
+
stat = fs.statSync(candidate.absolutePath);
|
|
74
|
+
}
|
|
75
|
+
catch {
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
if (!stat.isFile() || stat.size > MAX_SCANNABLE_FILE_BYTES)
|
|
79
|
+
continue;
|
|
80
|
+
try {
|
|
81
|
+
const content = fs.readFileSync(candidate.absolutePath, "utf8");
|
|
82
|
+
if (content.indexOf(NUL_CHAR) !== -1)
|
|
83
|
+
continue;
|
|
84
|
+
files.push({ relativePath: normalizedRel, absolutePath: candidate.absolutePath, content });
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return files;
|
|
91
|
+
}
|
|
92
|
+
// Returns the 1-based line number containing `index` within `content`.
|
|
93
|
+
export function lineNumberAt(content, index) {
|
|
94
|
+
let line = 1;
|
|
95
|
+
for (let i = 0; i < index && i < content.length; i++) {
|
|
96
|
+
if (content[i] === "\n")
|
|
97
|
+
line++;
|
|
98
|
+
}
|
|
99
|
+
return line;
|
|
100
|
+
}
|