@dailephd/my-dev-kit-lab 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +272 -0
- package/benchmarks/contracts/benchmark-project-profiles.json +1199 -0
- package/benchmarks/contracts/todo-behavior.md +70 -0
- package/benchmarks/contracts/todo-benchmark-case.json +227 -0
- package/benchmarks/projects/README.md +34 -0
- package/benchmarks/projects/task-analytics-large-mixed/README.md +1 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/__init__.py +3 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/fixtures.py +6 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/metrics.py +29 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/models.py +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/parser.py +16 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/pipeline.py +9 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/quality.py +8 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/reporting.py +11 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_metrics.py +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_parser.py +15 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_quality.py +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_reporting.py +15 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/package.json +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/index.ts +11 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/analyticsSnapshot.ts +20 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/project.ts +5 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/task.ts +10 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/reporting/buildProjectLeaderboard.ts +7 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/reporting/formatTaskHealthReport.ts +13 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/buildAnalyticsSnapshot.ts +39 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/completeTask.ts +10 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/createTask.ts +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/listTasksByProject.ts +6 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/store/projectStore.ts +20 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/store/taskStore.ts +44 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/validation/projectValidation.ts +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/validation/taskValidation.ts +18 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/buildAnalyticsSnapshot.test.ts +48 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/completeTask.test.ts +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/createTask.test.ts +31 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/listTasksByProject.test.ts +18 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/reporting.test.ts +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tsconfig.json +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/vitest.config.ts +5 -0
- package/benchmarks/projects/task-workflow-medium-ts/README.md +1 -0
- package/benchmarks/projects/task-workflow-medium-ts/package.json +12 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/index.ts +9 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/models/project.ts +6 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/models/task.ts +39 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/completeTask.ts +15 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/createTask.ts +26 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/filterTasks.ts +17 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/importTasks.ts +33 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/summarizeTasks.ts +30 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/store/taskStore.ts +76 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/utils/deterministicId.ts +3 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/validation/taskValidation.ts +45 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/completeTask.test.ts +16 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/createTask.test.ts +21 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/filterTasks.test.ts +18 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/importTasks.test.ts +22 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/summarizeTasks.test.ts +29 -0
- package/benchmarks/projects/task-workflow-medium-ts/tsconfig.json +12 -0
- package/benchmarks/projects/task-workflow-medium-ts/vitest.config.ts +5 -0
- package/benchmarks/projects/todo-js/README.md +3 -0
- package/benchmarks/projects/todo-js/package.json +11 -0
- package/benchmarks/projects/todo-js/src/index.js +2 -0
- package/benchmarks/projects/todo-js/src/taskService.js +37 -0
- package/benchmarks/projects/todo-js/src/taskStore.js +28 -0
- package/benchmarks/projects/todo-js/tests/taskService.test.js +45 -0
- package/benchmarks/projects/todo-js/vitest.config.js +5 -0
- package/benchmarks/projects/todo-mixed-ts-py/README.md +3 -0
- package/benchmarks/projects/todo-mixed-ts-py/package.json +13 -0
- package/benchmarks/projects/todo-mixed-ts-py/python/task_service.py +76 -0
- package/benchmarks/projects/todo-mixed-ts-py/src/taskCli.ts +38 -0
- package/benchmarks/projects/todo-mixed-ts-py/tests/mixedBoundary.test.ts +18 -0
- package/benchmarks/projects/todo-mixed-ts-py/tsconfig.json +12 -0
- package/benchmarks/projects/todo-mixed-ts-py/vitest.config.ts +5 -0
- package/benchmarks/projects/todo-python/README.md +3 -0
- package/benchmarks/projects/todo-python/src/__init__.py +4 -0
- package/benchmarks/projects/todo-python/src/task_service.py +32 -0
- package/benchmarks/projects/todo-python/src/task_store.py +28 -0
- package/benchmarks/projects/todo-python/tests/test_task_service.py +52 -0
- package/benchmarks/projects/todo-ts/README.md +3 -0
- package/benchmarks/projects/todo-ts/package.json +12 -0
- package/benchmarks/projects/todo-ts/src/index.ts +2 -0
- package/benchmarks/projects/todo-ts/src/taskService.ts +41 -0
- package/benchmarks/projects/todo-ts/src/taskStore.ts +34 -0
- package/benchmarks/projects/todo-ts/tests/taskService.test.ts +45 -0
- package/benchmarks/projects/todo-ts/tsconfig.json +12 -0
- package/benchmarks/projects/todo-ts/vitest.config.ts +5 -0
- package/dist/scripts/build-gallery.js +3 -0
- package/dist/scripts/capture-demo-report.js +3 -0
- package/dist/scripts/evaluate-token-savings.js +2 -0
- package/dist/scripts/experiments/describeExperiment.js +143 -0
- package/dist/scripts/experiments/listExperiments.js +44 -0
- package/dist/scripts/experiments/runExperiment.js +199 -0
- package/dist/scripts/generate-experiment-plots.js +3 -0
- package/dist/scripts/generate-prompt-variants.js +2 -0
- package/dist/scripts/render-experiment-report.js +2 -0
- package/dist/scripts/run-agent-prompt.js +2 -0
- package/dist/scripts/run-controlled-experiment.js +2 -0
- package/dist/scripts/run-final-demo.js +3 -0
- package/dist/scripts/run-lab-demo.js +5 -0
- package/dist/scripts/run-visualization-demos.js +3 -0
- package/dist/scripts/security/runCodeql.js +57 -0
- package/dist/scripts/security/runDependencyChecks.js +57 -0
- package/dist/scripts/security/runFuzzSmoke.js +29 -0
- package/dist/scripts/security/runPackageChecks.js +56 -0
- package/dist/scripts/security/runSemgrep.js +63 -0
- package/dist/scripts/security/validate.js +117 -0
- package/dist/scripts/verify-benchmarks.js +202 -0
- package/dist/src/agents/adapters/claudeAdapter.js +37 -0
- package/dist/src/agents/adapters/codexAdapter.js +110 -0
- package/dist/src/agents/adapters/fakeAgentAdapter.js +101 -0
- package/dist/src/agents/agentRegistry.js +21 -0
- package/dist/src/agents/index.js +7 -0
- package/dist/src/agents/parseAgentTokenUsage.js +137 -0
- package/dist/src/agents/runAgentPrompt.js +38 -0
- package/dist/src/agents/types.js +1 -0
- package/dist/src/commands/buildGalleryCommand.js +56 -0
- package/dist/src/commands/captureDemoReport.js +116 -0
- package/dist/src/commands/evaluateTokenSavings.js +175 -0
- package/dist/src/commands/generateExperimentPlotsCommand.js +38 -0
- package/dist/src/commands/generatePromptVariants.js +67 -0
- package/dist/src/commands/renderExperimentReportCommand.js +131 -0
- package/dist/src/commands/runAgentPromptCommand.js +132 -0
- package/dist/src/commands/runControlledExperimentCommand.js +174 -0
- package/dist/src/commands/runFinalDemoCommand.js +123 -0
- package/dist/src/commands/runLabDemo.js +62 -0
- package/dist/src/commands/runVisualizationDemosCommand.js +67 -0
- package/dist/src/core/commandLine.js +59 -0
- package/dist/src/core/countTokens.js +8 -0
- package/dist/src/core/fileGlobs.js +100 -0
- package/dist/src/core/localProjectTarget.js +75 -0
- package/dist/src/core/pathSafety.js +19 -0
- package/dist/src/core/pythonCommand.js +30 -0
- package/dist/src/core/resolveCommand.js +110 -0
- package/dist/src/core/runMeasuredCommand.js +143 -0
- package/dist/src/evaluation/benchmarkMetadata.js +207 -0
- package/dist/src/evaluation/buildExperimentMatrix.js +75 -0
- package/dist/src/evaluation/classifyAgentRunOutcome.js +40 -0
- package/dist/src/evaluation/compareExperimentRuns.js +79 -0
- package/dist/src/evaluation/compareTokenSavings.js +47 -0
- package/dist/src/evaluation/controlledExperimentTypes.js +1 -0
- package/dist/src/evaluation/index.js +18 -0
- package/dist/src/evaluation/parseAgentAnswer.js +230 -0
- package/dist/src/evaluation/projectComplexity.js +126 -0
- package/dist/src/evaluation/projectFileTree.js +83 -0
- package/dist/src/evaluation/readEvaluationCases.js +59 -0
- package/dist/src/evaluation/renderTokenSavingsReportInput.js +55 -0
- package/dist/src/evaluation/runControlledExperiment.js +158 -0
- package/dist/src/evaluation/runMyDevKitRetrieval.js +197 -0
- package/dist/src/evaluation/runRawFullFileBaseline.js +31 -0
- package/dist/src/evaluation/scoreCorrectness.js +127 -0
- package/dist/src/evaluation/types.js +1 -0
- package/dist/src/evaluation/writeExperimentArtifacts.js +104 -0
- package/dist/src/evaluation/writeTokenSavingsArtifacts.js +57 -0
- package/dist/src/experiments/config.js +24 -0
- package/dist/src/experiments/defaultRegistry.js +7 -0
- package/dist/src/experiments/errors.js +18 -0
- package/dist/src/experiments/index.js +9 -0
- package/dist/src/experiments/outputPaths.js +25 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/config.js +37 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/index.js +3 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +83 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +260 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/registry.js +43 -0
- package/dist/src/experiments/results.js +48 -0
- package/dist/src/experiments/runner.js +181 -0
- package/dist/src/experiments/target.js +8 -0
- package/dist/src/experiments/types.js +1 -0
- package/dist/src/gallery/index.js +2 -0
- package/dist/src/gallery/types.js +1 -0
- package/dist/src/gallery/writeGalleryManifest.js +214 -0
- package/dist/src/index.js +12 -0
- package/dist/src/plots/buildExperimentPlotData.js +137 -0
- package/dist/src/plots/index.js +4 -0
- package/dist/src/plots/renderSvgChart.js +82 -0
- package/dist/src/plots/types.js +1 -0
- package/dist/src/plots/writePlotArtifacts.js +46 -0
- package/dist/src/prompts/buildPromptContext.js +68 -0
- package/dist/src/prompts/generateMyDevKitPrompt.js +106 -0
- package/dist/src/prompts/generatePromptVariants.js +36 -0
- package/dist/src/prompts/generateRawFullFilePrompt.js +97 -0
- package/dist/src/prompts/index.js +7 -0
- package/dist/src/prompts/measurePromptComplexity.js +41 -0
- package/dist/src/prompts/types.js +1 -0
- package/dist/src/prompts/writePromptArtifacts.js +43 -0
- package/dist/src/report/buildExperimentReportInput.js +339 -0
- package/dist/src/report/experimentReportTypes.js +1 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +153 -0
- package/dist/src/report/experiments/experimentReportModel.js +1 -0
- package/dist/src/report/experiments/index.js +4 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +133 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +30 -0
- package/dist/src/report/index.js +8 -0
- package/dist/src/report/renderExperimentHtmlReport.js +354 -0
- package/dist/src/report/renderHtmlReport.js +103 -0
- package/dist/src/report/types.js +10 -0
- package/dist/src/report/writeExperimentReportArtifacts.js +38 -0
- package/dist/src/report/writeReportArtifacts.js +39 -0
- package/dist/src/screenshot/captureReportScreenshot.js +75 -0
- package/dist/src/screenshot/index.js +2 -0
- package/dist/src/screenshot/types.js +1 -0
- package/dist/src/securityValidation/artifacts.js +15 -0
- package/dist/src/securityValidation/cliAdversarial/adversarialCliConfig.js +38 -0
- package/dist/src/securityValidation/cliAdversarial/dataVolumeChecks.js +194 -0
- package/dist/src/securityValidation/cliAdversarial/jsonStdoutChecks.js +359 -0
- package/dist/src/securityValidation/cliAdversarial/malformedArtifactChecks.js +284 -0
- package/dist/src/securityValidation/cliAdversarial/malformedArtifactFixtures.js +79 -0
- package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +431 -0
- package/dist/src/securityValidation/cliAdversarial/pathCases.js +144 -0
- package/dist/src/securityValidation/cliAdversarial/readOnlyBoundaryChecks.js +294 -0
- package/dist/src/securityValidation/cliAdversarial/runAdversarialCheck.js +149 -0
- package/dist/src/securityValidation/cliAdversarial/subprocessSafetyChecks.js +214 -0
- package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +160 -0
- package/dist/src/securityValidation/commandRunner.js +136 -0
- package/dist/src/securityValidation/config.js +39 -0
- package/dist/src/securityValidation/dependencies/parseNpmAudit.js +115 -0
- package/dist/src/securityValidation/dependencies/parseNpmLs.js +71 -0
- package/dist/src/securityValidation/dependencies/parseNpmOutdated.js +41 -0
- package/dist/src/securityValidation/dependencies/runDependencyChecks.js +239 -0
- package/dist/src/securityValidation/dependencies/runOsvScanner.js +43 -0
- package/dist/src/securityValidation/fuzz/fuzzHarness.js +61 -0
- package/dist/src/securityValidation/fuzz/fuzzTargets.js +204 -0
- package/dist/src/securityValidation/fuzz/randomInput.js +0 -0
- package/dist/src/securityValidation/index.js +34 -0
- package/dist/src/securityValidation/packageChecks/forbiddenPackageContents.js +67 -0
- package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +56 -0
- package/dist/src/securityValidation/packageChecks/runPackageChecks.js +88 -0
- package/dist/src/securityValidation/report/renderSecurityReport.js +248 -0
- package/dist/src/securityValidation/report/securityReportTypes.js +1 -0
- package/dist/src/securityValidation/staticScans/codeql.js +66 -0
- package/dist/src/securityValidation/staticScans/semgrep.js +180 -0
- package/dist/src/securityValidation/testMatrix.js +535 -0
- package/dist/src/securityValidation/types.js +34 -0
- package/dist/src/securityValidation/validate/resolveTarget.js +32 -0
- package/dist/src/securityValidation/validate/runSecurityValidation.js +169 -0
- package/dist/src/securityValidation/validate/verdict.js +73 -0
- package/dist/src/visualizationDemos/buildMyDevKitVisualizationCommands.js +59 -0
- package/dist/src/visualizationDemos/index.js +4 -0
- package/dist/src/visualizationDemos/runVisualizationDemos.js +82 -0
- package/dist/src/visualizationDemos/types.js +1 -0
- package/dist/src/visualizationDemos/writeVisualizationDemoArtifacts.js +25 -0
- package/docs/METRICS.md +286 -0
- package/examples/demo-report-input.json +78 -0
- package/examples/lab-demo-cases.json +35 -0
- package/examples/real-agent-campaign-cases.json +118 -0
- package/examples/token-savings-cases.json +122 -0
- package/package.json +91 -0
- package/tests/fixtures/fake-adversarial-cli.js +152 -0
- package/tests/fixtures/fake-my-dev-kit-cli.js +83 -0
|
@@ -0,0 +1,535 @@
|
|
|
1
|
+
export const SECURITY_TEST_MATRIX = [
|
|
2
|
+
// ---------------------------------------------------------------------------
|
|
3
|
+
// Path and filesystem tests
|
|
4
|
+
// ---------------------------------------------------------------------------
|
|
5
|
+
{
|
|
6
|
+
id: "path-traversal-root",
|
|
7
|
+
title: "Path traversal via --root",
|
|
8
|
+
category: "cli-adversarial",
|
|
9
|
+
attackSurface: "--root",
|
|
10
|
+
inputExamples: ["--root ../../etc", "--root /etc/passwd", "--root ..\\..\\Windows"],
|
|
11
|
+
expectedBehavior: "CLI rejects the path or confines all operations within an allowed boundary; no file access outside the intended repo root",
|
|
12
|
+
severityIfFailed: "blocker",
|
|
13
|
+
implementationStatus: "implemented",
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
id: "path-traversal-out",
|
|
17
|
+
title: "Path traversal via --out",
|
|
18
|
+
category: "cli-adversarial",
|
|
19
|
+
attackSurface: "--out",
|
|
20
|
+
inputExamples: ["--out ../../etc/cron.d", "--out /tmp/escape"],
|
|
21
|
+
expectedBehavior: "Output is written only within the specified directory; no writes outside the intended output tree",
|
|
22
|
+
severityIfFailed: "blocker",
|
|
23
|
+
implementationStatus: "implemented",
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
id: "path-traversal-index",
|
|
27
|
+
title: "Path traversal via --index",
|
|
28
|
+
category: "cli-adversarial",
|
|
29
|
+
attackSurface: "--index",
|
|
30
|
+
inputExamples: ["--index ../../etc/.my-dev-kit"],
|
|
31
|
+
expectedBehavior: "Index path is validated; reads are confined to the specified index directory",
|
|
32
|
+
severityIfFailed: "blocker",
|
|
33
|
+
implementationStatus: "implemented",
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
id: "absolute-path-escape",
|
|
37
|
+
title: "Absolute path outside repo root",
|
|
38
|
+
category: "cli-adversarial",
|
|
39
|
+
attackSurface: "--root, --src, --file",
|
|
40
|
+
inputExamples: ["--file /etc/hosts", "--src C:\\Windows\\System32"],
|
|
41
|
+
expectedBehavior: "CLI rejects absolute paths that escape the declared root boundary",
|
|
42
|
+
severityIfFailed: "blocker",
|
|
43
|
+
implementationStatus: "implemented",
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
id: "symlink-junction-escape",
|
|
47
|
+
title: "Symlink or junction escape",
|
|
48
|
+
category: "cli-adversarial",
|
|
49
|
+
attackSurface: "--root, --src",
|
|
50
|
+
inputExamples: ["directory containing a symlink pointing outside the root"],
|
|
51
|
+
expectedBehavior: "Symlink targets outside the root are not followed or are treated as out-of-scope",
|
|
52
|
+
severityIfFailed: "major",
|
|
53
|
+
implementationStatus: "skipped-environment",
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
id: "generated-cleanup-user-files",
|
|
57
|
+
title: "Generated artifact cleanup must not delete user files",
|
|
58
|
+
category: "artifact-safety",
|
|
59
|
+
attackSurface: "artifact refresh / re-index cleanup",
|
|
60
|
+
inputExamples: ["index refresh over a directory that contains user source files"],
|
|
61
|
+
expectedBehavior: "Only generated artifacts in the explicitly declared output path are removed; user source files are never deleted",
|
|
62
|
+
severityIfFailed: "blocker",
|
|
63
|
+
implementationStatus: "implemented",
|
|
64
|
+
},
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
// Read-only boundary tests
|
|
67
|
+
// ---------------------------------------------------------------------------
|
|
68
|
+
{
|
|
69
|
+
id: "source-files-not-modified",
|
|
70
|
+
title: "Source files are not modified during indexing",
|
|
71
|
+
category: "cli-adversarial",
|
|
72
|
+
attackSurface: "index command on a real source tree",
|
|
73
|
+
inputExamples: ["npm run index -- --root benchmarks/projects/todo-ts"],
|
|
74
|
+
expectedBehavior: "All source files in the root directory have the same content and modification time after indexing completes",
|
|
75
|
+
severityIfFailed: "blocker",
|
|
76
|
+
implementationStatus: "implemented",
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
id: "writes-limited-to-output",
|
|
80
|
+
title: "Writes are limited to declared artifact paths",
|
|
81
|
+
category: "cli-adversarial",
|
|
82
|
+
attackSurface: "--out, --index",
|
|
83
|
+
inputExamples: ["run index with --out pointing to a temp directory; verify no writes elsewhere"],
|
|
84
|
+
expectedBehavior: "No files are created or modified outside the declared output and index paths",
|
|
85
|
+
severityIfFailed: "blocker",
|
|
86
|
+
implementationStatus: "implemented",
|
|
87
|
+
},
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
// Malformed artifact tests
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
{
|
|
92
|
+
id: "malformed-manifest-json",
|
|
93
|
+
title: "Malformed manifest.json is rejected safely",
|
|
94
|
+
category: "artifact-safety",
|
|
95
|
+
attackSurface: "manifest reader",
|
|
96
|
+
inputExamples: ["{", "null", "[]", "{\"version\": null}", "not JSON at all"],
|
|
97
|
+
expectedBehavior: "CLI surfaces a clear error without crashing or panicking; no partial state is committed",
|
|
98
|
+
severityIfFailed: "major",
|
|
99
|
+
implementationStatus: "implemented",
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
id: "malformed-symbol-index-json",
|
|
103
|
+
title: "Malformed symbol-index.json is rejected safely",
|
|
104
|
+
category: "artifact-safety",
|
|
105
|
+
attackSurface: "symbol-index reader",
|
|
106
|
+
inputExamples: ["truncated JSON", "array instead of object", "missing required fields"],
|
|
107
|
+
expectedBehavior: "Clear error message; no crash",
|
|
108
|
+
severityIfFailed: "major",
|
|
109
|
+
implementationStatus: "planned",
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
id: "malformed-code-graph-json",
|
|
113
|
+
title: "Malformed code-graph.json is rejected safely",
|
|
114
|
+
category: "artifact-safety",
|
|
115
|
+
attackSurface: "code-graph reader",
|
|
116
|
+
inputExamples: ["empty object", "nodes array is null", "cyclic reference marker"],
|
|
117
|
+
expectedBehavior: "Clear error message; no crash",
|
|
118
|
+
severityIfFailed: "major",
|
|
119
|
+
implementationStatus: "implemented",
|
|
120
|
+
},
|
|
121
|
+
{
|
|
122
|
+
id: "malformed-data-model-json",
|
|
123
|
+
title: "Malformed data-model.json is rejected safely",
|
|
124
|
+
category: "artifact-safety",
|
|
125
|
+
attackSurface: "data-model reader",
|
|
126
|
+
inputExamples: ["truncated JSON", "missing schema version"],
|
|
127
|
+
expectedBehavior: "Clear error message; no crash",
|
|
128
|
+
severityIfFailed: "major",
|
|
129
|
+
implementationStatus: "planned",
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
id: "malformed-frontend-semantic-json",
|
|
133
|
+
title: "Malformed frontend-semantic.json is rejected safely",
|
|
134
|
+
category: "artifact-safety",
|
|
135
|
+
attackSurface: "frontend-semantic reader",
|
|
136
|
+
inputExamples: ["empty file", "non-JSON content", "schema version mismatch"],
|
|
137
|
+
expectedBehavior: "Clear error message; no crash",
|
|
138
|
+
severityIfFailed: "major",
|
|
139
|
+
implementationStatus: "planned",
|
|
140
|
+
},
|
|
141
|
+
{
|
|
142
|
+
id: "unsupported-schema-version",
|
|
143
|
+
title: "Unsupported schema version produces a clear error",
|
|
144
|
+
category: "artifact-safety",
|
|
145
|
+
attackSurface: "all artifact readers",
|
|
146
|
+
inputExamples: ["{\"schemaVersion\": 9999}", "{\"schemaVersion\": \"future\"}"],
|
|
147
|
+
expectedBehavior: "CLI reports an unsupported-version error with the version it found; no silent data corruption",
|
|
148
|
+
severityIfFailed: "major",
|
|
149
|
+
implementationStatus: "implemented",
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
id: "missing-index-directory",
|
|
153
|
+
title: "Missing index directory produces a clear error",
|
|
154
|
+
category: "artifact-safety",
|
|
155
|
+
attackSurface: "--index pointing to a nonexistent path",
|
|
156
|
+
inputExamples: ["--index /nonexistent/.my-dev-kit"],
|
|
157
|
+
expectedBehavior: "Clear error stating the index directory does not exist; no crash",
|
|
158
|
+
severityIfFailed: "minor",
|
|
159
|
+
implementationStatus: "implemented",
|
|
160
|
+
},
|
|
161
|
+
// ---------------------------------------------------------------------------
|
|
162
|
+
// JSON stdout/stderr tests
|
|
163
|
+
// ---------------------------------------------------------------------------
|
|
164
|
+
{
|
|
165
|
+
id: "json-mode-parseable-output",
|
|
166
|
+
title: "JSON mode returns parseable JSON",
|
|
167
|
+
category: "cli-adversarial",
|
|
168
|
+
attackSurface: "--format json or equivalent JSON output mode",
|
|
169
|
+
inputExamples: ["any valid CLI command with JSON output enabled"],
|
|
170
|
+
expectedBehavior: "stdout is valid JSON that can be parsed without error",
|
|
171
|
+
severityIfFailed: "major",
|
|
172
|
+
implementationStatus: "implemented",
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
id: "warnings-go-to-stderr",
|
|
176
|
+
title: "Warnings go to stderr, not stdout",
|
|
177
|
+
category: "cli-adversarial",
|
|
178
|
+
attackSurface: "all commands",
|
|
179
|
+
inputExamples: ["run any command that emits a warning; capture stdout and stderr separately"],
|
|
180
|
+
expectedBehavior: "Warning messages appear only on stderr; stdout is not contaminated",
|
|
181
|
+
severityIfFailed: "major",
|
|
182
|
+
implementationStatus: "implemented",
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
id: "progress-not-in-json-stdout",
|
|
186
|
+
title: "Progress output does not corrupt JSON stdout",
|
|
187
|
+
category: "cli-adversarial",
|
|
188
|
+
attackSurface: "all commands that emit progress messages",
|
|
189
|
+
inputExamples: ["run a long-running index with progress messages; parse stdout as JSON"],
|
|
190
|
+
expectedBehavior: "stdout remains valid JSON even when progress or status messages are emitted",
|
|
191
|
+
severityIfFailed: "major",
|
|
192
|
+
implementationStatus: "implemented",
|
|
193
|
+
},
|
|
194
|
+
{
|
|
195
|
+
id: "json-error-object-on-failure",
|
|
196
|
+
title: "Valid JSON error object returned on failure in JSON mode",
|
|
197
|
+
category: "cli-adversarial",
|
|
198
|
+
attackSurface: "--format json with an error condition",
|
|
199
|
+
inputExamples: ["--format json --root /nonexistent"],
|
|
200
|
+
expectedBehavior: "stdout is a valid JSON object with an error field; not a raw stack trace",
|
|
201
|
+
severityIfFailed: "minor",
|
|
202
|
+
implementationStatus: "implemented",
|
|
203
|
+
},
|
|
204
|
+
// ---------------------------------------------------------------------------
|
|
205
|
+
// Graphviz and subprocess tests
|
|
206
|
+
// ---------------------------------------------------------------------------
|
|
207
|
+
{
|
|
208
|
+
id: "dot-output-no-graphviz",
|
|
209
|
+
title: "DOT output does not require Graphviz",
|
|
210
|
+
category: "cli-adversarial",
|
|
211
|
+
attackSurface: "DOT/graph output commands",
|
|
212
|
+
inputExamples: ["run a graph command that produces DOT output; do not install Graphviz"],
|
|
213
|
+
expectedBehavior: "DOT text is written successfully without Graphviz being installed",
|
|
214
|
+
severityIfFailed: "major",
|
|
215
|
+
implementationStatus: "planned",
|
|
216
|
+
},
|
|
217
|
+
{
|
|
218
|
+
id: "svg-png-safe-without-graphviz",
|
|
219
|
+
title: "SVG/PNG generation fails safely when Graphviz is unavailable",
|
|
220
|
+
category: "cli-adversarial",
|
|
221
|
+
attackSurface: "SVG/PNG graph rendering",
|
|
222
|
+
inputExamples: ["request SVG or PNG output without Graphviz installed"],
|
|
223
|
+
expectedBehavior: "Clear error explaining Graphviz is required; no crash; DOT output is still available",
|
|
224
|
+
severityIfFailed: "minor",
|
|
225
|
+
implementationStatus: "planned",
|
|
226
|
+
},
|
|
227
|
+
{
|
|
228
|
+
id: "graphviz-label-escaping",
|
|
229
|
+
title: "Graph labels escape quotes and shell metacharacters",
|
|
230
|
+
category: "cli-adversarial",
|
|
231
|
+
attackSurface: "DOT label generation",
|
|
232
|
+
inputExamples: ["node name with double quote: foo\"bar", "node name with backslash: foo\\bar", "path with semicolon: foo;bar"],
|
|
233
|
+
expectedBehavior: "All special characters in DOT labels are properly escaped; no broken DOT syntax or injection",
|
|
234
|
+
severityIfFailed: "major",
|
|
235
|
+
implementationStatus: "implemented",
|
|
236
|
+
},
|
|
237
|
+
{
|
|
238
|
+
id: "subprocess-no-shell-interpolation",
|
|
239
|
+
title: "Subprocess calls avoid shell-string interpolation",
|
|
240
|
+
category: "cli-adversarial",
|
|
241
|
+
attackSurface: "all child_process invocations",
|
|
242
|
+
inputExamples: ["path with semicolon: /tmp/foo;rm -rf ~", "path with backtick: /tmp/`whoami`"],
|
|
243
|
+
expectedBehavior: "Subprocess is invoked with an argument array, not a shell string; metacharacters are treated as literal path content",
|
|
244
|
+
severityIfFailed: "blocker",
|
|
245
|
+
implementationStatus: "implemented",
|
|
246
|
+
},
|
|
247
|
+
// ---------------------------------------------------------------------------
|
|
248
|
+
// Secret leakage tests
|
|
249
|
+
// ---------------------------------------------------------------------------
|
|
250
|
+
{
|
|
251
|
+
id: "env-values-not-indexed",
|
|
252
|
+
title: ".env values are not indexed into generated artifacts",
|
|
253
|
+
category: "secret-leakage",
|
|
254
|
+
attackSurface: "indexer artifact generation",
|
|
255
|
+
inputExamples: ["repo root containing a .env file with API_KEY=secret"],
|
|
256
|
+
expectedBehavior: ".env content does not appear in manifest, symbol index, code graph, or data model artifacts",
|
|
257
|
+
severityIfFailed: "blocker",
|
|
258
|
+
implementationStatus: "planned",
|
|
259
|
+
},
|
|
260
|
+
{
|
|
261
|
+
id: "token-patterns-not-emitted",
|
|
262
|
+
title: "Common token patterns are not emitted in artifacts",
|
|
263
|
+
category: "secret-leakage",
|
|
264
|
+
attackSurface: "indexer artifact generation",
|
|
265
|
+
inputExamples: ["source file containing a commented-out AWS key or GitHub token pattern"],
|
|
266
|
+
expectedBehavior: "Artifacts do not contain raw secret values from source file comments or string literals",
|
|
267
|
+
severityIfFailed: "major",
|
|
268
|
+
implementationStatus: "planned",
|
|
269
|
+
},
|
|
270
|
+
{
|
|
271
|
+
id: "ignored-dirs-excluded",
|
|
272
|
+
title: "Ignored and generated directories are excluded from indexing",
|
|
273
|
+
category: "secret-leakage",
|
|
274
|
+
attackSurface: "indexer --root scan",
|
|
275
|
+
inputExamples: ["node_modules/ with a package containing a .env file", ".git/ directory"],
|
|
276
|
+
expectedBehavior: "Standard ignored directories are not indexed; their content does not appear in artifacts",
|
|
277
|
+
severityIfFailed: "major",
|
|
278
|
+
implementationStatus: "planned",
|
|
279
|
+
},
|
|
280
|
+
// ---------------------------------------------------------------------------
|
|
281
|
+
// Scale and robustness tests
|
|
282
|
+
// ---------------------------------------------------------------------------
|
|
283
|
+
{
|
|
284
|
+
id: "huge-source-file",
|
|
285
|
+
title: "Huge source file is handled safely",
|
|
286
|
+
category: "cli-adversarial",
|
|
287
|
+
attackSurface: "indexer and source reader",
|
|
288
|
+
inputExamples: ["a .ts file with 100,000 lines"],
|
|
289
|
+
expectedBehavior: "Indexer completes or fails gracefully without running out of memory or hanging indefinitely",
|
|
290
|
+
severityIfFailed: "major",
|
|
291
|
+
implementationStatus: "implemented",
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
id: "huge-literal",
|
|
295
|
+
title: "Huge string literal is handled safely",
|
|
296
|
+
category: "cli-adversarial",
|
|
297
|
+
attackSurface: "TS/TSX analyzer",
|
|
298
|
+
inputExamples: ["a single string literal with 1 million characters"],
|
|
299
|
+
expectedBehavior: "Analyzer does not hang or run out of memory processing the literal",
|
|
300
|
+
severityIfFailed: "minor",
|
|
301
|
+
implementationStatus: "planned",
|
|
302
|
+
},
|
|
303
|
+
{
|
|
304
|
+
id: "deeply-nested-tsx",
|
|
305
|
+
title: "Deeply nested TSX is handled safely",
|
|
306
|
+
category: "cli-adversarial",
|
|
307
|
+
attackSurface: "TSX frontend analyzer",
|
|
308
|
+
inputExamples: ["TSX with 500 levels of nesting"],
|
|
309
|
+
expectedBehavior: "Analyzer completes or fails with a clear depth-limit error; no stack overflow",
|
|
310
|
+
severityIfFailed: "minor",
|
|
311
|
+
implementationStatus: "implemented",
|
|
312
|
+
},
|
|
313
|
+
{
|
|
314
|
+
id: "many-duplicate-strings",
|
|
315
|
+
title: "Many duplicate strings do not cause unbounded growth",
|
|
316
|
+
category: "cli-adversarial",
|
|
317
|
+
attackSurface: "symbol index and data model",
|
|
318
|
+
inputExamples: ["10,000 files each containing the string 'TODO'"],
|
|
319
|
+
expectedBehavior: "Artifact size remains bounded; deduplication or truncation is applied where appropriate",
|
|
320
|
+
severityIfFailed: "minor",
|
|
321
|
+
implementationStatus: "planned",
|
|
322
|
+
},
|
|
323
|
+
{
|
|
324
|
+
id: "many-graph-nodes-edges",
|
|
325
|
+
title: "Many graph nodes and edges do not cause unbounded growth",
|
|
326
|
+
category: "cli-adversarial",
|
|
327
|
+
attackSurface: "code-graph builder and renderer",
|
|
328
|
+
inputExamples: ["a project with 10,000 files and 100,000 import relationships"],
|
|
329
|
+
expectedBehavior: "Graph artifact size is bounded; graph view renderer completes without hanging",
|
|
330
|
+
severityIfFailed: "minor",
|
|
331
|
+
implementationStatus: "implemented",
|
|
332
|
+
},
|
|
333
|
+
{
|
|
334
|
+
id: "unicode-paths",
|
|
335
|
+
title: "Unicode paths are handled safely",
|
|
336
|
+
category: "cli-adversarial",
|
|
337
|
+
attackSurface: "--root, --file, --out",
|
|
338
|
+
inputExamples: ["--root /tmp/репозиторий", "--out /tmp/出力"],
|
|
339
|
+
expectedBehavior: "All path operations handle Unicode without corruption or crash",
|
|
340
|
+
severityIfFailed: "minor",
|
|
341
|
+
implementationStatus: "planned",
|
|
342
|
+
},
|
|
343
|
+
// ---------------------------------------------------------------------------
|
|
344
|
+
// CLI argument tests
|
|
345
|
+
// ---------------------------------------------------------------------------
|
|
346
|
+
{
|
|
347
|
+
id: "empty-contains",
|
|
348
|
+
title: "Empty --contains value is handled safely",
|
|
349
|
+
category: "cli-adversarial",
|
|
350
|
+
attackSurface: "--contains",
|
|
351
|
+
inputExamples: ["--contains \"\"", "--contains ''"],
|
|
352
|
+
expectedBehavior: "Returns an empty result set or a clear validation error; no crash",
|
|
353
|
+
severityIfFailed: "minor",
|
|
354
|
+
implementationStatus: "planned",
|
|
355
|
+
},
|
|
356
|
+
{
|
|
357
|
+
id: "negative-context",
|
|
358
|
+
title: "Negative --context value is rejected safely",
|
|
359
|
+
category: "cli-adversarial",
|
|
360
|
+
attackSurface: "--context",
|
|
361
|
+
inputExamples: ["--context -1", "--context -100"],
|
|
362
|
+
expectedBehavior: "Validation error with clear message; command does not execute with a negative context window",
|
|
363
|
+
severityIfFailed: "minor",
|
|
364
|
+
implementationStatus: "planned",
|
|
365
|
+
},
|
|
366
|
+
{
|
|
367
|
+
id: "huge-context",
|
|
368
|
+
title: "Huge --context value does not cause memory exhaustion",
|
|
369
|
+
category: "cli-adversarial",
|
|
370
|
+
attackSurface: "--context",
|
|
371
|
+
inputExamples: ["--context 99999999"],
|
|
372
|
+
expectedBehavior: "Command completes within memory limits or fails with a clear limit-exceeded error",
|
|
373
|
+
severityIfFailed: "minor",
|
|
374
|
+
implementationStatus: "planned",
|
|
375
|
+
},
|
|
376
|
+
{
|
|
377
|
+
id: "unknown-node-id",
|
|
378
|
+
title: "Unknown node ID produces a clear error",
|
|
379
|
+
category: "cli-adversarial",
|
|
380
|
+
attackSurface: "--node",
|
|
381
|
+
inputExamples: ["--node nonexistent-node-id-xyz"],
|
|
382
|
+
expectedBehavior: "Clear error message stating the node was not found; no crash",
|
|
383
|
+
severityIfFailed: "minor",
|
|
384
|
+
implementationStatus: "planned",
|
|
385
|
+
},
|
|
386
|
+
{
|
|
387
|
+
id: "path-with-metacharacters",
|
|
388
|
+
title: "Paths with spaces, quotes, semicolons, and Unicode are handled safely",
|
|
389
|
+
category: "cli-adversarial",
|
|
390
|
+
attackSurface: "--root, --out, --file, --index",
|
|
391
|
+
inputExamples: ["path with space", "path with double quote", "path with semicolon", "path with Unicode"],
|
|
392
|
+
expectedBehavior: "All path arguments are treated as literal strings; no shell injection; correct behavior on Windows and Unix",
|
|
393
|
+
severityIfFailed: "major",
|
|
394
|
+
implementationStatus: "implemented",
|
|
395
|
+
},
|
|
396
|
+
// ---------------------------------------------------------------------------
|
|
397
|
+
// Static scan tests
|
|
398
|
+
// ---------------------------------------------------------------------------
|
|
399
|
+
{
|
|
400
|
+
id: "codeql-cli-availability",
|
|
401
|
+
title: "CodeQL CLI local availability check",
|
|
402
|
+
category: "static-scan",
|
|
403
|
+
attackSurface: "codeql binary",
|
|
404
|
+
inputExamples: ["run codeql version --format terse"],
|
|
405
|
+
expectedBehavior: "Skipped with structured reason if CLI is absent; passed if CLI is present and functional",
|
|
406
|
+
severityIfFailed: "major",
|
|
407
|
+
implementationStatus: "implemented",
|
|
408
|
+
},
|
|
409
|
+
{
|
|
410
|
+
id: "semgrep-scan-subprocess-safety",
|
|
411
|
+
title: "Semgrep rules for subprocess safety",
|
|
412
|
+
category: "static-scan",
|
|
413
|
+
attackSurface: "spawn/exec usage in src/",
|
|
414
|
+
inputExamples: ["spawn($CMD, $ARGS, {shell: true})"],
|
|
415
|
+
expectedBehavior: "Rules detect shell:true, exec with interpolation, and similar patterns; skipped if semgrep unavailable",
|
|
416
|
+
severityIfFailed: "major",
|
|
417
|
+
implementationStatus: "implemented",
|
|
418
|
+
},
|
|
419
|
+
{
|
|
420
|
+
id: "semgrep-scan-path-traversal",
|
|
421
|
+
title: "Semgrep rules for path traversal",
|
|
422
|
+
category: "static-scan",
|
|
423
|
+
attackSurface: "path.join/resolve usage in src/",
|
|
424
|
+
inputExamples: ["path.join($BASE, $USER_INPUT)"],
|
|
425
|
+
expectedBehavior: "Rules flag unvalidated user-controlled path joins; skipped if semgrep unavailable",
|
|
426
|
+
severityIfFailed: "major",
|
|
427
|
+
implementationStatus: "implemented",
|
|
428
|
+
},
|
|
429
|
+
// ---------------------------------------------------------------------------
|
|
430
|
+
// Fuzz smoke tests
|
|
431
|
+
// ---------------------------------------------------------------------------
|
|
432
|
+
{
|
|
433
|
+
id: "fuzz-manifest-reader",
|
|
434
|
+
title: "Manifest JSON reader does not crash on malformed input",
|
|
435
|
+
category: "fuzz-smoke",
|
|
436
|
+
attackSurface: "manifest.json reader",
|
|
437
|
+
inputExamples: ["truncated JSON", "null values", "deeply nested objects", "very long strings"],
|
|
438
|
+
expectedBehavior: "Reader throws a structured parse error or returns a validation message; never a raw crash",
|
|
439
|
+
severityIfFailed: "major",
|
|
440
|
+
implementationStatus: "implemented",
|
|
441
|
+
},
|
|
442
|
+
{
|
|
443
|
+
id: "fuzz-code-graph-reader",
|
|
444
|
+
title: "Code-graph JSON reader does not crash on malformed input",
|
|
445
|
+
category: "fuzz-smoke",
|
|
446
|
+
attackSurface: "code-graph.json reader",
|
|
447
|
+
inputExamples: ["truncated JSON", "nodes: null", "wrong-type arrays"],
|
|
448
|
+
expectedBehavior: "Reader throws a structured parse error or returns a validation message; never a raw crash",
|
|
449
|
+
severityIfFailed: "major",
|
|
450
|
+
implementationStatus: "implemented",
|
|
451
|
+
},
|
|
452
|
+
{
|
|
453
|
+
id: "fuzz-npm-parsers",
|
|
454
|
+
title: "npm audit/ls/outdated parsers do not crash on malformed input",
|
|
455
|
+
category: "fuzz-smoke",
|
|
456
|
+
attackSurface: "parseNpmAudit, parseNpmLs, parseNpmOutdated, parseNpmPackDryRun",
|
|
457
|
+
inputExamples: ["random JSON", "empty string", "deeply nested garbage"],
|
|
458
|
+
expectedBehavior: "All parsers return a result with findings or parseError; never throw",
|
|
459
|
+
severityIfFailed: "major",
|
|
460
|
+
implementationStatus: "implemented",
|
|
461
|
+
},
|
|
462
|
+
{
|
|
463
|
+
id: "fuzz-dot-label-escaping",
|
|
464
|
+
title: "DOT label escaping never crashes on arbitrary string input",
|
|
465
|
+
category: "fuzz-smoke",
|
|
466
|
+
attackSurface: "escapeDotLabel helper",
|
|
467
|
+
inputExamples: ["shell metacharacters", "null bytes", "Unicode", "empty string"],
|
|
468
|
+
expectedBehavior: "escapeDotLabel always returns a string without throwing",
|
|
469
|
+
severityIfFailed: "major",
|
|
470
|
+
implementationStatus: "implemented",
|
|
471
|
+
},
|
|
472
|
+
{
|
|
473
|
+
id: "fuzz-path-normalization",
|
|
474
|
+
title: "Path normalization does not crash on traversal inputs",
|
|
475
|
+
category: "fuzz-smoke",
|
|
476
|
+
attackSurface: "path.normalize / path.resolve",
|
|
477
|
+
inputExamples: ["../../etc/passwd", "../../../Windows/System32", "/etc/hosts", "path\0with\0nulls"],
|
|
478
|
+
expectedBehavior: "path.normalize and path.resolve return strings without throwing for any input",
|
|
479
|
+
severityIfFailed: "minor",
|
|
480
|
+
implementationStatus: "implemented",
|
|
481
|
+
},
|
|
482
|
+
{
|
|
483
|
+
id: "fuzz-source-windowing",
|
|
484
|
+
title: "Source windowing handles edge-case window sizes",
|
|
485
|
+
category: "fuzz-smoke",
|
|
486
|
+
attackSurface: "source retrieval windowing logic",
|
|
487
|
+
inputExamples: ["windowSize: -1", "windowSize: 0", "windowSize: NaN", "windowSize: Infinity"],
|
|
488
|
+
expectedBehavior: "Windowing math clamps to safe values without crashing",
|
|
489
|
+
severityIfFailed: "minor",
|
|
490
|
+
implementationStatus: "implemented",
|
|
491
|
+
},
|
|
492
|
+
// ---------------------------------------------------------------------------
|
|
493
|
+
// Package content tests
|
|
494
|
+
// ---------------------------------------------------------------------------
|
|
495
|
+
{
|
|
496
|
+
id: "no-lab-output-in-tarball",
|
|
497
|
+
title: "lab-output/ must not appear in the npm tarball",
|
|
498
|
+
category: "package-content",
|
|
499
|
+
attackSurface: "npm pack --dry-run file list",
|
|
500
|
+
inputExamples: ["run npm pack --dry-run and inspect file list for lab-output/"],
|
|
501
|
+
expectedBehavior: "No lab-output/ paths appear in the tarball file list",
|
|
502
|
+
severityIfFailed: "blocker",
|
|
503
|
+
implementationStatus: "planned",
|
|
504
|
+
},
|
|
505
|
+
{
|
|
506
|
+
id: "no-my-dev-kit-artifacts-in-tarball",
|
|
507
|
+
title: ".my-dev-kit/ must not appear in the npm tarball",
|
|
508
|
+
category: "package-content",
|
|
509
|
+
attackSurface: "npm pack --dry-run file list",
|
|
510
|
+
inputExamples: ["run npm pack --dry-run and inspect file list for .my-dev-kit/"],
|
|
511
|
+
expectedBehavior: "No .my-dev-kit/ paths appear in the tarball file list",
|
|
512
|
+
severityIfFailed: "blocker",
|
|
513
|
+
implementationStatus: "planned",
|
|
514
|
+
},
|
|
515
|
+
{
|
|
516
|
+
id: "no-env-files-in-tarball",
|
|
517
|
+
title: ".env files must not appear in the npm tarball",
|
|
518
|
+
category: "package-content",
|
|
519
|
+
attackSurface: "npm pack --dry-run file list",
|
|
520
|
+
inputExamples: ["run npm pack --dry-run and inspect for .env, .env.local, .env.production"],
|
|
521
|
+
expectedBehavior: "No .env files appear in the tarball file list",
|
|
522
|
+
severityIfFailed: "blocker",
|
|
523
|
+
implementationStatus: "planned",
|
|
524
|
+
},
|
|
525
|
+
{
|
|
526
|
+
id: "no-private-docs-in-tarball",
|
|
527
|
+
title: "Private planning docs must not appear in the npm tarball",
|
|
528
|
+
category: "package-content",
|
|
529
|
+
attackSurface: "npm pack --dry-run file list",
|
|
530
|
+
inputExamples: ["check for docs/FINAL_BATCH_HANDOFF.txt, docs/coding_generation_guideline.md"],
|
|
531
|
+
expectedBehavior: "Internal planning documents do not appear in the published tarball",
|
|
532
|
+
severityIfFailed: "major",
|
|
533
|
+
implementationStatus: "planned",
|
|
534
|
+
},
|
|
535
|
+
];
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// Core types for the my-dev-kit-lab security-validation framework.
|
|
2
|
+
// These types describe the vocabulary for checks, findings, results, and verdicts
|
|
3
|
+
// used by dependency checks (Prompt 3), CLI adversarial tests (Prompts 4–5),
|
|
4
|
+
// static scans (Prompt 6), fuzz smoke tests (Prompt 7), and the release report (Prompt 8).
|
|
5
|
+
export const SECURITY_SEVERITIES = [
|
|
6
|
+
"blocker",
|
|
7
|
+
"major",
|
|
8
|
+
"minor",
|
|
9
|
+
"informational",
|
|
10
|
+
"skipped",
|
|
11
|
+
];
|
|
12
|
+
export const RELEASE_VERDICTS = [
|
|
13
|
+
"ready-for-release-preparation",
|
|
14
|
+
"not-ready-security-blocker-remains",
|
|
15
|
+
"ready-except-optional-manual-checks",
|
|
16
|
+
"inconclusive-audit-environment-incomplete",
|
|
17
|
+
];
|
|
18
|
+
export const SECURITY_CHECK_STATUSES = [
|
|
19
|
+
"passed",
|
|
20
|
+
"failed",
|
|
21
|
+
"skipped",
|
|
22
|
+
"warning",
|
|
23
|
+
];
|
|
24
|
+
export const SECURITY_CHECK_CATEGORIES = [
|
|
25
|
+
"static-scan",
|
|
26
|
+
"dependency-audit",
|
|
27
|
+
"package-content",
|
|
28
|
+
"cli-adversarial",
|
|
29
|
+
"fuzz-smoke",
|
|
30
|
+
"network-boundary",
|
|
31
|
+
"secret-leakage",
|
|
32
|
+
"artifact-safety",
|
|
33
|
+
"report-generation",
|
|
34
|
+
];
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { resolveLocalProjectTarget } from "../../core/localProjectTarget.js";
|
|
3
|
+
// Resolves and validates a target project path.
|
|
4
|
+
// Throws a descriptive Error (no stack trace wrapper) for bad paths.
|
|
5
|
+
export function resolveValidationTarget(targetPathArg, toolRoot) {
|
|
6
|
+
return resolveLocalProjectTarget(targetPathArg, toolRoot);
|
|
7
|
+
}
|
|
8
|
+
// Returns the prefix to use for report filenames based on target metadata.
|
|
9
|
+
// For self-validation: "v<version>" (preserves existing behavior).
|
|
10
|
+
// For external targets: "<sanitized-name>-v<version>" or "<sanitized-dirname>".
|
|
11
|
+
export function reportFilenamePrefix(target) {
|
|
12
|
+
if (target.isSelf && target.packageVersion) {
|
|
13
|
+
return `v${target.packageVersion}`;
|
|
14
|
+
}
|
|
15
|
+
const rawName = target.packageName ?? path.basename(target.targetRoot);
|
|
16
|
+
// Remove npm scope prefix (@scope/name -> name), then sanitize for filenames.
|
|
17
|
+
const nameWithoutScope = rawName.replace(/^@[^/]+\//, "");
|
|
18
|
+
const sanitized = nameWithoutScope
|
|
19
|
+
.replace(/[^a-z0-9._-]/gi, "-")
|
|
20
|
+
.replace(/-+/g, "-")
|
|
21
|
+
.replace(/^-|-$/g, "");
|
|
22
|
+
const versionSuffix = target.packageVersion ? `-v${target.packageVersion}` : "";
|
|
23
|
+
return `${sanitized}${versionSuffix}`;
|
|
24
|
+
}
|
|
25
|
+
// Returns a human-readable description of the target for report headers.
|
|
26
|
+
export function targetDescription(target) {
|
|
27
|
+
if (target.isSelf)
|
|
28
|
+
return "self (my-dev-kit-lab)";
|
|
29
|
+
const name = target.packageName ?? path.basename(target.targetRoot);
|
|
30
|
+
const version = target.packageVersion ? `@${target.packageVersion}` : "";
|
|
31
|
+
return `${name}${version} (${target.targetRoot})`;
|
|
32
|
+
}
|