@dailephd/my-dev-kit-lab 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +272 -0
- package/benchmarks/contracts/benchmark-project-profiles.json +1199 -0
- package/benchmarks/contracts/todo-behavior.md +70 -0
- package/benchmarks/contracts/todo-benchmark-case.json +227 -0
- package/benchmarks/projects/README.md +34 -0
- package/benchmarks/projects/task-analytics-large-mixed/README.md +1 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/__init__.py +3 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/fixtures.py +6 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/metrics.py +29 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/models.py +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/parser.py +16 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/pipeline.py +9 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/quality.py +8 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/reporting.py +11 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_metrics.py +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_parser.py +15 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_quality.py +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_reporting.py +15 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/package.json +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/index.ts +11 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/analyticsSnapshot.ts +20 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/project.ts +5 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/task.ts +10 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/reporting/buildProjectLeaderboard.ts +7 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/reporting/formatTaskHealthReport.ts +13 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/buildAnalyticsSnapshot.ts +39 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/completeTask.ts +10 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/createTask.ts +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/listTasksByProject.ts +6 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/store/projectStore.ts +20 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/store/taskStore.ts +44 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/validation/projectValidation.ts +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/validation/taskValidation.ts +18 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/buildAnalyticsSnapshot.test.ts +48 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/completeTask.test.ts +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/createTask.test.ts +31 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/listTasksByProject.test.ts +18 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/reporting.test.ts +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tsconfig.json +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/vitest.config.ts +5 -0
- package/benchmarks/projects/task-workflow-medium-ts/README.md +1 -0
- package/benchmarks/projects/task-workflow-medium-ts/package.json +12 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/index.ts +9 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/models/project.ts +6 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/models/task.ts +39 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/completeTask.ts +15 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/createTask.ts +26 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/filterTasks.ts +17 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/importTasks.ts +33 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/summarizeTasks.ts +30 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/store/taskStore.ts +76 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/utils/deterministicId.ts +3 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/validation/taskValidation.ts +45 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/completeTask.test.ts +16 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/createTask.test.ts +21 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/filterTasks.test.ts +18 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/importTasks.test.ts +22 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/summarizeTasks.test.ts +29 -0
- package/benchmarks/projects/task-workflow-medium-ts/tsconfig.json +12 -0
- package/benchmarks/projects/task-workflow-medium-ts/vitest.config.ts +5 -0
- package/benchmarks/projects/todo-js/README.md +3 -0
- package/benchmarks/projects/todo-js/package.json +11 -0
- package/benchmarks/projects/todo-js/src/index.js +2 -0
- package/benchmarks/projects/todo-js/src/taskService.js +37 -0
- package/benchmarks/projects/todo-js/src/taskStore.js +28 -0
- package/benchmarks/projects/todo-js/tests/taskService.test.js +45 -0
- package/benchmarks/projects/todo-js/vitest.config.js +5 -0
- package/benchmarks/projects/todo-mixed-ts-py/README.md +3 -0
- package/benchmarks/projects/todo-mixed-ts-py/package.json +13 -0
- package/benchmarks/projects/todo-mixed-ts-py/python/task_service.py +76 -0
- package/benchmarks/projects/todo-mixed-ts-py/src/taskCli.ts +38 -0
- package/benchmarks/projects/todo-mixed-ts-py/tests/mixedBoundary.test.ts +18 -0
- package/benchmarks/projects/todo-mixed-ts-py/tsconfig.json +12 -0
- package/benchmarks/projects/todo-mixed-ts-py/vitest.config.ts +5 -0
- package/benchmarks/projects/todo-python/README.md +3 -0
- package/benchmarks/projects/todo-python/src/__init__.py +4 -0
- package/benchmarks/projects/todo-python/src/task_service.py +32 -0
- package/benchmarks/projects/todo-python/src/task_store.py +28 -0
- package/benchmarks/projects/todo-python/tests/test_task_service.py +52 -0
- package/benchmarks/projects/todo-ts/README.md +3 -0
- package/benchmarks/projects/todo-ts/package.json +12 -0
- package/benchmarks/projects/todo-ts/src/index.ts +2 -0
- package/benchmarks/projects/todo-ts/src/taskService.ts +41 -0
- package/benchmarks/projects/todo-ts/src/taskStore.ts +34 -0
- package/benchmarks/projects/todo-ts/tests/taskService.test.ts +45 -0
- package/benchmarks/projects/todo-ts/tsconfig.json +12 -0
- package/benchmarks/projects/todo-ts/vitest.config.ts +5 -0
- package/dist/scripts/build-gallery.js +3 -0
- package/dist/scripts/capture-demo-report.js +3 -0
- package/dist/scripts/evaluate-token-savings.js +2 -0
- package/dist/scripts/experiments/describeExperiment.js +143 -0
- package/dist/scripts/experiments/listExperiments.js +44 -0
- package/dist/scripts/experiments/runExperiment.js +199 -0
- package/dist/scripts/generate-experiment-plots.js +3 -0
- package/dist/scripts/generate-prompt-variants.js +2 -0
- package/dist/scripts/render-experiment-report.js +2 -0
- package/dist/scripts/run-agent-prompt.js +2 -0
- package/dist/scripts/run-controlled-experiment.js +2 -0
- package/dist/scripts/run-final-demo.js +3 -0
- package/dist/scripts/run-lab-demo.js +5 -0
- package/dist/scripts/run-visualization-demos.js +3 -0
- package/dist/scripts/security/runCodeql.js +57 -0
- package/dist/scripts/security/runDependencyChecks.js +57 -0
- package/dist/scripts/security/runFuzzSmoke.js +29 -0
- package/dist/scripts/security/runPackageChecks.js +56 -0
- package/dist/scripts/security/runSemgrep.js +63 -0
- package/dist/scripts/security/validate.js +117 -0
- package/dist/scripts/verify-benchmarks.js +202 -0
- package/dist/src/agents/adapters/claudeAdapter.js +37 -0
- package/dist/src/agents/adapters/codexAdapter.js +110 -0
- package/dist/src/agents/adapters/fakeAgentAdapter.js +101 -0
- package/dist/src/agents/agentRegistry.js +21 -0
- package/dist/src/agents/index.js +7 -0
- package/dist/src/agents/parseAgentTokenUsage.js +137 -0
- package/dist/src/agents/runAgentPrompt.js +38 -0
- package/dist/src/agents/types.js +1 -0
- package/dist/src/commands/buildGalleryCommand.js +56 -0
- package/dist/src/commands/captureDemoReport.js +116 -0
- package/dist/src/commands/evaluateTokenSavings.js +175 -0
- package/dist/src/commands/generateExperimentPlotsCommand.js +38 -0
- package/dist/src/commands/generatePromptVariants.js +67 -0
- package/dist/src/commands/renderExperimentReportCommand.js +131 -0
- package/dist/src/commands/runAgentPromptCommand.js +132 -0
- package/dist/src/commands/runControlledExperimentCommand.js +174 -0
- package/dist/src/commands/runFinalDemoCommand.js +123 -0
- package/dist/src/commands/runLabDemo.js +62 -0
- package/dist/src/commands/runVisualizationDemosCommand.js +67 -0
- package/dist/src/core/commandLine.js +59 -0
- package/dist/src/core/countTokens.js +8 -0
- package/dist/src/core/fileGlobs.js +100 -0
- package/dist/src/core/localProjectTarget.js +75 -0
- package/dist/src/core/pathSafety.js +19 -0
- package/dist/src/core/pythonCommand.js +30 -0
- package/dist/src/core/resolveCommand.js +110 -0
- package/dist/src/core/runMeasuredCommand.js +143 -0
- package/dist/src/evaluation/benchmarkMetadata.js +207 -0
- package/dist/src/evaluation/buildExperimentMatrix.js +75 -0
- package/dist/src/evaluation/classifyAgentRunOutcome.js +40 -0
- package/dist/src/evaluation/compareExperimentRuns.js +79 -0
- package/dist/src/evaluation/compareTokenSavings.js +47 -0
- package/dist/src/evaluation/controlledExperimentTypes.js +1 -0
- package/dist/src/evaluation/index.js +18 -0
- package/dist/src/evaluation/parseAgentAnswer.js +230 -0
- package/dist/src/evaluation/projectComplexity.js +126 -0
- package/dist/src/evaluation/projectFileTree.js +83 -0
- package/dist/src/evaluation/readEvaluationCases.js +59 -0
- package/dist/src/evaluation/renderTokenSavingsReportInput.js +55 -0
- package/dist/src/evaluation/runControlledExperiment.js +158 -0
- package/dist/src/evaluation/runMyDevKitRetrieval.js +197 -0
- package/dist/src/evaluation/runRawFullFileBaseline.js +31 -0
- package/dist/src/evaluation/scoreCorrectness.js +127 -0
- package/dist/src/evaluation/types.js +1 -0
- package/dist/src/evaluation/writeExperimentArtifacts.js +104 -0
- package/dist/src/evaluation/writeTokenSavingsArtifacts.js +57 -0
- package/dist/src/experiments/config.js +24 -0
- package/dist/src/experiments/defaultRegistry.js +7 -0
- package/dist/src/experiments/errors.js +18 -0
- package/dist/src/experiments/index.js +9 -0
- package/dist/src/experiments/outputPaths.js +25 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/config.js +37 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/index.js +3 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +83 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +260 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/registry.js +43 -0
- package/dist/src/experiments/results.js +48 -0
- package/dist/src/experiments/runner.js +181 -0
- package/dist/src/experiments/target.js +8 -0
- package/dist/src/experiments/types.js +1 -0
- package/dist/src/gallery/index.js +2 -0
- package/dist/src/gallery/types.js +1 -0
- package/dist/src/gallery/writeGalleryManifest.js +214 -0
- package/dist/src/index.js +12 -0
- package/dist/src/plots/buildExperimentPlotData.js +137 -0
- package/dist/src/plots/index.js +4 -0
- package/dist/src/plots/renderSvgChart.js +82 -0
- package/dist/src/plots/types.js +1 -0
- package/dist/src/plots/writePlotArtifacts.js +46 -0
- package/dist/src/prompts/buildPromptContext.js +68 -0
- package/dist/src/prompts/generateMyDevKitPrompt.js +106 -0
- package/dist/src/prompts/generatePromptVariants.js +36 -0
- package/dist/src/prompts/generateRawFullFilePrompt.js +97 -0
- package/dist/src/prompts/index.js +7 -0
- package/dist/src/prompts/measurePromptComplexity.js +41 -0
- package/dist/src/prompts/types.js +1 -0
- package/dist/src/prompts/writePromptArtifacts.js +43 -0
- package/dist/src/report/buildExperimentReportInput.js +339 -0
- package/dist/src/report/experimentReportTypes.js +1 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +153 -0
- package/dist/src/report/experiments/experimentReportModel.js +1 -0
- package/dist/src/report/experiments/index.js +4 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +133 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +30 -0
- package/dist/src/report/index.js +8 -0
- package/dist/src/report/renderExperimentHtmlReport.js +354 -0
- package/dist/src/report/renderHtmlReport.js +103 -0
- package/dist/src/report/types.js +10 -0
- package/dist/src/report/writeExperimentReportArtifacts.js +38 -0
- package/dist/src/report/writeReportArtifacts.js +39 -0
- package/dist/src/screenshot/captureReportScreenshot.js +75 -0
- package/dist/src/screenshot/index.js +2 -0
- package/dist/src/screenshot/types.js +1 -0
- package/dist/src/securityValidation/artifacts.js +15 -0
- package/dist/src/securityValidation/cliAdversarial/adversarialCliConfig.js +38 -0
- package/dist/src/securityValidation/cliAdversarial/dataVolumeChecks.js +194 -0
- package/dist/src/securityValidation/cliAdversarial/jsonStdoutChecks.js +359 -0
- package/dist/src/securityValidation/cliAdversarial/malformedArtifactChecks.js +284 -0
- package/dist/src/securityValidation/cliAdversarial/malformedArtifactFixtures.js +79 -0
- package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +431 -0
- package/dist/src/securityValidation/cliAdversarial/pathCases.js +144 -0
- package/dist/src/securityValidation/cliAdversarial/readOnlyBoundaryChecks.js +294 -0
- package/dist/src/securityValidation/cliAdversarial/runAdversarialCheck.js +149 -0
- package/dist/src/securityValidation/cliAdversarial/subprocessSafetyChecks.js +214 -0
- package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +160 -0
- package/dist/src/securityValidation/commandRunner.js +136 -0
- package/dist/src/securityValidation/config.js +39 -0
- package/dist/src/securityValidation/dependencies/parseNpmAudit.js +115 -0
- package/dist/src/securityValidation/dependencies/parseNpmLs.js +71 -0
- package/dist/src/securityValidation/dependencies/parseNpmOutdated.js +41 -0
- package/dist/src/securityValidation/dependencies/runDependencyChecks.js +239 -0
- package/dist/src/securityValidation/dependencies/runOsvScanner.js +43 -0
- package/dist/src/securityValidation/fuzz/fuzzHarness.js +61 -0
- package/dist/src/securityValidation/fuzz/fuzzTargets.js +204 -0
- package/dist/src/securityValidation/fuzz/randomInput.js +0 -0
- package/dist/src/securityValidation/index.js +34 -0
- package/dist/src/securityValidation/packageChecks/forbiddenPackageContents.js +67 -0
- package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +56 -0
- package/dist/src/securityValidation/packageChecks/runPackageChecks.js +88 -0
- package/dist/src/securityValidation/report/renderSecurityReport.js +248 -0
- package/dist/src/securityValidation/report/securityReportTypes.js +1 -0
- package/dist/src/securityValidation/staticScans/codeql.js +66 -0
- package/dist/src/securityValidation/staticScans/semgrep.js +180 -0
- package/dist/src/securityValidation/testMatrix.js +535 -0
- package/dist/src/securityValidation/types.js +34 -0
- package/dist/src/securityValidation/validate/resolveTarget.js +32 -0
- package/dist/src/securityValidation/validate/runSecurityValidation.js +169 -0
- package/dist/src/securityValidation/validate/verdict.js +73 -0
- package/dist/src/visualizationDemos/buildMyDevKitVisualizationCommands.js +59 -0
- package/dist/src/visualizationDemos/index.js +4 -0
- package/dist/src/visualizationDemos/runVisualizationDemos.js +82 -0
- package/dist/src/visualizationDemos/types.js +1 -0
- package/dist/src/visualizationDemos/writeVisualizationDemoArtifacts.js +25 -0
- package/docs/METRICS.md +286 -0
- package/examples/demo-report-input.json +78 -0
- package/examples/lab-demo-cases.json +35 -0
- package/examples/real-agent-campaign-cases.json +118 -0
- package/examples/token-savings-cases.json +122 -0
- package/package.json +91 -0
- package/tests/fixtures/fake-adversarial-cli.js +152 -0
- package/tests/fixtures/fake-my-dev-kit-cli.js +83 -0
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { TaskService } from "../src/taskService.js";
|
|
3
|
+
|
|
4
|
+
describe("todo-ts TaskService", () => {
|
|
5
|
+
it("creates tasks with deterministic ids", () => {
|
|
6
|
+
const service = new TaskService();
|
|
7
|
+
expect(service.createTask("First").id).toBe("task-1");
|
|
8
|
+
expect(service.createTask("Second").id).toBe("task-2");
|
|
9
|
+
});
|
|
10
|
+
|
|
11
|
+
it("completes tasks", () => {
|
|
12
|
+
const service = new TaskService();
|
|
13
|
+
const created = service.createTask("Ship benchmark");
|
|
14
|
+
const completed = service.completeTask(created.id);
|
|
15
|
+
expect(completed.completed).toBe(true);
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it("lists all tasks", () => {
|
|
19
|
+
const service = new TaskService();
|
|
20
|
+
service.createTask("One");
|
|
21
|
+
service.createTask("Two");
|
|
22
|
+
expect(service.listTasks()).toHaveLength(2);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it("lists open tasks", () => {
|
|
26
|
+
const service = new TaskService();
|
|
27
|
+
const one = service.createTask("One");
|
|
28
|
+
service.createTask("Two");
|
|
29
|
+
service.completeTask(one.id);
|
|
30
|
+
expect(service.listOpenTasks().map((task) => task.title)).toEqual(["Two"]);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it("summarizes task counts", () => {
|
|
34
|
+
const service = new TaskService();
|
|
35
|
+
const one = service.createTask("One");
|
|
36
|
+
service.createTask("Two");
|
|
37
|
+
service.completeTask(one.id);
|
|
38
|
+
expect(service.summarizeTasks()).toEqual({ total: 2, open: 1, completed: 1 });
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("rejects empty task titles", () => {
|
|
42
|
+
const service = new TaskService();
|
|
43
|
+
expect(() => service.createTask(" ")).toThrowError("Task title must not be empty.");
|
|
44
|
+
});
|
|
45
|
+
});
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "ES2022",
|
|
4
|
+
"module": "NodeNext",
|
|
5
|
+
"moduleResolution": "NodeNext",
|
|
6
|
+
"strict": true,
|
|
7
|
+
"esModuleInterop": true,
|
|
8
|
+
"skipLibCheck": true,
|
|
9
|
+
"types": ["vitest/globals"]
|
|
10
|
+
},
|
|
11
|
+
"include": ["src/**/*.ts", "tests/**/*.ts"]
|
|
12
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { createDefaultExperimentPluginRegistry } from "../../src/experiments/index.js";
|
|
2
|
+
async function main(argv) {
|
|
3
|
+
try {
|
|
4
|
+
const args = parseDescribeArgs(argv);
|
|
5
|
+
const registry = createDefaultExperimentPluginRegistry();
|
|
6
|
+
const plugin = registry.get(args.experimentId);
|
|
7
|
+
if (args.json) {
|
|
8
|
+
console.log(JSON.stringify(buildDescription(plugin), null, 2));
|
|
9
|
+
return 0;
|
|
10
|
+
}
|
|
11
|
+
printDescription(plugin);
|
|
12
|
+
return 0;
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
if (process.env.DEBUG) {
|
|
16
|
+
console.error(error);
|
|
17
|
+
}
|
|
18
|
+
else {
|
|
19
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
20
|
+
}
|
|
21
|
+
return 1;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
function parseDescribeArgs(argv) {
|
|
25
|
+
let experimentId = "";
|
|
26
|
+
let json = false;
|
|
27
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
28
|
+
const arg = argv[index];
|
|
29
|
+
if (arg === "--experiment") {
|
|
30
|
+
experimentId = argv[++index] ?? "";
|
|
31
|
+
}
|
|
32
|
+
else if (arg === "--json") {
|
|
33
|
+
json = true;
|
|
34
|
+
}
|
|
35
|
+
else {
|
|
36
|
+
throw new Error(`Unknown argument: ${arg}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
if (!experimentId) {
|
|
40
|
+
throw new Error("Usage: --experiment <id> [--json]");
|
|
41
|
+
}
|
|
42
|
+
return { experimentId, json };
|
|
43
|
+
}
|
|
44
|
+
function buildDescription(plugin) {
|
|
45
|
+
return {
|
|
46
|
+
metadata: plugin.metadata,
|
|
47
|
+
purpose: describePurpose(plugin),
|
|
48
|
+
supportedVariants: readSupportedVariants(plugin),
|
|
49
|
+
requiredConfigFields: readConfigFields(plugin, true),
|
|
50
|
+
optionalConfigFields: readConfigFields(plugin, false),
|
|
51
|
+
targetBehavior: describeTargetBehavior(plugin),
|
|
52
|
+
expectedReports: describeExpectedReports(plugin),
|
|
53
|
+
examples: describeExamples(plugin),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
function printDescription(plugin) {
|
|
57
|
+
const description = buildDescription(plugin);
|
|
58
|
+
console.log(`${plugin.metadata.name}`);
|
|
59
|
+
console.log("");
|
|
60
|
+
console.log(`ID: ${plugin.metadata.id}`);
|
|
61
|
+
console.log(`Description: ${plugin.metadata.description}`);
|
|
62
|
+
console.log(`Status: ${plugin.metadata.status}`);
|
|
63
|
+
console.log(`Schema version: ${plugin.metadata.schemaVersion}`);
|
|
64
|
+
console.log(`Supported targets: ${plugin.metadata.supportedTargets.join(", ")}`);
|
|
65
|
+
console.log(`Supported outputs: ${plugin.metadata.supportedOutputs.join(", ")}`);
|
|
66
|
+
console.log("");
|
|
67
|
+
console.log(`Purpose: ${description.purpose}`);
|
|
68
|
+
console.log(`Supported variants: ${description.supportedVariants.join(", ") || "not declared"}`);
|
|
69
|
+
console.log("");
|
|
70
|
+
printConfigSection("Required config fields", description.requiredConfigFields);
|
|
71
|
+
printConfigSection("Optional config fields", description.optionalConfigFields);
|
|
72
|
+
console.log("");
|
|
73
|
+
console.log(`Target behavior: ${description.targetBehavior}`);
|
|
74
|
+
console.log(`Expected reports: ${description.expectedReports}`);
|
|
75
|
+
console.log("");
|
|
76
|
+
console.log("Examples:");
|
|
77
|
+
for (const example of description.examples) {
|
|
78
|
+
console.log(` ${example}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
function printConfigSection(title, fields) {
|
|
82
|
+
console.log(`${title}:`);
|
|
83
|
+
if (fields.length === 0) {
|
|
84
|
+
console.log(" none");
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
for (const field of fields) {
|
|
88
|
+
const defaultSuffix = field.defaultValue === undefined ? "" : ` default=${JSON.stringify(field.defaultValue)}`;
|
|
89
|
+
const typeSuffix = field.type ? ` (${field.type})` : "";
|
|
90
|
+
console.log(` ${field.name}${typeSuffix}${defaultSuffix}${field.description ? ` - ${field.description}` : ""}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function readConfigFields(plugin, required) {
|
|
94
|
+
return (plugin.configDefinition?.fields ?? [])
|
|
95
|
+
.filter((field) => Boolean(field.required) === required)
|
|
96
|
+
.map((field) => ({
|
|
97
|
+
...field,
|
|
98
|
+
defaultValue: readDefaultValue(plugin.defaultConfig, field.name),
|
|
99
|
+
}));
|
|
100
|
+
}
|
|
101
|
+
function readSupportedVariants(plugin) {
|
|
102
|
+
const strategies = readArrayField(plugin.defaultConfig, "strategies");
|
|
103
|
+
return strategies.filter((value) => typeof value === "string");
|
|
104
|
+
}
|
|
105
|
+
function readDefaultValue(config, key) {
|
|
106
|
+
if (!config || typeof config !== "object" || Array.isArray(config)) {
|
|
107
|
+
return undefined;
|
|
108
|
+
}
|
|
109
|
+
return config[key];
|
|
110
|
+
}
|
|
111
|
+
function readArrayField(value, key) {
|
|
112
|
+
const field = readDefaultValue(value, key);
|
|
113
|
+
return Array.isArray(field) ? field : [];
|
|
114
|
+
}
|
|
115
|
+
function describePurpose(plugin) {
|
|
116
|
+
if (plugin.metadata.id === "context-strategy-comparison") {
|
|
117
|
+
return "Compare raw full-file prompts with my-dev-kit-guided retrieval prompts using the existing controlled experiment workflow.";
|
|
118
|
+
}
|
|
119
|
+
return plugin.metadata.description;
|
|
120
|
+
}
|
|
121
|
+
function describeTargetBehavior(plugin) {
|
|
122
|
+
const supportsExternal = plugin.metadata.supportedTargets.includes("external-local");
|
|
123
|
+
if (supportsExternal) {
|
|
124
|
+
return "Runs against the current lab repository when --target is omitted, or against an explicit local target project when --target <path> is provided. Outputs stay under lab-controlled output directories by default.";
|
|
125
|
+
}
|
|
126
|
+
return "Runs against the current lab repository.";
|
|
127
|
+
}
|
|
128
|
+
function describeExpectedReports(plugin) {
|
|
129
|
+
if (plugin.metadata.supportedOutputs.includes("html") && plugin.metadata.supportedOutputs.includes("json")) {
|
|
130
|
+
return "Writes plugin-aware JSON and HTML reports with plugin, target, variant, case, metric, artifact, warning, skip, and failure metadata.";
|
|
131
|
+
}
|
|
132
|
+
return `Writes supported outputs: ${plugin.metadata.supportedOutputs.join(", ")}.`;
|
|
133
|
+
}
|
|
134
|
+
function describeExamples(plugin) {
|
|
135
|
+
return [
|
|
136
|
+
`npm run experiment:describe -- --experiment ${plugin.metadata.id}`,
|
|
137
|
+
`npm run experiment:run -- --experiment ${plugin.metadata.id} --agents fake-agent --complexities short`,
|
|
138
|
+
`npm run experiment:run -- --experiment ${plugin.metadata.id} --target "Z:\\Users\\newuser\\Projects\\my-dev-kit-v1" --agents fake-agent --complexities short --no-screenshot`,
|
|
139
|
+
];
|
|
140
|
+
}
|
|
141
|
+
if (process.argv[1] && import.meta.url.endsWith(process.argv[1].replaceAll("\\", "/"))) {
|
|
142
|
+
process.exitCode = await main(process.argv.slice(2));
|
|
143
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { createDefaultExperimentPluginRegistry } from "../../src/experiments/index.js";
|
|
2
|
+
async function main(argv) {
|
|
3
|
+
const json = argv.includes("--json");
|
|
4
|
+
const registry = createDefaultExperimentPluginRegistry();
|
|
5
|
+
const plugins = registry.list().map((metadata) => {
|
|
6
|
+
const plugin = registry.get(metadata.id);
|
|
7
|
+
return toListedExperiment(metadata, plugin);
|
|
8
|
+
});
|
|
9
|
+
if (json) {
|
|
10
|
+
console.log(JSON.stringify({ experiments: plugins }, null, 2));
|
|
11
|
+
return 0;
|
|
12
|
+
}
|
|
13
|
+
console.log("Registered experiment plugins");
|
|
14
|
+
console.log("");
|
|
15
|
+
for (const plugin of plugins) {
|
|
16
|
+
console.log(`${plugin.id}`);
|
|
17
|
+
console.log(` Name: ${plugin.name}`);
|
|
18
|
+
console.log(` Description: ${plugin.description}`);
|
|
19
|
+
console.log(` Status: ${plugin.status}`);
|
|
20
|
+
console.log(` Supported variants: ${plugin.supportedVariants.join(", ") || "not declared"}`);
|
|
21
|
+
console.log(` Outputs: ${plugin.supportedOutputs.join(", ")}`);
|
|
22
|
+
}
|
|
23
|
+
return 0;
|
|
24
|
+
}
|
|
25
|
+
function toListedExperiment(metadata, plugin) {
|
|
26
|
+
return {
|
|
27
|
+
...metadata,
|
|
28
|
+
supportedVariants: readSupportedVariants(plugin),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
function readSupportedVariants(plugin) {
|
|
32
|
+
const strategies = readArrayField(plugin.defaultConfig, "strategies");
|
|
33
|
+
return strategies.filter((value) => typeof value === "string");
|
|
34
|
+
}
|
|
35
|
+
function readArrayField(value, key) {
|
|
36
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
37
|
+
return [];
|
|
38
|
+
}
|
|
39
|
+
const field = value[key];
|
|
40
|
+
return Array.isArray(field) ? field : [];
|
|
41
|
+
}
|
|
42
|
+
if (process.argv[1] && import.meta.url.endsWith(process.argv[1].replaceAll("\\", "/"))) {
|
|
43
|
+
process.exitCode = await main(process.argv.slice(2));
|
|
44
|
+
}
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import { parseAgentCommandTemplate } from "../../src/agents/index.js";
|
|
3
|
+
import { parseAgentId } from "../../src/agents/agentRegistry.js";
|
|
4
|
+
import { readBenchmarkProjectProfiles, readEvaluationCases } from "../../src/evaluation/index.js";
|
|
5
|
+
import { createDefaultExperimentPluginRegistry, runExperiment } from "../../src/experiments/index.js";
|
|
6
|
+
import { contextStrategyComparisonPlugin } from "../../src/experiments/plugins/contextStrategyComparison/index.js";
|
|
7
|
+
import { parsePromptComplexityLevel, parsePromptStrategy } from "../../src/prompts/index.js";
|
|
8
|
+
import { writePluginExperimentReports } from "../../src/report/index.js";
|
|
9
|
+
async function main(argv) {
|
|
10
|
+
try {
|
|
11
|
+
const args = parseRunExperimentArgs(argv);
|
|
12
|
+
const toolRoot = process.cwd();
|
|
13
|
+
const registry = createDefaultExperimentPluginRegistry();
|
|
14
|
+
const inputs = await loadPluginInputs(args, toolRoot);
|
|
15
|
+
const result = await runExperiment({
|
|
16
|
+
pluginId: args.experimentId,
|
|
17
|
+
registry,
|
|
18
|
+
targetPath: args.targetPath,
|
|
19
|
+
outputRoot: args.outDir,
|
|
20
|
+
config: args.config,
|
|
21
|
+
inputs,
|
|
22
|
+
toolRoot,
|
|
23
|
+
});
|
|
24
|
+
const reports = await writePluginExperimentReports({
|
|
25
|
+
run: result,
|
|
26
|
+
plugin: registry.describe(result.pluginId),
|
|
27
|
+
outputRoot: String(result.metadata?.outputRoot ?? ""),
|
|
28
|
+
});
|
|
29
|
+
console.log([
|
|
30
|
+
`Experiment: ${result.pluginId}`,
|
|
31
|
+
`Run ID: ${result.runId}`,
|
|
32
|
+
`Status: ${result.status}`,
|
|
33
|
+
`Mode: ${result.target.isSelf ? "self" : "external target"}`,
|
|
34
|
+
`Tool root: ${result.target.toolRoot}`,
|
|
35
|
+
`Target root: ${result.target.targetRoot}`,
|
|
36
|
+
`Output: ${String(result.metadata?.outputRoot ?? "")}`,
|
|
37
|
+
`Report JSON: ${reports.outputPaths.jsonPath}`,
|
|
38
|
+
`Report HTML: ${reports.outputPaths.htmlPath}`,
|
|
39
|
+
].join("\n"));
|
|
40
|
+
return result.status === "failed" ? 1 : 0;
|
|
41
|
+
}
|
|
42
|
+
catch (error) {
|
|
43
|
+
if (process.env.DEBUG) {
|
|
44
|
+
console.error(error);
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
console.error(error instanceof Error ? error.message : String(error));
|
|
48
|
+
}
|
|
49
|
+
return 1;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
export function parseRunExperimentArgs(argv) {
|
|
53
|
+
let experimentId = "";
|
|
54
|
+
let targetPath;
|
|
55
|
+
let outDir;
|
|
56
|
+
let casesPath;
|
|
57
|
+
let projectProfilesPath;
|
|
58
|
+
const caseIds = [];
|
|
59
|
+
const benchmarkProjects = [];
|
|
60
|
+
let agents;
|
|
61
|
+
let strategies;
|
|
62
|
+
let complexityLevels;
|
|
63
|
+
let timeoutMs;
|
|
64
|
+
let maxRuns;
|
|
65
|
+
let continueOnFailure;
|
|
66
|
+
let requireAgents;
|
|
67
|
+
let includeRealAgents;
|
|
68
|
+
const commandTemplates = {};
|
|
69
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
70
|
+
const arg = argv[index];
|
|
71
|
+
if (arg === "--experiment") {
|
|
72
|
+
experimentId = readRequiredValue(argv, ++index, "--experiment");
|
|
73
|
+
}
|
|
74
|
+
else if (arg === "--target") {
|
|
75
|
+
targetPath = readRequiredValue(argv, ++index, "--target");
|
|
76
|
+
}
|
|
77
|
+
else if (arg === "--out") {
|
|
78
|
+
outDir = readRequiredValue(argv, ++index, "--out");
|
|
79
|
+
}
|
|
80
|
+
else if (arg === "--cases") {
|
|
81
|
+
casesPath = readRequiredValue(argv, ++index, "--cases");
|
|
82
|
+
}
|
|
83
|
+
else if (arg === "--project-profiles") {
|
|
84
|
+
projectProfilesPath = readRequiredValue(argv, ++index, "--project-profiles");
|
|
85
|
+
}
|
|
86
|
+
else if (arg === "--case") {
|
|
87
|
+
caseIds.push(...splitList(readRequiredValue(argv, ++index, "--case")));
|
|
88
|
+
}
|
|
89
|
+
else if (arg === "--benchmark-project") {
|
|
90
|
+
benchmarkProjects.push(...splitList(readRequiredValue(argv, ++index, "--benchmark-project")));
|
|
91
|
+
}
|
|
92
|
+
else if (arg === "--agents") {
|
|
93
|
+
agents = splitList(readRequiredValue(argv, ++index, "--agents")).map((value) => parseAgentId(value));
|
|
94
|
+
}
|
|
95
|
+
else if (arg === "--strategies") {
|
|
96
|
+
strategies = splitList(readRequiredValue(argv, ++index, "--strategies")).map(parsePromptStrategy);
|
|
97
|
+
}
|
|
98
|
+
else if (arg === "--complexities") {
|
|
99
|
+
complexityLevels = splitList(readRequiredValue(argv, ++index, "--complexities")).map(parsePromptComplexityLevel);
|
|
100
|
+
}
|
|
101
|
+
else if (arg === "--timeout-ms") {
|
|
102
|
+
timeoutMs = parsePositiveInteger("--timeout-ms", readRequiredValue(argv, ++index, "--timeout-ms"));
|
|
103
|
+
}
|
|
104
|
+
else if (arg === "--max-runs") {
|
|
105
|
+
maxRuns = parsePositiveInteger("--max-runs", readRequiredValue(argv, ++index, "--max-runs"));
|
|
106
|
+
}
|
|
107
|
+
else if (arg === "--continue-on-failure") {
|
|
108
|
+
continueOnFailure = true;
|
|
109
|
+
}
|
|
110
|
+
else if (arg === "--no-continue-on-failure") {
|
|
111
|
+
continueOnFailure = false;
|
|
112
|
+
}
|
|
113
|
+
else if (arg === "--require-agents") {
|
|
114
|
+
requireAgents = true;
|
|
115
|
+
}
|
|
116
|
+
else if (arg === "--include-real-agents") {
|
|
117
|
+
includeRealAgents = true;
|
|
118
|
+
}
|
|
119
|
+
else if (arg === "--command-template-codex") {
|
|
120
|
+
commandTemplates.codex = parseAgentCommandTemplate(readRequiredValue(argv, ++index, "--command-template-codex"));
|
|
121
|
+
}
|
|
122
|
+
else if (arg === "--command-template-claude") {
|
|
123
|
+
commandTemplates.claude = parseAgentCommandTemplate(readRequiredValue(argv, ++index, "--command-template-claude"));
|
|
124
|
+
}
|
|
125
|
+
else if (arg === "--no-screenshot") {
|
|
126
|
+
// The plugin-aware report path does not capture screenshots yet; accept this
|
|
127
|
+
// flag so smoke commands can share the legacy demo option set.
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
throw new Error(`Unknown argument: ${arg}`);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (!experimentId) {
|
|
134
|
+
throw new Error("Usage: --experiment <id> [--target <path>] [--out <directory>]");
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
experimentId,
|
|
138
|
+
targetPath,
|
|
139
|
+
outDir,
|
|
140
|
+
config: withoutUndefined({
|
|
141
|
+
casesPath,
|
|
142
|
+
projectProfilesPath,
|
|
143
|
+
caseIds: caseIds.length > 0 ? caseIds : undefined,
|
|
144
|
+
benchmarkProjects: benchmarkProjects.length > 0 ? benchmarkProjects : undefined,
|
|
145
|
+
agents,
|
|
146
|
+
strategies,
|
|
147
|
+
complexityLevels,
|
|
148
|
+
timeoutMs,
|
|
149
|
+
maxRuns,
|
|
150
|
+
continueOnFailure,
|
|
151
|
+
requireAgents,
|
|
152
|
+
includeRealAgents,
|
|
153
|
+
commandTemplates: Object.keys(commandTemplates).length > 0 ? commandTemplates : undefined,
|
|
154
|
+
}),
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
async function loadPluginInputs(args, toolRoot) {
|
|
158
|
+
if (args.experimentId !== contextStrategyComparisonPlugin.metadata.id) {
|
|
159
|
+
return undefined;
|
|
160
|
+
}
|
|
161
|
+
const validation = contextStrategyComparisonPlugin.validateConfig(args.config);
|
|
162
|
+
if (!validation.valid || !validation.config) {
|
|
163
|
+
throw new Error(`Invalid context strategy comparison config: ${validation.errors.join("; ")}`);
|
|
164
|
+
}
|
|
165
|
+
const projectProfilesPath = path.resolve(toolRoot, validation.config.projectProfilesPath ?? "benchmarks/contracts/benchmark-project-profiles.json");
|
|
166
|
+
const casesPath = path.resolve(toolRoot, validation.config.casesPath);
|
|
167
|
+
const projectProfiles = await readBenchmarkProjectProfiles(projectProfilesPath, toolRoot);
|
|
168
|
+
const cases = await readEvaluationCases(casesPath, toolRoot, {
|
|
169
|
+
projectProfiles,
|
|
170
|
+
requireProjectProfileRef: true,
|
|
171
|
+
});
|
|
172
|
+
return { cases, projectProfiles, env: process.env };
|
|
173
|
+
}
|
|
174
|
+
function splitList(value) {
|
|
175
|
+
return value
|
|
176
|
+
.split(",")
|
|
177
|
+
.map((item) => item.trim())
|
|
178
|
+
.filter(Boolean);
|
|
179
|
+
}
|
|
180
|
+
function readRequiredValue(argv, index, label) {
|
|
181
|
+
const value = argv[index];
|
|
182
|
+
if (!value || value.startsWith("--")) {
|
|
183
|
+
throw new Error(`${label} requires a value.`);
|
|
184
|
+
}
|
|
185
|
+
return value;
|
|
186
|
+
}
|
|
187
|
+
function parsePositiveInteger(label, value) {
|
|
188
|
+
const parsed = Number(value);
|
|
189
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
190
|
+
throw new Error(`${label} must be a positive integer.`);
|
|
191
|
+
}
|
|
192
|
+
return parsed;
|
|
193
|
+
}
|
|
194
|
+
function withoutUndefined(value) {
|
|
195
|
+
return Object.fromEntries(Object.entries(value).filter(([, entryValue]) => entryValue !== undefined));
|
|
196
|
+
}
|
|
197
|
+
if (process.argv[1] && import.meta.url.endsWith(process.argv[1].replaceAll("\\", "/"))) {
|
|
198
|
+
process.exitCode = await main(process.argv.slice(2));
|
|
199
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { runCodeqlCheck } from "../../src/securityValidation/staticScans/codeql.js";
|
|
3
|
+
import { resolveValidationTarget } from "../../src/securityValidation/validate/resolveTarget.js";
|
|
4
|
+
const rawArgs = process.argv.slice(2);
|
|
5
|
+
const args = parseArgs(rawArgs);
|
|
6
|
+
const toolRoot = process.cwd();
|
|
7
|
+
let targetRoot;
|
|
8
|
+
try {
|
|
9
|
+
const target = resolveValidationTarget(args.target, toolRoot);
|
|
10
|
+
targetRoot = target.targetRoot;
|
|
11
|
+
if (!target.isSelf) {
|
|
12
|
+
console.log(`Target: ${targetRoot}`);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
console.error(`ERROR: ${err instanceof Error ? err.message : String(err)}`);
|
|
17
|
+
process.exitCode = 1;
|
|
18
|
+
process.exit(1);
|
|
19
|
+
}
|
|
20
|
+
console.log("Running CodeQL static analysis check...");
|
|
21
|
+
const result = await runCodeqlCheck({
|
|
22
|
+
cwd: toolRoot,
|
|
23
|
+
targetRoot,
|
|
24
|
+
timeoutMs: 30_000,
|
|
25
|
+
});
|
|
26
|
+
const label = result.status === "skipped"
|
|
27
|
+
? `SKIPPED — ${result.skippedReason ?? "tool unavailable"}`
|
|
28
|
+
: result.status.toUpperCase();
|
|
29
|
+
console.log(`Status: ${label}`);
|
|
30
|
+
if (result.findings.length > 0) {
|
|
31
|
+
console.log("\nFindings:");
|
|
32
|
+
for (const f of result.findings) {
|
|
33
|
+
console.log(` [${f.severity.toUpperCase()}] ${f.title}`);
|
|
34
|
+
if (f.description)
|
|
35
|
+
console.log(` ${f.description.slice(0, 120)}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
console.log(`\nDuration: ${result.durationMs}ms`);
|
|
39
|
+
if (result.status === "skipped") {
|
|
40
|
+
console.log("\nCodeQL is optional. Absence does not block release.");
|
|
41
|
+
process.exitCode = 0;
|
|
42
|
+
}
|
|
43
|
+
else if (result.status === "failed") {
|
|
44
|
+
process.exitCode = 1;
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
process.exitCode = 0;
|
|
48
|
+
}
|
|
49
|
+
function parseArgs(argv) {
|
|
50
|
+
const result = {};
|
|
51
|
+
for (let i = 0; i < argv.length; i++) {
|
|
52
|
+
if ((argv[i] === "--target" || argv[i] === "-t") && i + 1 < argv.length) {
|
|
53
|
+
result.target = argv[++i];
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { runDependencyChecks } from "../../src/securityValidation/index.js";
|
|
4
|
+
import { DEFAULT_SECURITY_CONFIG } from "../../src/securityValidation/index.js";
|
|
5
|
+
import { resolveValidationTarget } from "../../src/securityValidation/validate/resolveTarget.js";
|
|
6
|
+
const rawArgs = process.argv.slice(2);
|
|
7
|
+
const args = parseArgs(rawArgs);
|
|
8
|
+
const toolRoot = process.cwd();
|
|
9
|
+
let targetRoot;
|
|
10
|
+
try {
|
|
11
|
+
const target = resolveValidationTarget(args.target, toolRoot);
|
|
12
|
+
targetRoot = target.targetRoot;
|
|
13
|
+
if (!target.isSelf) {
|
|
14
|
+
console.log(`Target: ${targetRoot}`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
catch (err) {
|
|
18
|
+
console.error(`ERROR: ${err instanceof Error ? err.message : String(err)}`);
|
|
19
|
+
process.exitCode = 1;
|
|
20
|
+
process.exit(1);
|
|
21
|
+
}
|
|
22
|
+
const config = {
|
|
23
|
+
...DEFAULT_SECURITY_CONFIG,
|
|
24
|
+
reportDir: path.join(toolRoot, DEFAULT_SECURITY_CONFIG.reportDir),
|
|
25
|
+
rawOutputDir: path.join(toolRoot, DEFAULT_SECURITY_CONFIG.rawOutputDir),
|
|
26
|
+
};
|
|
27
|
+
console.log("Running dependency checks...");
|
|
28
|
+
console.log(`Report directory: ${config.reportDir}`);
|
|
29
|
+
const output = await runDependencyChecks({ cwd: targetRoot, config });
|
|
30
|
+
const passed = output.checks.filter((c) => c.status === "passed").length;
|
|
31
|
+
const failed = output.checks.filter((c) => c.status === "failed").length;
|
|
32
|
+
const warned = output.checks.filter((c) => c.status === "warning").length;
|
|
33
|
+
const skipped = output.checks.filter((c) => c.status === "skipped").length;
|
|
34
|
+
console.log(`\nDependency checks complete:`);
|
|
35
|
+
console.log(` Passed: ${passed}`);
|
|
36
|
+
console.log(` Warned: ${warned}`);
|
|
37
|
+
console.log(` Failed: ${failed}`);
|
|
38
|
+
console.log(` Skipped: ${skipped}`);
|
|
39
|
+
console.log(` Findings: ${output.findings.length}`);
|
|
40
|
+
if (output.findings.length > 0) {
|
|
41
|
+
console.log("\nFindings:");
|
|
42
|
+
for (const f of output.findings) {
|
|
43
|
+
console.log(` [${f.severity.toUpperCase()}] ${f.title}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
console.log(`\nResults written to ${config.reportDir}`);
|
|
47
|
+
const hasBlocker = output.findings.some((f) => f.severity === "blocker");
|
|
48
|
+
process.exitCode = hasBlocker ? 1 : 0;
|
|
49
|
+
function parseArgs(argv) {
|
|
50
|
+
const result = {};
|
|
51
|
+
for (let i = 0; i < argv.length; i++) {
|
|
52
|
+
if ((argv[i] === "--target" || argv[i] === "-t") && i + 1 < argv.length) {
|
|
53
|
+
result.target = argv[++i];
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return result;
|
|
57
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { runAllFuzzTargets } from "../../src/securityValidation/fuzz/fuzzHarness.js";
|
|
3
|
+
import { ALL_FUZZ_TARGETS } from "../../src/securityValidation/fuzz/fuzzTargets.js";
|
|
4
|
+
const SEED = parseInt(process.env["FUZZ_SEED"] ?? "0xDEADBEEF", 16);
|
|
5
|
+
const ITERATIONS = parseInt(process.env["FUZZ_ITERATIONS"] ?? "50", 10);
|
|
6
|
+
console.log(`Running fuzz smoke tests...`);
|
|
7
|
+
console.log(` Targets : ${ALL_FUZZ_TARGETS.length}`);
|
|
8
|
+
console.log(` Seed : 0x${SEED.toString(16).toUpperCase()}`);
|
|
9
|
+
console.log(` Iterations: ${ITERATIONS} per target`);
|
|
10
|
+
const result = await runAllFuzzTargets(ALL_FUZZ_TARGETS, {
|
|
11
|
+
seed: SEED,
|
|
12
|
+
iterations: ITERATIONS,
|
|
13
|
+
});
|
|
14
|
+
console.log(`\nStatus : ${result.status.toUpperCase()}`);
|
|
15
|
+
console.log(`Duration: ${result.durationMs}ms`);
|
|
16
|
+
console.log(`Crashes : ${result.findings.length}`);
|
|
17
|
+
if (result.findings.length > 0) {
|
|
18
|
+
console.log("\nCrash findings:");
|
|
19
|
+
for (const f of result.findings) {
|
|
20
|
+
console.log(` [${f.severity.toUpperCase()}] ${f.title}`);
|
|
21
|
+
if (f.evidence)
|
|
22
|
+
console.log(` Evidence: ${f.evidence.slice(0, 160)}`);
|
|
23
|
+
}
|
|
24
|
+
process.exitCode = 1;
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
console.log("\nAll fuzz targets completed without crashes.");
|
|
28
|
+
process.exitCode = 0;
|
|
29
|
+
}
|