@dailephd/my-dev-kit-lab 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +272 -0
- package/benchmarks/contracts/benchmark-project-profiles.json +1199 -0
- package/benchmarks/contracts/todo-behavior.md +70 -0
- package/benchmarks/contracts/todo-benchmark-case.json +227 -0
- package/benchmarks/projects/README.md +34 -0
- package/benchmarks/projects/task-analytics-large-mixed/README.md +1 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/__init__.py +3 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/fixtures.py +6 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/metrics.py +29 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/models.py +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/parser.py +16 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/pipeline.py +9 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/quality.py +8 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/reporting.py +11 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_metrics.py +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_parser.py +15 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_quality.py +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_reporting.py +15 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/package.json +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/index.ts +11 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/analyticsSnapshot.ts +20 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/project.ts +5 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/task.ts +10 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/reporting/buildProjectLeaderboard.ts +7 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/reporting/formatTaskHealthReport.ts +13 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/buildAnalyticsSnapshot.ts +39 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/completeTask.ts +10 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/createTask.ts +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/listTasksByProject.ts +6 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/store/projectStore.ts +20 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/store/taskStore.ts +44 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/validation/projectValidation.ts +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/src/validation/taskValidation.ts +18 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/buildAnalyticsSnapshot.test.ts +48 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/completeTask.test.ts +21 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/createTask.test.ts +31 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/listTasksByProject.test.ts +18 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tests/reporting.test.ts +19 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/tsconfig.json +12 -0
- package/benchmarks/projects/task-analytics-large-mixed/ts/vitest.config.ts +5 -0
- package/benchmarks/projects/task-workflow-medium-ts/README.md +1 -0
- package/benchmarks/projects/task-workflow-medium-ts/package.json +12 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/index.ts +9 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/models/project.ts +6 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/models/task.ts +39 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/completeTask.ts +15 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/createTask.ts +26 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/filterTasks.ts +17 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/importTasks.ts +33 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/services/summarizeTasks.ts +30 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/store/taskStore.ts +76 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/utils/deterministicId.ts +3 -0
- package/benchmarks/projects/task-workflow-medium-ts/src/validation/taskValidation.ts +45 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/completeTask.test.ts +16 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/createTask.test.ts +21 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/filterTasks.test.ts +18 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/importTasks.test.ts +22 -0
- package/benchmarks/projects/task-workflow-medium-ts/tests/summarizeTasks.test.ts +29 -0
- package/benchmarks/projects/task-workflow-medium-ts/tsconfig.json +12 -0
- package/benchmarks/projects/task-workflow-medium-ts/vitest.config.ts +5 -0
- package/benchmarks/projects/todo-js/README.md +3 -0
- package/benchmarks/projects/todo-js/package.json +11 -0
- package/benchmarks/projects/todo-js/src/index.js +2 -0
- package/benchmarks/projects/todo-js/src/taskService.js +37 -0
- package/benchmarks/projects/todo-js/src/taskStore.js +28 -0
- package/benchmarks/projects/todo-js/tests/taskService.test.js +45 -0
- package/benchmarks/projects/todo-js/vitest.config.js +5 -0
- package/benchmarks/projects/todo-mixed-ts-py/README.md +3 -0
- package/benchmarks/projects/todo-mixed-ts-py/package.json +13 -0
- package/benchmarks/projects/todo-mixed-ts-py/python/task_service.py +76 -0
- package/benchmarks/projects/todo-mixed-ts-py/src/taskCli.ts +38 -0
- package/benchmarks/projects/todo-mixed-ts-py/tests/mixedBoundary.test.ts +18 -0
- package/benchmarks/projects/todo-mixed-ts-py/tsconfig.json +12 -0
- package/benchmarks/projects/todo-mixed-ts-py/vitest.config.ts +5 -0
- package/benchmarks/projects/todo-python/README.md +3 -0
- package/benchmarks/projects/todo-python/src/__init__.py +4 -0
- package/benchmarks/projects/todo-python/src/task_service.py +32 -0
- package/benchmarks/projects/todo-python/src/task_store.py +28 -0
- package/benchmarks/projects/todo-python/tests/test_task_service.py +52 -0
- package/benchmarks/projects/todo-ts/README.md +3 -0
- package/benchmarks/projects/todo-ts/package.json +12 -0
- package/benchmarks/projects/todo-ts/src/index.ts +2 -0
- package/benchmarks/projects/todo-ts/src/taskService.ts +41 -0
- package/benchmarks/projects/todo-ts/src/taskStore.ts +34 -0
- package/benchmarks/projects/todo-ts/tests/taskService.test.ts +45 -0
- package/benchmarks/projects/todo-ts/tsconfig.json +12 -0
- package/benchmarks/projects/todo-ts/vitest.config.ts +5 -0
- package/dist/scripts/build-gallery.js +3 -0
- package/dist/scripts/capture-demo-report.js +3 -0
- package/dist/scripts/evaluate-token-savings.js +2 -0
- package/dist/scripts/experiments/describeExperiment.js +143 -0
- package/dist/scripts/experiments/listExperiments.js +44 -0
- package/dist/scripts/experiments/runExperiment.js +199 -0
- package/dist/scripts/generate-experiment-plots.js +3 -0
- package/dist/scripts/generate-prompt-variants.js +2 -0
- package/dist/scripts/render-experiment-report.js +2 -0
- package/dist/scripts/run-agent-prompt.js +2 -0
- package/dist/scripts/run-controlled-experiment.js +2 -0
- package/dist/scripts/run-final-demo.js +3 -0
- package/dist/scripts/run-lab-demo.js +5 -0
- package/dist/scripts/run-visualization-demos.js +3 -0
- package/dist/scripts/security/runCodeql.js +57 -0
- package/dist/scripts/security/runDependencyChecks.js +57 -0
- package/dist/scripts/security/runFuzzSmoke.js +29 -0
- package/dist/scripts/security/runPackageChecks.js +56 -0
- package/dist/scripts/security/runSemgrep.js +63 -0
- package/dist/scripts/security/validate.js +117 -0
- package/dist/scripts/verify-benchmarks.js +202 -0
- package/dist/src/agents/adapters/claudeAdapter.js +37 -0
- package/dist/src/agents/adapters/codexAdapter.js +110 -0
- package/dist/src/agents/adapters/fakeAgentAdapter.js +101 -0
- package/dist/src/agents/agentRegistry.js +21 -0
- package/dist/src/agents/index.js +7 -0
- package/dist/src/agents/parseAgentTokenUsage.js +137 -0
- package/dist/src/agents/runAgentPrompt.js +38 -0
- package/dist/src/agents/types.js +1 -0
- package/dist/src/commands/buildGalleryCommand.js +56 -0
- package/dist/src/commands/captureDemoReport.js +116 -0
- package/dist/src/commands/evaluateTokenSavings.js +175 -0
- package/dist/src/commands/generateExperimentPlotsCommand.js +38 -0
- package/dist/src/commands/generatePromptVariants.js +67 -0
- package/dist/src/commands/renderExperimentReportCommand.js +131 -0
- package/dist/src/commands/runAgentPromptCommand.js +132 -0
- package/dist/src/commands/runControlledExperimentCommand.js +174 -0
- package/dist/src/commands/runFinalDemoCommand.js +123 -0
- package/dist/src/commands/runLabDemo.js +62 -0
- package/dist/src/commands/runVisualizationDemosCommand.js +67 -0
- package/dist/src/core/commandLine.js +59 -0
- package/dist/src/core/countTokens.js +8 -0
- package/dist/src/core/fileGlobs.js +100 -0
- package/dist/src/core/localProjectTarget.js +75 -0
- package/dist/src/core/pathSafety.js +19 -0
- package/dist/src/core/pythonCommand.js +30 -0
- package/dist/src/core/resolveCommand.js +110 -0
- package/dist/src/core/runMeasuredCommand.js +143 -0
- package/dist/src/evaluation/benchmarkMetadata.js +207 -0
- package/dist/src/evaluation/buildExperimentMatrix.js +75 -0
- package/dist/src/evaluation/classifyAgentRunOutcome.js +40 -0
- package/dist/src/evaluation/compareExperimentRuns.js +79 -0
- package/dist/src/evaluation/compareTokenSavings.js +47 -0
- package/dist/src/evaluation/controlledExperimentTypes.js +1 -0
- package/dist/src/evaluation/index.js +18 -0
- package/dist/src/evaluation/parseAgentAnswer.js +230 -0
- package/dist/src/evaluation/projectComplexity.js +126 -0
- package/dist/src/evaluation/projectFileTree.js +83 -0
- package/dist/src/evaluation/readEvaluationCases.js +59 -0
- package/dist/src/evaluation/renderTokenSavingsReportInput.js +55 -0
- package/dist/src/evaluation/runControlledExperiment.js +158 -0
- package/dist/src/evaluation/runMyDevKitRetrieval.js +197 -0
- package/dist/src/evaluation/runRawFullFileBaseline.js +31 -0
- package/dist/src/evaluation/scoreCorrectness.js +127 -0
- package/dist/src/evaluation/types.js +1 -0
- package/dist/src/evaluation/writeExperimentArtifacts.js +104 -0
- package/dist/src/evaluation/writeTokenSavingsArtifacts.js +57 -0
- package/dist/src/experiments/config.js +24 -0
- package/dist/src/experiments/defaultRegistry.js +7 -0
- package/dist/src/experiments/errors.js +18 -0
- package/dist/src/experiments/index.js +9 -0
- package/dist/src/experiments/outputPaths.js +25 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/config.js +37 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/index.js +3 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +83 -0
- package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +260 -0
- package/dist/src/experiments/plugins/index.js +1 -0
- package/dist/src/experiments/registry.js +43 -0
- package/dist/src/experiments/results.js +48 -0
- package/dist/src/experiments/runner.js +181 -0
- package/dist/src/experiments/target.js +8 -0
- package/dist/src/experiments/types.js +1 -0
- package/dist/src/gallery/index.js +2 -0
- package/dist/src/gallery/types.js +1 -0
- package/dist/src/gallery/writeGalleryManifest.js +214 -0
- package/dist/src/index.js +12 -0
- package/dist/src/plots/buildExperimentPlotData.js +137 -0
- package/dist/src/plots/index.js +4 -0
- package/dist/src/plots/renderSvgChart.js +82 -0
- package/dist/src/plots/types.js +1 -0
- package/dist/src/plots/writePlotArtifacts.js +46 -0
- package/dist/src/prompts/buildPromptContext.js +68 -0
- package/dist/src/prompts/generateMyDevKitPrompt.js +106 -0
- package/dist/src/prompts/generatePromptVariants.js +36 -0
- package/dist/src/prompts/generateRawFullFilePrompt.js +97 -0
- package/dist/src/prompts/index.js +7 -0
- package/dist/src/prompts/measurePromptComplexity.js +41 -0
- package/dist/src/prompts/types.js +1 -0
- package/dist/src/prompts/writePromptArtifacts.js +43 -0
- package/dist/src/report/buildExperimentReportInput.js +339 -0
- package/dist/src/report/experimentReportTypes.js +1 -0
- package/dist/src/report/experiments/buildPluginExperimentReport.js +153 -0
- package/dist/src/report/experiments/experimentReportModel.js +1 -0
- package/dist/src/report/experiments/index.js +4 -0
- package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +133 -0
- package/dist/src/report/experiments/writePluginExperimentReports.js +30 -0
- package/dist/src/report/index.js +8 -0
- package/dist/src/report/renderExperimentHtmlReport.js +354 -0
- package/dist/src/report/renderHtmlReport.js +103 -0
- package/dist/src/report/types.js +10 -0
- package/dist/src/report/writeExperimentReportArtifacts.js +38 -0
- package/dist/src/report/writeReportArtifacts.js +39 -0
- package/dist/src/screenshot/captureReportScreenshot.js +75 -0
- package/dist/src/screenshot/index.js +2 -0
- package/dist/src/screenshot/types.js +1 -0
- package/dist/src/securityValidation/artifacts.js +15 -0
- package/dist/src/securityValidation/cliAdversarial/adversarialCliConfig.js +38 -0
- package/dist/src/securityValidation/cliAdversarial/dataVolumeChecks.js +194 -0
- package/dist/src/securityValidation/cliAdversarial/jsonStdoutChecks.js +359 -0
- package/dist/src/securityValidation/cliAdversarial/malformedArtifactChecks.js +284 -0
- package/dist/src/securityValidation/cliAdversarial/malformedArtifactFixtures.js +79 -0
- package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +431 -0
- package/dist/src/securityValidation/cliAdversarial/pathCases.js +144 -0
- package/dist/src/securityValidation/cliAdversarial/readOnlyBoundaryChecks.js +294 -0
- package/dist/src/securityValidation/cliAdversarial/runAdversarialCheck.js +149 -0
- package/dist/src/securityValidation/cliAdversarial/subprocessSafetyChecks.js +214 -0
- package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +160 -0
- package/dist/src/securityValidation/commandRunner.js +136 -0
- package/dist/src/securityValidation/config.js +39 -0
- package/dist/src/securityValidation/dependencies/parseNpmAudit.js +115 -0
- package/dist/src/securityValidation/dependencies/parseNpmLs.js +71 -0
- package/dist/src/securityValidation/dependencies/parseNpmOutdated.js +41 -0
- package/dist/src/securityValidation/dependencies/runDependencyChecks.js +239 -0
- package/dist/src/securityValidation/dependencies/runOsvScanner.js +43 -0
- package/dist/src/securityValidation/fuzz/fuzzHarness.js +61 -0
- package/dist/src/securityValidation/fuzz/fuzzTargets.js +204 -0
- package/dist/src/securityValidation/fuzz/randomInput.js +0 -0
- package/dist/src/securityValidation/index.js +34 -0
- package/dist/src/securityValidation/packageChecks/forbiddenPackageContents.js +67 -0
- package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +56 -0
- package/dist/src/securityValidation/packageChecks/runPackageChecks.js +88 -0
- package/dist/src/securityValidation/report/renderSecurityReport.js +248 -0
- package/dist/src/securityValidation/report/securityReportTypes.js +1 -0
- package/dist/src/securityValidation/staticScans/codeql.js +66 -0
- package/dist/src/securityValidation/staticScans/semgrep.js +180 -0
- package/dist/src/securityValidation/testMatrix.js +535 -0
- package/dist/src/securityValidation/types.js +34 -0
- package/dist/src/securityValidation/validate/resolveTarget.js +32 -0
- package/dist/src/securityValidation/validate/runSecurityValidation.js +169 -0
- package/dist/src/securityValidation/validate/verdict.js +73 -0
- package/dist/src/visualizationDemos/buildMyDevKitVisualizationCommands.js +59 -0
- package/dist/src/visualizationDemos/index.js +4 -0
- package/dist/src/visualizationDemos/runVisualizationDemos.js +82 -0
- package/dist/src/visualizationDemos/types.js +1 -0
- package/dist/src/visualizationDemos/writeVisualizationDemoArtifacts.js +25 -0
- package/docs/METRICS.md +286 -0
- package/examples/demo-report-input.json +78 -0
- package/examples/lab-demo-cases.json +35 -0
- package/examples/real-agent-campaign-cases.json +118 -0
- package/examples/token-savings-cases.json +122 -0
- package/package.json +91 -0
- package/tests/fixtures/fake-adversarial-cli.js +152 -0
- package/tests/fixtures/fake-my-dev-kit-cli.js +83 -0
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import { runDependencyChecks } from "../dependencies/runDependencyChecks.js";
|
|
4
|
+
import { runPackageChecks } from "../packageChecks/runPackageChecks.js";
|
|
5
|
+
import { runCodeqlCheck } from "../staticScans/codeql.js";
|
|
6
|
+
import { runSemgrepCheck } from "../staticScans/semgrep.js";
|
|
7
|
+
import { runAllFuzzTargets } from "../fuzz/fuzzHarness.js";
|
|
8
|
+
import { ALL_FUZZ_TARGETS } from "../fuzz/fuzzTargets.js";
|
|
9
|
+
import { runSecurityCommand, resolveNpmCommand } from "../commandRunner.js";
|
|
10
|
+
import { calculateVerdict } from "./verdict.js";
|
|
11
|
+
import { resolveValidationTarget, targetDescription as describeTarget } from "./resolveTarget.js";
|
|
12
|
+
import { DEFAULT_SECURITY_CONFIG } from "../config.js";
|
|
13
|
+
export async function runSecurityValidation(options) {
|
|
14
|
+
const toolRoot = options.cwd;
|
|
15
|
+
const config = {
|
|
16
|
+
...DEFAULT_SECURITY_CONFIG,
|
|
17
|
+
reportDir: path.join(toolRoot, DEFAULT_SECURITY_CONFIG.reportDir),
|
|
18
|
+
rawOutputDir: path.join(toolRoot, DEFAULT_SECURITY_CONFIG.rawOutputDir),
|
|
19
|
+
...options.config,
|
|
20
|
+
};
|
|
21
|
+
// Resolve and validate the target project
|
|
22
|
+
const target = resolveValidationTarget(options.targetPath, toolRoot);
|
|
23
|
+
const startedAt = new Date().toISOString();
|
|
24
|
+
const allChecks = [];
|
|
25
|
+
const allFindings = [];
|
|
26
|
+
function collect(check) {
|
|
27
|
+
allChecks.push(check);
|
|
28
|
+
allFindings.push(...check.findings);
|
|
29
|
+
}
|
|
30
|
+
function collectMany(checks) {
|
|
31
|
+
for (const c of checks)
|
|
32
|
+
collect(c);
|
|
33
|
+
}
|
|
34
|
+
// --- Dependency checks (runs in target root) ---
|
|
35
|
+
// If target has no package.json/lockfile, runDependencyChecks will surface that
|
|
36
|
+
// as a structured finding rather than crashing.
|
|
37
|
+
try {
|
|
38
|
+
const depOutput = await runDependencyChecks({ cwd: target.targetRoot, config });
|
|
39
|
+
collectMany(depOutput.checks);
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
collect(errorCheck("dependency-checks", "Dependency checks", "dependency-audit", err));
|
|
43
|
+
}
|
|
44
|
+
// --- Package content checks (runs in target root) ---
|
|
45
|
+
// Not all projects are publishable npm packages; runPackageChecks handles
|
|
46
|
+
// missing package.json gracefully.
|
|
47
|
+
try {
|
|
48
|
+
const pkgOutput = await runPackageChecks({ cwd: target.targetRoot, config });
|
|
49
|
+
collectMany(pkgOutput.checks);
|
|
50
|
+
}
|
|
51
|
+
catch (err) {
|
|
52
|
+
collect(errorCheck("package-content", "Package content checks", "package-content", err));
|
|
53
|
+
}
|
|
54
|
+
// --- Static scans (scan target root; config from tool root) ---
|
|
55
|
+
try {
|
|
56
|
+
collect(await runCodeqlCheck({ cwd: toolRoot, targetRoot: target.targetRoot, timeoutMs: config.commandTimeoutMs }));
|
|
57
|
+
}
|
|
58
|
+
catch (err) {
|
|
59
|
+
collect(errorCheck("codeql-scan", "CodeQL static analysis", "static-scan", err));
|
|
60
|
+
}
|
|
61
|
+
try {
|
|
62
|
+
collect(await runSemgrepCheck({
|
|
63
|
+
targetRoot: target.targetRoot,
|
|
64
|
+
toolRoot,
|
|
65
|
+
timeoutMs: config.commandTimeoutMs,
|
|
66
|
+
}));
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
collect(errorCheck("semgrep-scan", "Semgrep static analysis", "static-scan", err));
|
|
70
|
+
}
|
|
71
|
+
// --- CLI adversarial tests (always tool-internal; tests my-dev-kit-lab harness) ---
|
|
72
|
+
// These run in the tool root because they use fake-adversarial-cli.js fixtures.
|
|
73
|
+
// Labeled clearly so the report shows they are not target-specific checks.
|
|
74
|
+
{
|
|
75
|
+
const startedAt2 = new Date().toISOString();
|
|
76
|
+
const npm = resolveNpmCommand();
|
|
77
|
+
const cmd = await runSecurityCommand({
|
|
78
|
+
command: npm,
|
|
79
|
+
args: ["run", "test:security", "--", "--reporter=json"],
|
|
80
|
+
cwd: toolRoot,
|
|
81
|
+
timeoutMs: Math.min(config.commandTimeoutMs * 3, 180_000),
|
|
82
|
+
});
|
|
83
|
+
const finishedAt2 = new Date().toISOString();
|
|
84
|
+
const passed = cmd.exitCode === 0;
|
|
85
|
+
collect({
|
|
86
|
+
id: "cli-adversarial-suite",
|
|
87
|
+
name: "CLI adversarial test suite (tool self-tests, not target-specific)",
|
|
88
|
+
category: "cli-adversarial",
|
|
89
|
+
status: passed ? "passed" : "failed",
|
|
90
|
+
severity: "major",
|
|
91
|
+
startedAt: startedAt2,
|
|
92
|
+
finishedAt: finishedAt2,
|
|
93
|
+
durationMs: cmd.durationMs,
|
|
94
|
+
findings: passed
|
|
95
|
+
? []
|
|
96
|
+
: [
|
|
97
|
+
{
|
|
98
|
+
id: "cli-adversarial-suite-failed",
|
|
99
|
+
title: "CLI adversarial test suite had failures",
|
|
100
|
+
severity: "major",
|
|
101
|
+
category: "cli-adversarial",
|
|
102
|
+
description: `test:security exited with code ${String(cmd.exitCode)}. Run 'npm run test:security' for details.`,
|
|
103
|
+
recommendation: "Fix failing adversarial tests before release.",
|
|
104
|
+
releaseImpact: "Should fix before release",
|
|
105
|
+
},
|
|
106
|
+
],
|
|
107
|
+
command: `${npm} run test:security`,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
// --- Fuzz smoke (always tool-internal; tests library parser/helper code) ---
|
|
111
|
+
try {
|
|
112
|
+
collect(await runAllFuzzTargets(ALL_FUZZ_TARGETS, {
|
|
113
|
+
seed: options.fuzzSeed ?? 0xdeadbeef,
|
|
114
|
+
iterations: options.fuzzIterations ?? 50,
|
|
115
|
+
}));
|
|
116
|
+
}
|
|
117
|
+
catch (err) {
|
|
118
|
+
collect(errorCheck("fuzz-smoke", "Fuzz smoke", "fuzz-smoke", err));
|
|
119
|
+
}
|
|
120
|
+
const finishedAt = new Date().toISOString();
|
|
121
|
+
// --- Tool root metadata ---
|
|
122
|
+
let toolPackageName = "my-dev-kit-lab";
|
|
123
|
+
let toolPackageVersion = "unknown";
|
|
124
|
+
try {
|
|
125
|
+
const pkgRaw = fs.readFileSync(path.join(toolRoot, "package.json"), "utf8");
|
|
126
|
+
const pkg = JSON.parse(pkgRaw);
|
|
127
|
+
if (pkg.name)
|
|
128
|
+
toolPackageName = pkg.name;
|
|
129
|
+
if (pkg.version)
|
|
130
|
+
toolPackageVersion = pkg.version;
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
// Use defaults.
|
|
134
|
+
}
|
|
135
|
+
const { verdict, recommendedNextStep } = calculateVerdict(allChecks, allFindings);
|
|
136
|
+
return {
|
|
137
|
+
toolRoot,
|
|
138
|
+
toolPackageName,
|
|
139
|
+
toolPackageVersion,
|
|
140
|
+
targetRoot: target.targetRoot,
|
|
141
|
+
targetDescription: describeTarget(target),
|
|
142
|
+
packageName: target.packageName ?? (target.isSelf ? toolPackageName : path.basename(target.targetRoot)),
|
|
143
|
+
packageVersion: target.packageVersion ?? (target.isSelf ? toolPackageVersion : "unknown"),
|
|
144
|
+
auditedBranch: target.branch ?? "unknown",
|
|
145
|
+
auditedCommit: target.commit ?? "unknown",
|
|
146
|
+
isSelf: target.isSelf,
|
|
147
|
+
startedAt,
|
|
148
|
+
finishedAt,
|
|
149
|
+
checks: allChecks,
|
|
150
|
+
findings: allFindings,
|
|
151
|
+
verdict,
|
|
152
|
+
recommendedNextStep,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
function errorCheck(id, name, category, err) {
|
|
156
|
+
const now = new Date().toISOString();
|
|
157
|
+
return {
|
|
158
|
+
id,
|
|
159
|
+
name: `${name} (orchestrator error)`,
|
|
160
|
+
category,
|
|
161
|
+
status: "failed",
|
|
162
|
+
severity: "major",
|
|
163
|
+
startedAt: now,
|
|
164
|
+
finishedAt: now,
|
|
165
|
+
durationMs: 0,
|
|
166
|
+
findings: [],
|
|
167
|
+
skippedReason: err instanceof Error ? err.message : String(err),
|
|
168
|
+
};
|
|
169
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// Verdict calculation
|
|
3
|
+
//
|
|
4
|
+
// Rules (evaluated in order, first match wins):
|
|
5
|
+
// 1. Any blocker finding → not-ready-security-blocker-remains
|
|
6
|
+
// 2. Any mandatory check "failed" → not-ready-security-blocker-remains
|
|
7
|
+
// 3. Too many mandatory checks skipped (>= 4) → inconclusive-audit-environment-incomplete
|
|
8
|
+
// 4. Any major finding → not-ready-security-blocker-remains
|
|
9
|
+
// 5. Any optional check skipped → ready-except-optional-manual-checks
|
|
10
|
+
// 6. Otherwise → ready-for-release-preparation
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
// Checks that must not be skipped for a clean verdict.
|
|
13
|
+
const MANDATORY_CHECK_IDS = new Set([
|
|
14
|
+
"npm-audit-full",
|
|
15
|
+
"npm-audit-runtime",
|
|
16
|
+
"npm-pack-dry-run",
|
|
17
|
+
"source-files-not-modified",
|
|
18
|
+
"writes-limited-to-output",
|
|
19
|
+
]);
|
|
20
|
+
export function calculateVerdict(checks, findings) {
|
|
21
|
+
const blockerFindings = findings.filter((f) => f.severity === "blocker");
|
|
22
|
+
const majorFindings = findings.filter((f) => f.severity === "major");
|
|
23
|
+
const failedMandatory = checks.filter((c) => MANDATORY_CHECK_IDS.has(c.id) && c.status === "failed");
|
|
24
|
+
const skippedMandatory = checks.filter((c) => MANDATORY_CHECK_IDS.has(c.id) && c.status === "skipped");
|
|
25
|
+
const skippedOptional = checks.filter((c) => !MANDATORY_CHECK_IDS.has(c.id) && c.status === "skipped");
|
|
26
|
+
if (blockerFindings.length > 0) {
|
|
27
|
+
return {
|
|
28
|
+
verdict: "not-ready-security-blocker-remains",
|
|
29
|
+
recommendedNextStep: `Fix ${blockerFindings.length} blocker finding(s) before release: ${blockerFindings.map((f) => f.title).join("; ")}`,
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
if (failedMandatory.length > 0) {
|
|
33
|
+
return {
|
|
34
|
+
verdict: "not-ready-security-blocker-remains",
|
|
35
|
+
recommendedNextStep: `Mandatory check(s) failed: ${failedMandatory.map((c) => c.name).join(", ")}. Investigate and re-run security:validate.`,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
if (skippedMandatory.length >= 4) {
|
|
39
|
+
return {
|
|
40
|
+
verdict: "inconclusive-audit-environment-incomplete",
|
|
41
|
+
recommendedNextStep: `Too many mandatory checks were skipped (${skippedMandatory.length}). Ensure npm, npm audit, and package tools are available.`,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
if (majorFindings.length > 0) {
|
|
45
|
+
return {
|
|
46
|
+
verdict: "not-ready-security-blocker-remains",
|
|
47
|
+
recommendedNextStep: `Fix or review ${majorFindings.length} major finding(s) before release: ${majorFindings.map((f) => f.title).join("; ")}`,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
if (skippedOptional.length > 0) {
|
|
51
|
+
const names = skippedOptional.map((c) => c.name).join(", ");
|
|
52
|
+
return {
|
|
53
|
+
verdict: "ready-except-optional-manual-checks",
|
|
54
|
+
recommendedNextStep: `Optional check(s) were skipped due to unavailable tools: ${names}. Run manually if possible before publishing.`,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
verdict: "ready-for-release-preparation",
|
|
59
|
+
recommendedNextStep: "All mandatory checks passed. Proceed with version bump, changelog, and npm publish checklist.",
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
export function verdictToHumanLabel(verdict) {
|
|
63
|
+
switch (verdict) {
|
|
64
|
+
case "ready-for-release-preparation":
|
|
65
|
+
return "ready for release preparation";
|
|
66
|
+
case "not-ready-security-blocker-remains":
|
|
67
|
+
return "not ready: security blocker remains";
|
|
68
|
+
case "ready-except-optional-manual-checks":
|
|
69
|
+
return "ready except optional manual checks";
|
|
70
|
+
case "inconclusive-audit-environment-incomplete":
|
|
71
|
+
return "inconclusive: audit environment incomplete";
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import path from "node:path";
|
|
2
|
+
export function buildMyDevKitVisualizationCommands(options) {
|
|
3
|
+
const projectPath = path.resolve(options.projectPath);
|
|
4
|
+
const outDir = path.resolve(options.outDir);
|
|
5
|
+
const indexDir = path.join(outDir, "artifacts", "index");
|
|
6
|
+
const artifactsDir = path.join(outDir, "artifacts");
|
|
7
|
+
const query = options.query ?? "create task";
|
|
8
|
+
const nodeId = options.nodeId ?? "todo-ts:createTask";
|
|
9
|
+
return [
|
|
10
|
+
{
|
|
11
|
+
id: "index-project",
|
|
12
|
+
name: "Index project",
|
|
13
|
+
command: options.kitCommand,
|
|
14
|
+
args: ["index", "--src", projectPath, "--out", indexDir, "--call-graph", "--json"],
|
|
15
|
+
cwd: projectPath,
|
|
16
|
+
expectedArtifacts: [path.join(indexDir, "manifest.json")]
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
id: "search-task-symbols",
|
|
20
|
+
name: "Search task symbols",
|
|
21
|
+
command: options.kitCommand,
|
|
22
|
+
args: ["search", "--index", indexDir, "--query", query, "--limit", "10"],
|
|
23
|
+
cwd: projectPath,
|
|
24
|
+
expectedArtifacts: []
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
id: "view-call-graph",
|
|
28
|
+
name: "View call graph",
|
|
29
|
+
command: options.kitCommand,
|
|
30
|
+
args: ["view", "--index", indexDir, "--graph", "code", "--out", path.join(artifactsDir, "call-graph.svg")],
|
|
31
|
+
cwd: projectPath,
|
|
32
|
+
expectedArtifacts: [path.join(artifactsDir, "call-graph.svg")]
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
id: "view-data-model",
|
|
36
|
+
name: "View data model",
|
|
37
|
+
command: options.kitCommand,
|
|
38
|
+
args: ["view", "--index", indexDir, "--graph", "data-model", "--out", path.join(artifactsDir, "data-model.svg")],
|
|
39
|
+
cwd: projectPath,
|
|
40
|
+
expectedArtifacts: [path.join(artifactsDir, "data-model.svg")]
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
id: "view-model-view-lineage",
|
|
44
|
+
name: "View model-view-lineage",
|
|
45
|
+
command: options.kitCommand,
|
|
46
|
+
args: ["view", "--index", indexDir, "--graph", "model-view-lineage", "--out", path.join(artifactsDir, "model-view-lineage.svg")],
|
|
47
|
+
cwd: projectPath,
|
|
48
|
+
expectedArtifacts: [path.join(artifactsDir, "model-view-lineage.svg")]
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
id: "source-known-symbol",
|
|
52
|
+
name: "Source known symbol",
|
|
53
|
+
command: options.kitCommand,
|
|
54
|
+
args: ["source", "--index", indexDir, "--node", nodeId],
|
|
55
|
+
cwd: projectPath,
|
|
56
|
+
expectedArtifacts: []
|
|
57
|
+
}
|
|
58
|
+
];
|
|
59
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { existsSync } from "node:fs";
|
|
2
|
+
import { mkdir } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { parseCommandString, serializeCommand } from "../core/commandLine.js";
|
|
5
|
+
import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
|
|
6
|
+
import { buildMyDevKitVisualizationCommands } from "./buildMyDevKitVisualizationCommands.js";
|
|
7
|
+
import { writeVisualizationDemoArtifacts } from "./writeVisualizationDemoArtifacts.js";
|
|
8
|
+
export async function runVisualizationDemos(options) {
|
|
9
|
+
const outDir = path.resolve(options.outDir);
|
|
10
|
+
const projectPath = path.resolve(options.projectPath);
|
|
11
|
+
const kitCommand = normalizeKitCommand(options.kitCommand);
|
|
12
|
+
const commands = options.commands ??
|
|
13
|
+
buildMyDevKitVisualizationCommands({
|
|
14
|
+
projectPath,
|
|
15
|
+
kitCommand,
|
|
16
|
+
outDir,
|
|
17
|
+
query: options.query,
|
|
18
|
+
nodeId: options.nodeId
|
|
19
|
+
});
|
|
20
|
+
await mkdir(path.join(outDir, "commands"), { recursive: true });
|
|
21
|
+
await mkdir(path.join(outDir, "artifacts"), { recursive: true });
|
|
22
|
+
const runs = [];
|
|
23
|
+
const warnings = [];
|
|
24
|
+
for (const command of commands) {
|
|
25
|
+
const commandOutDir = path.join(outDir, "commands", command.id);
|
|
26
|
+
const measured = await runMeasuredCommand({
|
|
27
|
+
commandId: command.id,
|
|
28
|
+
commandString: command.command,
|
|
29
|
+
cwd: command.cwd,
|
|
30
|
+
outDir: commandOutDir,
|
|
31
|
+
extraArgs: command.args,
|
|
32
|
+
timeoutMs: options.timeoutMs
|
|
33
|
+
});
|
|
34
|
+
const expectedArtifacts = command.expectedArtifacts.map((artifactPath) => ({ path: artifactPath, exists: existsSync(artifactPath) }));
|
|
35
|
+
const producedArtifactPaths = expectedArtifacts.filter((artifact) => artifact.exists).map((artifact) => artifact.path);
|
|
36
|
+
const runWarnings = [
|
|
37
|
+
...expectedArtifacts.filter((artifact) => !artifact.exists).map((artifact) => `Expected artifact was not produced: ${artifact.path}`)
|
|
38
|
+
];
|
|
39
|
+
const errors = measured.ok ? [] : [(measured.error ?? measured.stderr) || `Command exited with code ${measured.exitCode}`];
|
|
40
|
+
if (runWarnings.length > 0)
|
|
41
|
+
warnings.push(...runWarnings.map((warning) => `${command.id}: ${warning}`));
|
|
42
|
+
if (!measured.ok)
|
|
43
|
+
warnings.push(`${command.id}: command failed or is unsupported.`);
|
|
44
|
+
runs.push({
|
|
45
|
+
id: command.id,
|
|
46
|
+
name: command.name,
|
|
47
|
+
commandString: command.command,
|
|
48
|
+
args: command.args,
|
|
49
|
+
cwd: command.cwd,
|
|
50
|
+
exitCode: measured.exitCode,
|
|
51
|
+
durationMs: measured.durationMs,
|
|
52
|
+
stdoutPath: measured.stdoutPath,
|
|
53
|
+
stderrPath: measured.stderrPath,
|
|
54
|
+
producedArtifactPaths,
|
|
55
|
+
expectedArtifacts,
|
|
56
|
+
warnings: runWarnings,
|
|
57
|
+
errors,
|
|
58
|
+
ok: measured.ok,
|
|
59
|
+
measured
|
|
60
|
+
});
|
|
61
|
+
if (!measured.ok && options.requireAll) {
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return writeVisualizationDemoArtifacts({
|
|
66
|
+
outDir,
|
|
67
|
+
projectPath,
|
|
68
|
+
kitCommand,
|
|
69
|
+
runs,
|
|
70
|
+
warnings
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
function normalizeKitCommand(command) {
|
|
74
|
+
const parsed = parseCommandString(command);
|
|
75
|
+
if ((parsed.executable === "node" || parsed.executable === "node.exe") && parsed.args[0] && !path.isAbsolute(parsed.args[0])) {
|
|
76
|
+
const scriptPath = path.resolve(process.cwd(), parsed.args[0]);
|
|
77
|
+
if (existsSync(scriptPath)) {
|
|
78
|
+
return serializeCommand([parsed.executable, scriptPath, ...parsed.args.slice(1)]);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return command;
|
|
82
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { mkdir, writeFile } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { resolveWithinRoot } from "../core/pathSafety.js";
|
|
4
|
+
export async function writeVisualizationDemoArtifacts(options) {
|
|
5
|
+
const outDir = path.resolve(options.outDir);
|
|
6
|
+
await mkdir(outDir, { recursive: true });
|
|
7
|
+
const artifactPaths = {
|
|
8
|
+
summaryPath: resolveWithinRoot(outDir, "visualization-demo-summary.json"),
|
|
9
|
+
runsPath: resolveWithinRoot(outDir, "visualization-demo-runs.json"),
|
|
10
|
+
commandsDir: resolveWithinRoot(outDir, "commands"),
|
|
11
|
+
artifactsDir: resolveWithinRoot(outDir, "artifacts")
|
|
12
|
+
};
|
|
13
|
+
const summary = {
|
|
14
|
+
generatedAt: options.generatedAt ?? new Date().toISOString(),
|
|
15
|
+
projectPath: path.resolve(options.projectPath),
|
|
16
|
+
kitCommand: options.kitCommand,
|
|
17
|
+
totalRuns: options.runs.length,
|
|
18
|
+
completedRuns: options.runs.filter((run) => run.ok).length,
|
|
19
|
+
failedRuns: options.runs.filter((run) => !run.ok).length,
|
|
20
|
+
warnings: options.warnings
|
|
21
|
+
};
|
|
22
|
+
await writeFile(artifactPaths.summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
|
23
|
+
await writeFile(artifactPaths.runsPath, `${JSON.stringify({ generatedAt: summary.generatedAt, runs: options.runs }, null, 2)}\n`, "utf8");
|
|
24
|
+
return { summary, runs: options.runs, artifactPaths, warnings: options.warnings };
|
|
25
|
+
}
|