@dailephd/my-dev-kit-lab 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (250) hide show
  1. package/README.md +272 -0
  2. package/benchmarks/contracts/benchmark-project-profiles.json +1199 -0
  3. package/benchmarks/contracts/todo-behavior.md +70 -0
  4. package/benchmarks/contracts/todo-benchmark-case.json +227 -0
  5. package/benchmarks/projects/README.md +34 -0
  6. package/benchmarks/projects/task-analytics-large-mixed/README.md +1 -0
  7. package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/__init__.py +3 -0
  8. package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/fixtures.py +6 -0
  9. package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/metrics.py +29 -0
  10. package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/models.py +21 -0
  11. package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/parser.py +16 -0
  12. package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/pipeline.py +9 -0
  13. package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/quality.py +8 -0
  14. package/benchmarks/projects/task-analytics-large-mixed/py/task_analytics/reporting.py +11 -0
  15. package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_metrics.py +19 -0
  16. package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_parser.py +15 -0
  17. package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_quality.py +19 -0
  18. package/benchmarks/projects/task-analytics-large-mixed/py/tests/test_reporting.py +15 -0
  19. package/benchmarks/projects/task-analytics-large-mixed/ts/package.json +12 -0
  20. package/benchmarks/projects/task-analytics-large-mixed/ts/src/index.ts +11 -0
  21. package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/analyticsSnapshot.ts +20 -0
  22. package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/project.ts +5 -0
  23. package/benchmarks/projects/task-analytics-large-mixed/ts/src/models/task.ts +10 -0
  24. package/benchmarks/projects/task-analytics-large-mixed/ts/src/reporting/buildProjectLeaderboard.ts +7 -0
  25. package/benchmarks/projects/task-analytics-large-mixed/ts/src/reporting/formatTaskHealthReport.ts +13 -0
  26. package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/buildAnalyticsSnapshot.ts +39 -0
  27. package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/completeTask.ts +10 -0
  28. package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/createTask.ts +21 -0
  29. package/benchmarks/projects/task-analytics-large-mixed/ts/src/services/listTasksByProject.ts +6 -0
  30. package/benchmarks/projects/task-analytics-large-mixed/ts/src/store/projectStore.ts +20 -0
  31. package/benchmarks/projects/task-analytics-large-mixed/ts/src/store/taskStore.ts +44 -0
  32. package/benchmarks/projects/task-analytics-large-mixed/ts/src/validation/projectValidation.ts +12 -0
  33. package/benchmarks/projects/task-analytics-large-mixed/ts/src/validation/taskValidation.ts +18 -0
  34. package/benchmarks/projects/task-analytics-large-mixed/ts/tests/buildAnalyticsSnapshot.test.ts +48 -0
  35. package/benchmarks/projects/task-analytics-large-mixed/ts/tests/completeTask.test.ts +21 -0
  36. package/benchmarks/projects/task-analytics-large-mixed/ts/tests/createTask.test.ts +31 -0
  37. package/benchmarks/projects/task-analytics-large-mixed/ts/tests/listTasksByProject.test.ts +18 -0
  38. package/benchmarks/projects/task-analytics-large-mixed/ts/tests/reporting.test.ts +19 -0
  39. package/benchmarks/projects/task-analytics-large-mixed/ts/tsconfig.json +12 -0
  40. package/benchmarks/projects/task-analytics-large-mixed/ts/vitest.config.ts +5 -0
  41. package/benchmarks/projects/task-workflow-medium-ts/README.md +1 -0
  42. package/benchmarks/projects/task-workflow-medium-ts/package.json +12 -0
  43. package/benchmarks/projects/task-workflow-medium-ts/src/index.ts +9 -0
  44. package/benchmarks/projects/task-workflow-medium-ts/src/models/project.ts +6 -0
  45. package/benchmarks/projects/task-workflow-medium-ts/src/models/task.ts +39 -0
  46. package/benchmarks/projects/task-workflow-medium-ts/src/services/completeTask.ts +15 -0
  47. package/benchmarks/projects/task-workflow-medium-ts/src/services/createTask.ts +26 -0
  48. package/benchmarks/projects/task-workflow-medium-ts/src/services/filterTasks.ts +17 -0
  49. package/benchmarks/projects/task-workflow-medium-ts/src/services/importTasks.ts +33 -0
  50. package/benchmarks/projects/task-workflow-medium-ts/src/services/summarizeTasks.ts +30 -0
  51. package/benchmarks/projects/task-workflow-medium-ts/src/store/taskStore.ts +76 -0
  52. package/benchmarks/projects/task-workflow-medium-ts/src/utils/deterministicId.ts +3 -0
  53. package/benchmarks/projects/task-workflow-medium-ts/src/validation/taskValidation.ts +45 -0
  54. package/benchmarks/projects/task-workflow-medium-ts/tests/completeTask.test.ts +16 -0
  55. package/benchmarks/projects/task-workflow-medium-ts/tests/createTask.test.ts +21 -0
  56. package/benchmarks/projects/task-workflow-medium-ts/tests/filterTasks.test.ts +18 -0
  57. package/benchmarks/projects/task-workflow-medium-ts/tests/importTasks.test.ts +22 -0
  58. package/benchmarks/projects/task-workflow-medium-ts/tests/summarizeTasks.test.ts +29 -0
  59. package/benchmarks/projects/task-workflow-medium-ts/tsconfig.json +12 -0
  60. package/benchmarks/projects/task-workflow-medium-ts/vitest.config.ts +5 -0
  61. package/benchmarks/projects/todo-js/README.md +3 -0
  62. package/benchmarks/projects/todo-js/package.json +11 -0
  63. package/benchmarks/projects/todo-js/src/index.js +2 -0
  64. package/benchmarks/projects/todo-js/src/taskService.js +37 -0
  65. package/benchmarks/projects/todo-js/src/taskStore.js +28 -0
  66. package/benchmarks/projects/todo-js/tests/taskService.test.js +45 -0
  67. package/benchmarks/projects/todo-js/vitest.config.js +5 -0
  68. package/benchmarks/projects/todo-mixed-ts-py/README.md +3 -0
  69. package/benchmarks/projects/todo-mixed-ts-py/package.json +13 -0
  70. package/benchmarks/projects/todo-mixed-ts-py/python/task_service.py +76 -0
  71. package/benchmarks/projects/todo-mixed-ts-py/src/taskCli.ts +38 -0
  72. package/benchmarks/projects/todo-mixed-ts-py/tests/mixedBoundary.test.ts +18 -0
  73. package/benchmarks/projects/todo-mixed-ts-py/tsconfig.json +12 -0
  74. package/benchmarks/projects/todo-mixed-ts-py/vitest.config.ts +5 -0
  75. package/benchmarks/projects/todo-python/README.md +3 -0
  76. package/benchmarks/projects/todo-python/src/__init__.py +4 -0
  77. package/benchmarks/projects/todo-python/src/task_service.py +32 -0
  78. package/benchmarks/projects/todo-python/src/task_store.py +28 -0
  79. package/benchmarks/projects/todo-python/tests/test_task_service.py +52 -0
  80. package/benchmarks/projects/todo-ts/README.md +3 -0
  81. package/benchmarks/projects/todo-ts/package.json +12 -0
  82. package/benchmarks/projects/todo-ts/src/index.ts +2 -0
  83. package/benchmarks/projects/todo-ts/src/taskService.ts +41 -0
  84. package/benchmarks/projects/todo-ts/src/taskStore.ts +34 -0
  85. package/benchmarks/projects/todo-ts/tests/taskService.test.ts +45 -0
  86. package/benchmarks/projects/todo-ts/tsconfig.json +12 -0
  87. package/benchmarks/projects/todo-ts/vitest.config.ts +5 -0
  88. package/dist/scripts/build-gallery.js +3 -0
  89. package/dist/scripts/capture-demo-report.js +3 -0
  90. package/dist/scripts/evaluate-token-savings.js +2 -0
  91. package/dist/scripts/experiments/describeExperiment.js +143 -0
  92. package/dist/scripts/experiments/listExperiments.js +44 -0
  93. package/dist/scripts/experiments/runExperiment.js +199 -0
  94. package/dist/scripts/generate-experiment-plots.js +3 -0
  95. package/dist/scripts/generate-prompt-variants.js +2 -0
  96. package/dist/scripts/render-experiment-report.js +2 -0
  97. package/dist/scripts/run-agent-prompt.js +2 -0
  98. package/dist/scripts/run-controlled-experiment.js +2 -0
  99. package/dist/scripts/run-final-demo.js +3 -0
  100. package/dist/scripts/run-lab-demo.js +5 -0
  101. package/dist/scripts/run-visualization-demos.js +3 -0
  102. package/dist/scripts/security/runCodeql.js +57 -0
  103. package/dist/scripts/security/runDependencyChecks.js +57 -0
  104. package/dist/scripts/security/runFuzzSmoke.js +29 -0
  105. package/dist/scripts/security/runPackageChecks.js +56 -0
  106. package/dist/scripts/security/runSemgrep.js +63 -0
  107. package/dist/scripts/security/validate.js +117 -0
  108. package/dist/scripts/verify-benchmarks.js +202 -0
  109. package/dist/src/agents/adapters/claudeAdapter.js +37 -0
  110. package/dist/src/agents/adapters/codexAdapter.js +110 -0
  111. package/dist/src/agents/adapters/fakeAgentAdapter.js +101 -0
  112. package/dist/src/agents/agentRegistry.js +21 -0
  113. package/dist/src/agents/index.js +7 -0
  114. package/dist/src/agents/parseAgentTokenUsage.js +137 -0
  115. package/dist/src/agents/runAgentPrompt.js +38 -0
  116. package/dist/src/agents/types.js +1 -0
  117. package/dist/src/commands/buildGalleryCommand.js +56 -0
  118. package/dist/src/commands/captureDemoReport.js +116 -0
  119. package/dist/src/commands/evaluateTokenSavings.js +175 -0
  120. package/dist/src/commands/generateExperimentPlotsCommand.js +38 -0
  121. package/dist/src/commands/generatePromptVariants.js +67 -0
  122. package/dist/src/commands/renderExperimentReportCommand.js +131 -0
  123. package/dist/src/commands/runAgentPromptCommand.js +132 -0
  124. package/dist/src/commands/runControlledExperimentCommand.js +174 -0
  125. package/dist/src/commands/runFinalDemoCommand.js +123 -0
  126. package/dist/src/commands/runLabDemo.js +62 -0
  127. package/dist/src/commands/runVisualizationDemosCommand.js +67 -0
  128. package/dist/src/core/commandLine.js +59 -0
  129. package/dist/src/core/countTokens.js +8 -0
  130. package/dist/src/core/fileGlobs.js +100 -0
  131. package/dist/src/core/localProjectTarget.js +75 -0
  132. package/dist/src/core/pathSafety.js +19 -0
  133. package/dist/src/core/pythonCommand.js +30 -0
  134. package/dist/src/core/resolveCommand.js +110 -0
  135. package/dist/src/core/runMeasuredCommand.js +143 -0
  136. package/dist/src/evaluation/benchmarkMetadata.js +207 -0
  137. package/dist/src/evaluation/buildExperimentMatrix.js +75 -0
  138. package/dist/src/evaluation/classifyAgentRunOutcome.js +40 -0
  139. package/dist/src/evaluation/compareExperimentRuns.js +79 -0
  140. package/dist/src/evaluation/compareTokenSavings.js +47 -0
  141. package/dist/src/evaluation/controlledExperimentTypes.js +1 -0
  142. package/dist/src/evaluation/index.js +18 -0
  143. package/dist/src/evaluation/parseAgentAnswer.js +230 -0
  144. package/dist/src/evaluation/projectComplexity.js +126 -0
  145. package/dist/src/evaluation/projectFileTree.js +83 -0
  146. package/dist/src/evaluation/readEvaluationCases.js +59 -0
  147. package/dist/src/evaluation/renderTokenSavingsReportInput.js +55 -0
  148. package/dist/src/evaluation/runControlledExperiment.js +158 -0
  149. package/dist/src/evaluation/runMyDevKitRetrieval.js +197 -0
  150. package/dist/src/evaluation/runRawFullFileBaseline.js +31 -0
  151. package/dist/src/evaluation/scoreCorrectness.js +127 -0
  152. package/dist/src/evaluation/types.js +1 -0
  153. package/dist/src/evaluation/writeExperimentArtifacts.js +104 -0
  154. package/dist/src/evaluation/writeTokenSavingsArtifacts.js +57 -0
  155. package/dist/src/experiments/config.js +24 -0
  156. package/dist/src/experiments/defaultRegistry.js +7 -0
  157. package/dist/src/experiments/errors.js +18 -0
  158. package/dist/src/experiments/index.js +9 -0
  159. package/dist/src/experiments/outputPaths.js +25 -0
  160. package/dist/src/experiments/plugins/contextStrategyComparison/config.js +37 -0
  161. package/dist/src/experiments/plugins/contextStrategyComparison/index.js +3 -0
  162. package/dist/src/experiments/plugins/contextStrategyComparison/plugin.js +83 -0
  163. package/dist/src/experiments/plugins/contextStrategyComparison/resultMapping.js +260 -0
  164. package/dist/src/experiments/plugins/index.js +1 -0
  165. package/dist/src/experiments/registry.js +43 -0
  166. package/dist/src/experiments/results.js +48 -0
  167. package/dist/src/experiments/runner.js +181 -0
  168. package/dist/src/experiments/target.js +8 -0
  169. package/dist/src/experiments/types.js +1 -0
  170. package/dist/src/gallery/index.js +2 -0
  171. package/dist/src/gallery/types.js +1 -0
  172. package/dist/src/gallery/writeGalleryManifest.js +214 -0
  173. package/dist/src/index.js +12 -0
  174. package/dist/src/plots/buildExperimentPlotData.js +137 -0
  175. package/dist/src/plots/index.js +4 -0
  176. package/dist/src/plots/renderSvgChart.js +82 -0
  177. package/dist/src/plots/types.js +1 -0
  178. package/dist/src/plots/writePlotArtifacts.js +46 -0
  179. package/dist/src/prompts/buildPromptContext.js +68 -0
  180. package/dist/src/prompts/generateMyDevKitPrompt.js +106 -0
  181. package/dist/src/prompts/generatePromptVariants.js +36 -0
  182. package/dist/src/prompts/generateRawFullFilePrompt.js +97 -0
  183. package/dist/src/prompts/index.js +7 -0
  184. package/dist/src/prompts/measurePromptComplexity.js +41 -0
  185. package/dist/src/prompts/types.js +1 -0
  186. package/dist/src/prompts/writePromptArtifacts.js +43 -0
  187. package/dist/src/report/buildExperimentReportInput.js +339 -0
  188. package/dist/src/report/experimentReportTypes.js +1 -0
  189. package/dist/src/report/experiments/buildPluginExperimentReport.js +153 -0
  190. package/dist/src/report/experiments/experimentReportModel.js +1 -0
  191. package/dist/src/report/experiments/index.js +4 -0
  192. package/dist/src/report/experiments/renderPluginExperimentReportHtml.js +133 -0
  193. package/dist/src/report/experiments/writePluginExperimentReports.js +30 -0
  194. package/dist/src/report/index.js +8 -0
  195. package/dist/src/report/renderExperimentHtmlReport.js +354 -0
  196. package/dist/src/report/renderHtmlReport.js +103 -0
  197. package/dist/src/report/types.js +10 -0
  198. package/dist/src/report/writeExperimentReportArtifacts.js +38 -0
  199. package/dist/src/report/writeReportArtifacts.js +39 -0
  200. package/dist/src/screenshot/captureReportScreenshot.js +75 -0
  201. package/dist/src/screenshot/index.js +2 -0
  202. package/dist/src/screenshot/types.js +1 -0
  203. package/dist/src/securityValidation/artifacts.js +15 -0
  204. package/dist/src/securityValidation/cliAdversarial/adversarialCliConfig.js +38 -0
  205. package/dist/src/securityValidation/cliAdversarial/dataVolumeChecks.js +194 -0
  206. package/dist/src/securityValidation/cliAdversarial/jsonStdoutChecks.js +359 -0
  207. package/dist/src/securityValidation/cliAdversarial/malformedArtifactChecks.js +284 -0
  208. package/dist/src/securityValidation/cliAdversarial/malformedArtifactFixtures.js +79 -0
  209. package/dist/src/securityValidation/cliAdversarial/pathBoundaryChecks.js +431 -0
  210. package/dist/src/securityValidation/cliAdversarial/pathCases.js +144 -0
  211. package/dist/src/securityValidation/cliAdversarial/readOnlyBoundaryChecks.js +294 -0
  212. package/dist/src/securityValidation/cliAdversarial/runAdversarialCheck.js +149 -0
  213. package/dist/src/securityValidation/cliAdversarial/subprocessSafetyChecks.js +214 -0
  214. package/dist/src/securityValidation/cliAdversarial/tempWorkspace.js +160 -0
  215. package/dist/src/securityValidation/commandRunner.js +136 -0
  216. package/dist/src/securityValidation/config.js +39 -0
  217. package/dist/src/securityValidation/dependencies/parseNpmAudit.js +115 -0
  218. package/dist/src/securityValidation/dependencies/parseNpmLs.js +71 -0
  219. package/dist/src/securityValidation/dependencies/parseNpmOutdated.js +41 -0
  220. package/dist/src/securityValidation/dependencies/runDependencyChecks.js +239 -0
  221. package/dist/src/securityValidation/dependencies/runOsvScanner.js +43 -0
  222. package/dist/src/securityValidation/fuzz/fuzzHarness.js +61 -0
  223. package/dist/src/securityValidation/fuzz/fuzzTargets.js +204 -0
  224. package/dist/src/securityValidation/fuzz/randomInput.js +0 -0
  225. package/dist/src/securityValidation/index.js +34 -0
  226. package/dist/src/securityValidation/packageChecks/forbiddenPackageContents.js +67 -0
  227. package/dist/src/securityValidation/packageChecks/parseNpmPackDryRun.js +56 -0
  228. package/dist/src/securityValidation/packageChecks/runPackageChecks.js +88 -0
  229. package/dist/src/securityValidation/report/renderSecurityReport.js +248 -0
  230. package/dist/src/securityValidation/report/securityReportTypes.js +1 -0
  231. package/dist/src/securityValidation/staticScans/codeql.js +66 -0
  232. package/dist/src/securityValidation/staticScans/semgrep.js +180 -0
  233. package/dist/src/securityValidation/testMatrix.js +535 -0
  234. package/dist/src/securityValidation/types.js +34 -0
  235. package/dist/src/securityValidation/validate/resolveTarget.js +32 -0
  236. package/dist/src/securityValidation/validate/runSecurityValidation.js +169 -0
  237. package/dist/src/securityValidation/validate/verdict.js +73 -0
  238. package/dist/src/visualizationDemos/buildMyDevKitVisualizationCommands.js +59 -0
  239. package/dist/src/visualizationDemos/index.js +4 -0
  240. package/dist/src/visualizationDemos/runVisualizationDemos.js +82 -0
  241. package/dist/src/visualizationDemos/types.js +1 -0
  242. package/dist/src/visualizationDemos/writeVisualizationDemoArtifacts.js +25 -0
  243. package/docs/METRICS.md +286 -0
  244. package/examples/demo-report-input.json +78 -0
  245. package/examples/lab-demo-cases.json +35 -0
  246. package/examples/real-agent-campaign-cases.json +118 -0
  247. package/examples/token-savings-cases.json +122 -0
  248. package/package.json +91 -0
  249. package/tests/fixtures/fake-adversarial-cli.js +152 -0
  250. package/tests/fixtures/fake-my-dev-kit-cli.js +83 -0
@@ -0,0 +1,160 @@
1
+ import { createHash } from "node:crypto";
2
+ import { mkdirSync, mkdtempSync, readdirSync, readFileSync, statSync, writeFileSync, } from "node:fs";
3
+ import { rm } from "node:fs/promises";
4
+ import os from "node:os";
5
+ import path from "node:path";
6
+ // ---------------------------------------------------------------------------
7
+ // Workspace creation
8
+ // ---------------------------------------------------------------------------
9
+ export function createTempWorkspace(prefix = "adv-cli-") {
10
+ const root = mkdtempSync(path.join(os.tmpdir(), prefix));
11
+ const sourceDir = path.join(root, "source");
12
+ const outputDir = path.join(root, "output");
13
+ const indexDir = path.join(root, "index");
14
+ // Create outside dir as a sibling of root (not inside it).
15
+ const outsideDir = mkdtempSync(path.join(os.tmpdir(), `${prefix}outside-`));
16
+ mkdirSync(sourceDir, { recursive: true });
17
+ mkdirSync(outputDir, { recursive: true });
18
+ mkdirSync(indexDir, { recursive: true });
19
+ // Populate source dir with sentinel files that must not be modified.
20
+ writeFileSync(path.join(sourceDir, "index.ts"), "export const x = 1;\n", "utf8");
21
+ writeFileSync(path.join(sourceDir, "utils.ts"), "export function noop() {}\n", "utf8");
22
+ writeFileSync(path.join(sourceDir, "package.json"), JSON.stringify({ name: "sentinel-pkg", version: "1.0.0" }, null, 2) + "\n", "utf8");
23
+ // Place a sentinel file in outsideDir to verify it is not touched.
24
+ writeFileSync(path.join(outsideDir, "outside-sentinel.txt"), "must-not-be-modified\n", "utf8");
25
+ return {
26
+ root,
27
+ sourceDir,
28
+ outputDir,
29
+ indexDir,
30
+ outsideDir,
31
+ cleanup: async () => {
32
+ await Promise.all([
33
+ rm(root, { recursive: true, force: true }),
34
+ rm(outsideDir, { recursive: true, force: true }),
35
+ ]);
36
+ },
37
+ };
38
+ }
39
+ // ---------------------------------------------------------------------------
40
+ // File snapshotting
41
+ // ---------------------------------------------------------------------------
42
+ function hashFile(filePath) {
43
+ try {
44
+ const content = readFileSync(filePath);
45
+ return createHash("sha256").update(content).digest("hex");
46
+ }
47
+ catch {
48
+ return "<unreadable>";
49
+ }
50
+ }
51
+ export function snapshotDir(dir) {
52
+ const snapshots = [];
53
+ collectSnapshots(dir, dir, snapshots);
54
+ return snapshots.sort((a, b) => a.relativePath.localeCompare(b.relativePath));
55
+ }
56
+ function collectSnapshots(baseDir, currentDir, out) {
57
+ let entries;
58
+ try {
59
+ entries = readdirSync(currentDir);
60
+ }
61
+ catch {
62
+ return;
63
+ }
64
+ for (const entry of entries) {
65
+ const abs = path.join(currentDir, entry);
66
+ let stat;
67
+ try {
68
+ stat = statSync(abs);
69
+ }
70
+ catch {
71
+ continue;
72
+ }
73
+ if (stat.isDirectory()) {
74
+ collectSnapshots(baseDir, abs, out);
75
+ }
76
+ else if (stat.isFile()) {
77
+ out.push({
78
+ relativePath: path.relative(baseDir, abs),
79
+ absolutePath: abs,
80
+ contentHash: hashFile(abs),
81
+ });
82
+ }
83
+ }
84
+ }
85
+ export function diffSnapshots(before, after) {
86
+ const beforeMap = new Map(before.map((s) => [s.relativePath, s.contentHash]));
87
+ const afterMap = new Map(after.map((s) => [s.relativePath, s.contentHash]));
88
+ const added = [];
89
+ const removed = [];
90
+ const modified = [];
91
+ for (const [rel, hash] of afterMap) {
92
+ if (!beforeMap.has(rel)) {
93
+ added.push(rel);
94
+ }
95
+ else if (beforeMap.get(rel) !== hash) {
96
+ modified.push(rel);
97
+ }
98
+ }
99
+ for (const rel of beforeMap.keys()) {
100
+ if (!afterMap.has(rel)) {
101
+ removed.push(rel);
102
+ }
103
+ }
104
+ return { added, removed, modified };
105
+ }
106
+ // ---------------------------------------------------------------------------
107
+ // Write-escape detection
108
+ // ---------------------------------------------------------------------------
109
+ /**
110
+ * Returns any files found in `checkDir` that are NOT underneath any of the
111
+ * `allowedDirs`. Used to detect writes outside declared output paths.
112
+ */
113
+ export function findWritesOutside(checkDir, allowedDirs) {
114
+ const normalizedAllowed = allowedDirs.map((d) => path.resolve(d) + path.sep);
115
+ const escaped = [];
116
+ collectEscapedFiles(checkDir, normalizedAllowed, escaped);
117
+ return escaped;
118
+ }
119
+ function collectEscapedFiles(dir, allowedDirs, out) {
120
+ let entries;
121
+ try {
122
+ entries = readdirSync(dir);
123
+ }
124
+ catch {
125
+ return;
126
+ }
127
+ for (const entry of entries) {
128
+ const abs = path.join(dir, entry);
129
+ let stat;
130
+ try {
131
+ stat = statSync(abs);
132
+ }
133
+ catch {
134
+ continue;
135
+ }
136
+ const normalizedAbs = path.resolve(abs);
137
+ const isAllowed = allowedDirs.some((allowed) => normalizedAbs === allowed.slice(0, -1) ||
138
+ normalizedAbs.startsWith(allowed));
139
+ if (!isAllowed) {
140
+ if (stat.isDirectory()) {
141
+ collectEscapedFiles(abs, allowedDirs, out);
142
+ }
143
+ else {
144
+ out.push(normalizedAbs);
145
+ }
146
+ }
147
+ else if (stat.isDirectory()) {
148
+ collectEscapedFiles(abs, allowedDirs, out);
149
+ }
150
+ }
151
+ }
152
+ /**
153
+ * Returns files in `dir` that were not present in the `beforeSnapshot`.
154
+ * Simple alternative to `findWritesOutside` when checking one directory.
155
+ */
156
+ export function findNewFiles(dir, beforeSnapshot) {
157
+ const afterSnapshot = snapshotDir(dir);
158
+ const diff = diffSnapshots(beforeSnapshot, afterSnapshot);
159
+ return diff.added.map((rel) => path.join(dir, rel));
160
+ }
@@ -0,0 +1,136 @@
1
+ import { spawn } from "node:child_process";
2
+ import { resolveCommand } from "../core/resolveCommand.js";
3
+ // Prefer the unqualified npm command and let the shared resolver locate
4
+ // npm.cmd/.ps1 shims or direct executables per platform.
5
+ export function resolveNpmCommand() {
6
+ return "npm";
7
+ }
8
+ // Runs a security check command with an argument array (no shell interpolation).
9
+ // npm audit returns exit code 1 on vulnerabilities; that is not treated as a
10
+ // spawn failure here — the caller inspects the output and exit code separately.
11
+ export async function runSecurityCommand(options) {
12
+ const started = Date.now();
13
+ const resolved = resolveCommand(options.command, {
14
+ cwd: options.cwd,
15
+ env: process.env,
16
+ allowPowerShellShim: false,
17
+ });
18
+ const command = resolved.command;
19
+ const needsResolvedPathArg = resolved.resolutionKind === "windows-cmd-shim" ||
20
+ resolved.resolutionKind === "windows-powershell-shim";
21
+ const args = [
22
+ ...resolved.argsPrefix,
23
+ ...(needsResolvedPathArg && resolved.resolvedPath ? [resolved.resolvedPath] : []),
24
+ ...options.args,
25
+ ];
26
+ if (resolved.resolutionKind === "unavailable") {
27
+ return {
28
+ command: options.command,
29
+ args: options.args,
30
+ cwd: options.cwd,
31
+ exitCode: null,
32
+ durationMs: Date.now() - started,
33
+ stdout: "",
34
+ stderr: resolved.warnings.join("\n"),
35
+ timedOut: false,
36
+ skipped: false,
37
+ };
38
+ }
39
+ return new Promise((resolve) => {
40
+ let stdout = "";
41
+ let stderr = "";
42
+ let timedOut = false;
43
+ let settled = false;
44
+ let timeout;
45
+ let child;
46
+ try {
47
+ child = spawn(command, args, {
48
+ cwd: options.cwd,
49
+ shell: false,
50
+ stdio: ["ignore", "pipe", "pipe"],
51
+ });
52
+ }
53
+ catch (spawnErr) {
54
+ resolve({
55
+ command,
56
+ args,
57
+ cwd: options.cwd,
58
+ exitCode: null,
59
+ durationMs: Date.now() - started,
60
+ stdout: "",
61
+ stderr: spawnErr instanceof Error ? spawnErr.message : String(spawnErr),
62
+ timedOut: false,
63
+ skipped: false,
64
+ });
65
+ return;
66
+ }
67
+ child.stdout.on("data", (chunk) => {
68
+ stdout += chunk.toString("utf8");
69
+ });
70
+ child.stderr.on("data", (chunk) => {
71
+ stderr += chunk.toString("utf8");
72
+ });
73
+ child.on("error", (error) => {
74
+ if (settled)
75
+ return;
76
+ settled = true;
77
+ if (timeout)
78
+ clearTimeout(timeout);
79
+ resolve({
80
+ command,
81
+ args,
82
+ cwd: options.cwd,
83
+ exitCode: null,
84
+ durationMs: Date.now() - started,
85
+ stdout,
86
+ stderr: error.message,
87
+ timedOut,
88
+ skipped: false,
89
+ });
90
+ });
91
+ if (options.timeoutMs > 0) {
92
+ timeout = setTimeout(() => {
93
+ timedOut = true;
94
+ try {
95
+ child.kill("SIGTERM");
96
+ }
97
+ catch {
98
+ // Process may have already exited.
99
+ }
100
+ }, options.timeoutMs);
101
+ }
102
+ child.on("close", (exitCode) => {
103
+ if (settled)
104
+ return;
105
+ settled = true;
106
+ if (timeout)
107
+ clearTimeout(timeout);
108
+ resolve({
109
+ command,
110
+ args,
111
+ cwd: options.cwd,
112
+ exitCode,
113
+ durationMs: Date.now() - started,
114
+ stdout,
115
+ stderr,
116
+ timedOut,
117
+ skipped: false,
118
+ });
119
+ });
120
+ });
121
+ }
122
+ // Returns a skipped result when a tool is not available.
123
+ export function skippedResult(options) {
124
+ return {
125
+ command: options.command,
126
+ args: options.args,
127
+ cwd: options.cwd,
128
+ exitCode: null,
129
+ durationMs: 0,
130
+ stdout: "",
131
+ stderr: "",
132
+ timedOut: false,
133
+ skipped: true,
134
+ skippedReason: options.reason,
135
+ };
136
+ }
@@ -0,0 +1,39 @@
1
+ import path from "path";
2
+ // Patterns that should not appear in the npm tarball.
3
+ // These represent generated lab artifacts, secrets, and machine-specific files
4
+ // that are safe in the repo but must not be published.
5
+ const DEFAULT_FORBIDDEN_PATTERNS = [
6
+ "lab-output/",
7
+ ".my-dev-kit/",
8
+ "reports/security/",
9
+ "reports/security/raw/",
10
+ "__pycache__/",
11
+ "*.pyc",
12
+ ".env",
13
+ ".env.",
14
+ "*.pem",
15
+ "*.key",
16
+ "*.p12",
17
+ "*.tgz",
18
+ "*.tar",
19
+ "node_modules/",
20
+ "coverage/",
21
+ "tmp/",
22
+ "temp/",
23
+ "smoke/",
24
+ "docs/coding_generation_guideline.md",
25
+ "docs/DOCUMENTATION_AUDIT_REPORT.txt",
26
+ "docs/EXPERIMENT_REPORT_MIGRATION_PLAN.md",
27
+ "docs/FINAL_BATCH_HANDOFF.txt",
28
+ "docs/FINAL_VERIFICATION_REPORT.txt",
29
+ "docs/PUBLIC_RELEASE_CHECKLIST.md",
30
+ "docs/REAL_AGENT_BENCHMARK_CAMPAIGN_REPORT.txt",
31
+ ];
32
+ export const DEFAULT_SECURITY_CONFIG = {
33
+ reportDir: path.join("reports", "security"),
34
+ rawOutputDir: path.join("reports", "security", "raw"),
35
+ forbiddenPackagePatterns: DEFAULT_FORBIDDEN_PATTERNS,
36
+ allowedPackageExceptions: [],
37
+ commandTimeoutMs: 60_000,
38
+ requireOsvScanner: false,
39
+ };
@@ -0,0 +1,115 @@
1
+ const EMPTY_COUNTS = {
2
+ critical: 0,
3
+ high: 0,
4
+ moderate: 0,
5
+ low: 0,
6
+ info: 0,
7
+ total: 0,
8
+ };
9
+ function npmSeverityToInternal(npmSeverity) {
10
+ switch (npmSeverity.toLowerCase()) {
11
+ case "critical":
12
+ case "high":
13
+ return "blocker";
14
+ case "moderate":
15
+ return "major";
16
+ case "low":
17
+ return "minor";
18
+ default:
19
+ return "informational";
20
+ }
21
+ }
22
+ // Parse npm audit --json stdout into structured findings.
23
+ // npm audit exits with code 1 when vulnerabilities are found; that is not an
24
+ // error here — the exit code is checked separately by the runner.
25
+ export function parseNpmAudit(stdout, checkId) {
26
+ if (!stdout.trim()) {
27
+ return {
28
+ ok: true,
29
+ severityCounts: { ...EMPTY_COUNTS },
30
+ findings: [],
31
+ };
32
+ }
33
+ let parsed;
34
+ try {
35
+ parsed = JSON.parse(stdout);
36
+ }
37
+ catch (err) {
38
+ return {
39
+ ok: false,
40
+ severityCounts: { ...EMPTY_COUNTS },
41
+ findings: [],
42
+ parseError: `Failed to parse npm audit JSON: ${err instanceof Error ? err.message : String(err)}`,
43
+ };
44
+ }
45
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
46
+ return {
47
+ ok: false,
48
+ severityCounts: { ...EMPTY_COUNTS },
49
+ findings: [],
50
+ parseError: "npm audit JSON is not an object",
51
+ };
52
+ }
53
+ const auditData = parsed;
54
+ const metadata = auditData["metadata"];
55
+ const vulnerabilities = metadata?.["vulnerabilities"];
56
+ const counts = {
57
+ critical: Number(vulnerabilities?.["critical"] ?? 0),
58
+ high: Number(vulnerabilities?.["high"] ?? 0),
59
+ moderate: Number(vulnerabilities?.["moderate"] ?? 0),
60
+ low: Number(vulnerabilities?.["low"] ?? 0),
61
+ info: Number(vulnerabilities?.["info"] ?? 0),
62
+ total: Number(vulnerabilities?.["total"] ?? 0),
63
+ };
64
+ const findings = [];
65
+ const vulnsMap = auditData["vulnerabilities"];
66
+ if (vulnsMap) {
67
+ for (const [pkgName, vulnData] of Object.entries(vulnsMap)) {
68
+ const vuln = vulnData;
69
+ const severity = typeof vuln["severity"] === "string" ? vuln["severity"] : "moderate";
70
+ const via = vuln["via"];
71
+ const advisories = Array.isArray(via)
72
+ ? via.filter((v) => typeof v === "object" && v !== null && "title" in v)
73
+ : [];
74
+ if (advisories.length > 0) {
75
+ for (const advisory of advisories) {
76
+ findings.push({
77
+ id: `${checkId}-${pkgName}-${String(advisory["source"] ?? advisory["title"] ?? "vuln")}`,
78
+ title: `Vulnerability in ${pkgName}: ${String(advisory["title"] ?? "unknown")}`,
79
+ severity: npmSeverityToInternal(severity),
80
+ category: "dependency-audit",
81
+ description: String(advisory["title"] ?? "No description available"),
82
+ evidence: `Package: ${pkgName}, Severity: ${severity}, Range: ${String(vuln["range"] ?? "unknown")}`,
83
+ affectedFiles: [],
84
+ recommendation: `Update ${pkgName} to a fixed version. See: ${String(advisory["url"] ?? "npm audit for details")}`,
85
+ releaseImpact: severity === "critical" || severity === "high"
86
+ ? "Blocker: must fix before release"
87
+ : severity === "moderate"
88
+ ? "Major: should fix before release"
89
+ : "Minor: can fix in patch release",
90
+ });
91
+ }
92
+ }
93
+ else {
94
+ findings.push({
95
+ id: `${checkId}-${pkgName}-${severity}`,
96
+ title: `Vulnerability in ${pkgName}`,
97
+ severity: npmSeverityToInternal(severity),
98
+ category: "dependency-audit",
99
+ description: `${pkgName} has a ${severity} severity vulnerability`,
100
+ evidence: `Package: ${pkgName}, Severity: ${severity}`,
101
+ affectedFiles: [],
102
+ recommendation: `Update ${pkgName} to a patched version`,
103
+ releaseImpact: severity === "critical" || severity === "high"
104
+ ? "Blocker: must fix before release"
105
+ : "Minor: review before release",
106
+ });
107
+ }
108
+ }
109
+ }
110
+ return {
111
+ ok: counts.total === 0,
112
+ severityCounts: counts,
113
+ findings,
114
+ };
115
+ }
@@ -0,0 +1,71 @@
1
+ // Parse npm ls --all --json stdout for missing or invalid dependencies.
2
+ // npm ls exits non-zero when there are problems; that is expected and handled here.
3
+ export function parseNpmLs(stdout, checkId) {
4
+ if (!stdout.trim()) {
5
+ return { ok: true, missingCount: 0, invalidCount: 0, findings: [] };
6
+ }
7
+ let parsed;
8
+ try {
9
+ parsed = JSON.parse(stdout);
10
+ }
11
+ catch (err) {
12
+ return {
13
+ ok: false,
14
+ missingCount: 0,
15
+ invalidCount: 0,
16
+ findings: [],
17
+ parseError: `Failed to parse npm ls JSON: ${err instanceof Error ? err.message : String(err)}`,
18
+ };
19
+ }
20
+ const findings = [];
21
+ let missingCount = 0;
22
+ let invalidCount = 0;
23
+ function walkDeps(node, depth) {
24
+ if (depth > 20 || typeof node !== "object" || node === null)
25
+ return;
26
+ const n = node;
27
+ if (n["missing"] === true) {
28
+ missingCount++;
29
+ const name = typeof n["name"] === "string" ? n["name"] : "unknown";
30
+ findings.push({
31
+ id: `${checkId}-missing-${name}-${missingCount}`,
32
+ title: `Missing dependency: ${name}`,
33
+ severity: "major",
34
+ category: "dependency-audit",
35
+ description: `Dependency '${name}' is declared but not installed`,
36
+ evidence: `npm ls reported missing: ${name}`,
37
+ affectedFiles: ["package.json"],
38
+ recommendation: "Run npm install or npm ci to restore the missing dependency",
39
+ releaseImpact: "Major: missing dependency may cause build or runtime failures",
40
+ });
41
+ }
42
+ if (n["invalid"] === true) {
43
+ invalidCount++;
44
+ const name = typeof n["name"] === "string" ? n["name"] : "unknown";
45
+ findings.push({
46
+ id: `${checkId}-invalid-${name}-${invalidCount}`,
47
+ title: `Invalid dependency: ${name}`,
48
+ severity: "minor",
49
+ category: "dependency-audit",
50
+ description: `Dependency '${name}' version does not satisfy the declared range`,
51
+ evidence: `npm ls reported invalid: ${name}`,
52
+ affectedFiles: ["package.json"],
53
+ recommendation: "Run npm install to resolve version mismatches",
54
+ releaseImpact: "Minor: version mismatch; review before release",
55
+ });
56
+ }
57
+ const deps = n["dependencies"];
58
+ if (typeof deps === "object" && deps !== null) {
59
+ for (const child of Object.values(deps)) {
60
+ walkDeps(child, depth + 1);
61
+ }
62
+ }
63
+ }
64
+ walkDeps(parsed, 0);
65
+ return {
66
+ ok: missingCount === 0 && invalidCount === 0,
67
+ missingCount,
68
+ invalidCount,
69
+ findings,
70
+ };
71
+ }
@@ -0,0 +1,41 @@
1
+ // Parse npm outdated --json stdout.
2
+ // npm outdated exits 1 when packages are outdated; that is expected.
3
+ export function parseNpmOutdated(stdout, checkId) {
4
+ if (!stdout.trim()) {
5
+ return { outdatedCount: 0, findings: [] };
6
+ }
7
+ let parsed;
8
+ try {
9
+ parsed = JSON.parse(stdout);
10
+ }
11
+ catch (err) {
12
+ return {
13
+ outdatedCount: 0,
14
+ findings: [],
15
+ parseError: `Failed to parse npm outdated JSON: ${err instanceof Error ? err.message : String(err)}`,
16
+ };
17
+ }
18
+ if (typeof parsed !== "object" || parsed === null) {
19
+ return { outdatedCount: 0, findings: [] };
20
+ }
21
+ const outdatedMap = parsed;
22
+ const entries = Object.entries(outdatedMap);
23
+ const findings = entries.map(([pkgName, info]) => {
24
+ const pkg = info !== null && typeof info === "object" ? info : {};
25
+ return {
26
+ id: `${checkId}-outdated-${pkgName}`,
27
+ title: `Outdated dependency: ${pkgName}`,
28
+ severity: "informational",
29
+ category: "dependency-audit",
30
+ description: `${pkgName} has a newer version available`,
31
+ evidence: `Current: ${pkg["current"] ?? "unknown"}, Wanted: ${pkg["wanted"] ?? "unknown"}, Latest: ${pkg["latest"] ?? "unknown"}`,
32
+ affectedFiles: ["package.json"],
33
+ recommendation: `Run npm install ${pkgName}@latest to update`,
34
+ releaseImpact: "Informational: no direct release impact; consider updating before major releases",
35
+ };
36
+ });
37
+ return {
38
+ outdatedCount: entries.length,
39
+ findings,
40
+ };
41
+ }