@xiuper/cli 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. package/README.md +180 -0
  2. package/dist/jsMain/typescript/agents/LLMService.d.ts +40 -0
  3. package/dist/jsMain/typescript/agents/LLMService.d.ts.map +1 -0
  4. package/dist/jsMain/typescript/agents/LLMService.js +156 -0
  5. package/dist/jsMain/typescript/agents/LLMService.js.map +1 -0
  6. package/dist/jsMain/typescript/agents/ServerAgentClient.d.ts +103 -0
  7. package/dist/jsMain/typescript/agents/ServerAgentClient.d.ts.map +1 -0
  8. package/dist/jsMain/typescript/agents/ServerAgentClient.js +142 -0
  9. package/dist/jsMain/typescript/agents/ServerAgentClient.js.map +1 -0
  10. package/dist/jsMain/typescript/agents/render/BaseRenderer.d.ts +87 -0
  11. package/dist/jsMain/typescript/agents/render/BaseRenderer.d.ts.map +1 -0
  12. package/dist/jsMain/typescript/agents/render/BaseRenderer.js +128 -0
  13. package/dist/jsMain/typescript/agents/render/BaseRenderer.js.map +1 -0
  14. package/dist/jsMain/typescript/agents/render/CliRenderer.d.ts +51 -0
  15. package/dist/jsMain/typescript/agents/render/CliRenderer.d.ts.map +1 -0
  16. package/dist/jsMain/typescript/agents/render/CliRenderer.js +564 -0
  17. package/dist/jsMain/typescript/agents/render/CliRenderer.js.map +1 -0
  18. package/dist/jsMain/typescript/agents/render/ServerRenderer.d.ts +55 -0
  19. package/dist/jsMain/typescript/agents/render/ServerRenderer.d.ts.map +1 -0
  20. package/dist/jsMain/typescript/agents/render/ServerRenderer.js +503 -0
  21. package/dist/jsMain/typescript/agents/render/ServerRenderer.js.map +1 -0
  22. package/dist/jsMain/typescript/agents/render/TuiRenderer.d.ts +104 -0
  23. package/dist/jsMain/typescript/agents/render/TuiRenderer.d.ts.map +1 -0
  24. package/dist/jsMain/typescript/agents/render/TuiRenderer.js +279 -0
  25. package/dist/jsMain/typescript/agents/render/TuiRenderer.js.map +1 -0
  26. package/dist/jsMain/typescript/config/ConfigManager.d.ts +146 -0
  27. package/dist/jsMain/typescript/config/ConfigManager.d.ts.map +1 -0
  28. package/dist/jsMain/typescript/config/ConfigManager.js +257 -0
  29. package/dist/jsMain/typescript/config/ConfigManager.js.map +1 -0
  30. package/dist/jsMain/typescript/constants/asciiArt.d.ts +9 -0
  31. package/dist/jsMain/typescript/constants/asciiArt.d.ts.map +1 -0
  32. package/dist/jsMain/typescript/constants/asciiArt.js +47 -0
  33. package/dist/jsMain/typescript/constants/asciiArt.js.map +1 -0
  34. package/dist/jsMain/typescript/design-system/colors.d.ts +492 -0
  35. package/dist/jsMain/typescript/design-system/colors.d.ts.map +1 -0
  36. package/dist/jsMain/typescript/design-system/colors.js +333 -0
  37. package/dist/jsMain/typescript/design-system/colors.js.map +1 -0
  38. package/dist/jsMain/typescript/design-system/index.d.ts +31 -0
  39. package/dist/jsMain/typescript/design-system/index.d.ts.map +1 -0
  40. package/dist/jsMain/typescript/design-system/index.js +195 -0
  41. package/dist/jsMain/typescript/design-system/index.js.map +1 -0
  42. package/dist/jsMain/typescript/design-system/theme-helpers.d.ts +161 -0
  43. package/dist/jsMain/typescript/design-system/theme-helpers.d.ts.map +1 -0
  44. package/dist/jsMain/typescript/design-system/theme-helpers.js +153 -0
  45. package/dist/jsMain/typescript/design-system/theme-helpers.js.map +1 -0
  46. package/dist/jsMain/typescript/i18n/index.d.ts +37 -0
  47. package/dist/jsMain/typescript/i18n/index.d.ts.map +1 -0
  48. package/dist/jsMain/typescript/i18n/index.js +139 -0
  49. package/dist/jsMain/typescript/i18n/index.js.map +1 -0
  50. package/dist/jsMain/typescript/i18n/locales/en.d.ts +6 -0
  51. package/dist/jsMain/typescript/i18n/locales/en.d.ts.map +1 -0
  52. package/dist/jsMain/typescript/i18n/locales/en.js +120 -0
  53. package/dist/jsMain/typescript/i18n/locales/en.js.map +1 -0
  54. package/dist/jsMain/typescript/i18n/locales/zh.d.ts +6 -0
  55. package/dist/jsMain/typescript/i18n/locales/zh.d.ts.map +1 -0
  56. package/dist/jsMain/typescript/i18n/locales/zh.js +120 -0
  57. package/dist/jsMain/typescript/i18n/locales/zh.js.map +1 -0
  58. package/dist/jsMain/typescript/i18n/types.d.ts +124 -0
  59. package/dist/jsMain/typescript/i18n/types.d.ts.map +1 -0
  60. package/dist/jsMain/typescript/i18n/types.js +7 -0
  61. package/dist/jsMain/typescript/i18n/types.js.map +1 -0
  62. package/dist/jsMain/typescript/index.d.ts +9 -0
  63. package/dist/jsMain/typescript/index.d.ts.map +1 -0
  64. package/dist/jsMain/typescript/index.js +380 -0
  65. package/dist/jsMain/typescript/index.js.map +1 -0
  66. package/dist/jsMain/typescript/modes/AgentMode.d.ts +33 -0
  67. package/dist/jsMain/typescript/modes/AgentMode.d.ts.map +1 -0
  68. package/dist/jsMain/typescript/modes/AgentMode.js +243 -0
  69. package/dist/jsMain/typescript/modes/AgentMode.js.map +1 -0
  70. package/dist/jsMain/typescript/modes/ChatMode.d.ts +30 -0
  71. package/dist/jsMain/typescript/modes/ChatMode.d.ts.map +1 -0
  72. package/dist/jsMain/typescript/modes/ChatMode.js +260 -0
  73. package/dist/jsMain/typescript/modes/ChatMode.js.map +1 -0
  74. package/dist/jsMain/typescript/modes/DocumentMode.d.ts +26 -0
  75. package/dist/jsMain/typescript/modes/DocumentMode.d.ts.map +1 -0
  76. package/dist/jsMain/typescript/modes/DocumentMode.js +217 -0
  77. package/dist/jsMain/typescript/modes/DocumentMode.js.map +1 -0
  78. package/dist/jsMain/typescript/modes/Mode.d.ts +72 -0
  79. package/dist/jsMain/typescript/modes/Mode.d.ts.map +1 -0
  80. package/dist/jsMain/typescript/modes/Mode.js +7 -0
  81. package/dist/jsMain/typescript/modes/Mode.js.map +1 -0
  82. package/dist/jsMain/typescript/modes/ModeManager.d.ts +76 -0
  83. package/dist/jsMain/typescript/modes/ModeManager.d.ts.map +1 -0
  84. package/dist/jsMain/typescript/modes/ModeManager.js +156 -0
  85. package/dist/jsMain/typescript/modes/ModeManager.js.map +1 -0
  86. package/dist/jsMain/typescript/modes/ReviewMode.d.ts +52 -0
  87. package/dist/jsMain/typescript/modes/ReviewMode.d.ts.map +1 -0
  88. package/dist/jsMain/typescript/modes/ReviewMode.js +435 -0
  89. package/dist/jsMain/typescript/modes/ReviewMode.js.map +1 -0
  90. package/dist/jsMain/typescript/modes/index.d.ts +9 -0
  91. package/dist/jsMain/typescript/modes/index.d.ts.map +1 -0
  92. package/dist/jsMain/typescript/modes/index.js +7 -0
  93. package/dist/jsMain/typescript/modes/index.js.map +1 -0
  94. package/dist/jsMain/typescript/processors/AtCommandProcessor.d.ts +34 -0
  95. package/dist/jsMain/typescript/processors/AtCommandProcessor.d.ts.map +1 -0
  96. package/dist/jsMain/typescript/processors/AtCommandProcessor.js +118 -0
  97. package/dist/jsMain/typescript/processors/AtCommandProcessor.js.map +1 -0
  98. package/dist/jsMain/typescript/processors/InputRouter.d.ts +84 -0
  99. package/dist/jsMain/typescript/processors/InputRouter.d.ts.map +1 -0
  100. package/dist/jsMain/typescript/processors/InputRouter.js +78 -0
  101. package/dist/jsMain/typescript/processors/InputRouter.js.map +1 -0
  102. package/dist/jsMain/typescript/processors/ModeCommandProcessor.d.ts +25 -0
  103. package/dist/jsMain/typescript/processors/ModeCommandProcessor.d.ts.map +1 -0
  104. package/dist/jsMain/typescript/processors/ModeCommandProcessor.js +127 -0
  105. package/dist/jsMain/typescript/processors/ModeCommandProcessor.js.map +1 -0
  106. package/dist/jsMain/typescript/processors/SlashCommandProcessor.d.ts +68 -0
  107. package/dist/jsMain/typescript/processors/SlashCommandProcessor.d.ts.map +1 -0
  108. package/dist/jsMain/typescript/processors/SlashCommandProcessor.js +391 -0
  109. package/dist/jsMain/typescript/processors/SlashCommandProcessor.js.map +1 -0
  110. package/dist/jsMain/typescript/processors/VariableProcessor.d.ts +44 -0
  111. package/dist/jsMain/typescript/processors/VariableProcessor.d.ts.map +1 -0
  112. package/dist/jsMain/typescript/processors/VariableProcessor.js +105 -0
  113. package/dist/jsMain/typescript/processors/VariableProcessor.js.map +1 -0
  114. package/dist/jsMain/typescript/ui/App.d.ts +16 -0
  115. package/dist/jsMain/typescript/ui/App.d.ts.map +1 -0
  116. package/dist/jsMain/typescript/ui/App.js +208 -0
  117. package/dist/jsMain/typescript/ui/App.js.map +1 -0
  118. package/dist/jsMain/typescript/ui/Banner.d.ts +6 -0
  119. package/dist/jsMain/typescript/ui/Banner.d.ts.map +1 -0
  120. package/dist/jsMain/typescript/ui/Banner.js +13 -0
  121. package/dist/jsMain/typescript/ui/Banner.js.map +1 -0
  122. package/dist/jsMain/typescript/ui/ChatInterface.d.ts +19 -0
  123. package/dist/jsMain/typescript/ui/ChatInterface.d.ts.map +1 -0
  124. package/dist/jsMain/typescript/ui/ChatInterface.js +223 -0
  125. package/dist/jsMain/typescript/ui/ChatInterface.js.map +1 -0
  126. package/dist/jsMain/typescript/ui/CommandSuggestions.d.ts +21 -0
  127. package/dist/jsMain/typescript/ui/CommandSuggestions.d.ts.map +1 -0
  128. package/dist/jsMain/typescript/ui/CommandSuggestions.js +43 -0
  129. package/dist/jsMain/typescript/ui/CommandSuggestions.js.map +1 -0
  130. package/dist/jsMain/typescript/ui/LanguageSwitcher.d.ts +11 -0
  131. package/dist/jsMain/typescript/ui/LanguageSwitcher.d.ts.map +1 -0
  132. package/dist/jsMain/typescript/ui/LanguageSwitcher.js +37 -0
  133. package/dist/jsMain/typescript/ui/LanguageSwitcher.js.map +1 -0
  134. package/dist/jsMain/typescript/ui/MessageRenderer.d.ts +70 -0
  135. package/dist/jsMain/typescript/ui/MessageRenderer.d.ts.map +1 -0
  136. package/dist/jsMain/typescript/ui/MessageRenderer.js +255 -0
  137. package/dist/jsMain/typescript/ui/MessageRenderer.js.map +1 -0
  138. package/dist/jsMain/typescript/ui/ModelConfigForm.d.ts +19 -0
  139. package/dist/jsMain/typescript/ui/ModelConfigForm.d.ts.map +1 -0
  140. package/dist/jsMain/typescript/ui/ModelConfigForm.js +257 -0
  141. package/dist/jsMain/typescript/ui/ModelConfigForm.js.map +1 -0
  142. package/dist/jsMain/typescript/ui/WelcomeScreen.d.ts +12 -0
  143. package/dist/jsMain/typescript/ui/WelcomeScreen.d.ts.map +1 -0
  144. package/dist/jsMain/typescript/ui/WelcomeScreen.js +93 -0
  145. package/dist/jsMain/typescript/ui/WelcomeScreen.js.map +1 -0
  146. package/dist/jsMain/typescript/utils/commandUtils.d.ts +64 -0
  147. package/dist/jsMain/typescript/utils/commandUtils.d.ts.map +1 -0
  148. package/dist/jsMain/typescript/utils/commandUtils.js +195 -0
  149. package/dist/jsMain/typescript/utils/commandUtils.js.map +1 -0
  150. package/dist/jsMain/typescript/utils/domainDictUtils.d.ts +53 -0
  151. package/dist/jsMain/typescript/utils/domainDictUtils.d.ts.map +1 -0
  152. package/dist/jsMain/typescript/utils/domainDictUtils.js +130 -0
  153. package/dist/jsMain/typescript/utils/domainDictUtils.js.map +1 -0
  154. package/dist/jsMain/typescript/utils/markdownSplitter.d.ts +23 -0
  155. package/dist/jsMain/typescript/utils/markdownSplitter.d.ts.map +1 -0
  156. package/dist/jsMain/typescript/utils/markdownSplitter.js +90 -0
  157. package/dist/jsMain/typescript/utils/markdownSplitter.js.map +1 -0
  158. package/dist/jsMain/typescript/utils/outputFormatter.d.ts +70 -0
  159. package/dist/jsMain/typescript/utils/outputFormatter.d.ts.map +1 -0
  160. package/dist/jsMain/typescript/utils/outputFormatter.js +140 -0
  161. package/dist/jsMain/typescript/utils/outputFormatter.js.map +1 -0
  162. package/dist/jsMain/typescript/utils/renderUtils.d.ts +17 -0
  163. package/dist/jsMain/typescript/utils/renderUtils.d.ts.map +1 -0
  164. package/dist/jsMain/typescript/utils/renderUtils.js +128 -0
  165. package/dist/jsMain/typescript/utils/renderUtils.js.map +1 -0
  166. package/dist/test/framework/analyzers/CodeChangeAnalyzer.d.ts +87 -0
  167. package/dist/test/framework/analyzers/CodeChangeAnalyzer.d.ts.map +1 -0
  168. package/dist/test/framework/analyzers/CodeChangeAnalyzer.js +412 -0
  169. package/dist/test/framework/analyzers/CodeChangeAnalyzer.js.map +1 -0
  170. package/dist/test/framework/analyzers/PromptAnalyzer.d.ts +46 -0
  171. package/dist/test/framework/analyzers/PromptAnalyzer.d.ts.map +1 -0
  172. package/dist/test/framework/analyzers/PromptAnalyzer.js +185 -0
  173. package/dist/test/framework/analyzers/PromptAnalyzer.js.map +1 -0
  174. package/dist/test/framework/analyzers/ToolCallAnalyzer.d.ts +45 -0
  175. package/dist/test/framework/analyzers/ToolCallAnalyzer.d.ts.map +1 -0
  176. package/dist/test/framework/analyzers/ToolCallAnalyzer.js +215 -0
  177. package/dist/test/framework/analyzers/ToolCallAnalyzer.js.map +1 -0
  178. package/dist/test/framework/cli.d.ts +8 -0
  179. package/dist/test/framework/cli.d.ts.map +1 -0
  180. package/dist/test/framework/cli.js +175 -0
  181. package/dist/test/framework/cli.js.map +1 -0
  182. package/dist/test/framework/core/TestCase.d.ts +100 -0
  183. package/dist/test/framework/core/TestCase.d.ts.map +1 -0
  184. package/dist/test/framework/core/TestCase.js +101 -0
  185. package/dist/test/framework/core/TestCase.js.map +1 -0
  186. package/dist/test/framework/core/TestEngine.d.ts +89 -0
  187. package/dist/test/framework/core/TestEngine.d.ts.map +1 -0
  188. package/dist/test/framework/core/TestEngine.js +552 -0
  189. package/dist/test/framework/core/TestEngine.js.map +1 -0
  190. package/dist/test/framework/core/TestResult.d.ts +143 -0
  191. package/dist/test/framework/core/TestResult.d.ts.map +1 -0
  192. package/dist/test/framework/core/TestResult.js +76 -0
  193. package/dist/test/framework/core/TestResult.js.map +1 -0
  194. package/dist/test/framework/examples/BasicRobustnessTest.d.ts +19 -0
  195. package/dist/test/framework/examples/BasicRobustnessTest.d.ts.map +1 -0
  196. package/dist/test/framework/examples/BasicRobustnessTest.js +116 -0
  197. package/dist/test/framework/examples/BasicRobustnessTest.js.map +1 -0
  198. package/dist/test/framework/index.d.ts +30 -0
  199. package/dist/test/framework/index.d.ts.map +1 -0
  200. package/dist/test/framework/index.js +38 -0
  201. package/dist/test/framework/index.js.map +1 -0
  202. package/dist/test/framework/loaders/JsonScenarioLoader.d.ts +99 -0
  203. package/dist/test/framework/loaders/JsonScenarioLoader.d.ts.map +1 -0
  204. package/dist/test/framework/loaders/JsonScenarioLoader.js +197 -0
  205. package/dist/test/framework/loaders/JsonScenarioLoader.js.map +1 -0
  206. package/dist/test/framework/reporters/ConsoleReporter.d.ts +25 -0
  207. package/dist/test/framework/reporters/ConsoleReporter.d.ts.map +1 -0
  208. package/dist/test/framework/reporters/ConsoleReporter.js +196 -0
  209. package/dist/test/framework/reporters/ConsoleReporter.js.map +1 -0
  210. package/dist/test/framework/scenarios/ScenarioBuilder.d.ts +62 -0
  211. package/dist/test/framework/scenarios/ScenarioBuilder.d.ts.map +1 -0
  212. package/dist/test/framework/scenarios/ScenarioBuilder.js +285 -0
  213. package/dist/test/framework/scenarios/ScenarioBuilder.js.map +1 -0
  214. package/dist/test/framework/validate-framework.d.ts +9 -0
  215. package/dist/test/framework/validate-framework.d.ts.map +1 -0
  216. package/dist/test/framework/validate-framework.js +187 -0
  217. package/dist/test/framework/validate-framework.js.map +1 -0
  218. package/package.json +84 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"PromptAnalyzer.js","sourceRoot":"","sources":["../../../../src/test/framework/analyzers/PromptAnalyzer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAYH,MAAM,OAAO,cAAc;IACzB,eAAe;IACP,MAAM,CAAU,eAAe,GAAoB;QACzD;YACE,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,6CAA6C;YACtD,MAAM,EAAE,GAAG;YACX,WAAW,EAAE,YAAY;SAC1B;QACD;YACE,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,sCAAsC;YAC/C,MAAM,EAAE,GAAG;YACX,WAAW,EAAE,YAAY;SAC1B;QACD;YACE,IAAI,EAAE,gBAAgB;YACtB,OAAO,EAAE,sCAAsC;YAC/C,MAAM,EAAE,GAAG;YACX,WAAW,EAAE,WAAW;SACzB;QACD;YACE,IAAI,EAAE,qBAAqB;YAC3B,OAAO,EAAE,mCAAmC;YAC5C,MAAM,EAAE,IAAI;YACZ,WAAW,EAAE,WAAW;SACzB;QACD;YACE,IAAI,EAAE,cAAc;YACpB,OAAO,EAAE,0CAA0C;YACnD,MAAM,EAAE,IAAI;YACZ,WAAW,EAAE,UAAU;SACxB;KACF,CAAC;IAEF;;OAEG;IACH,MAAM,CAAC,OAAO,CACZ,QAAkB,EAClB,aAA4B,EAC5B,SAAyB;QAEzB,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QAC3D,MAAM,WAAW,GAAG,QAAQ,CAAC,sBAAsB,CAAC;QAEpD,MAAM,MAAM,GAAyB;YACnC,oBAAoB,EAAE,KAAK;YAC3B,oBAAoB,EAAE,KAAK;YAC3B,oBAAoB,EAAE,KAAK;YAC3B,uBAAuB,EAAE,KAAK;YAC9B,wBAAwB,EAAE,CAAC;YAC3B,MAAM,EAAE,EAAE;SACX,CAAC;QAEF,kBAAkB;QAClB,MAAM,CAAC,oBAAoB,GAAG,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAEpF,gBAAgB;QAChB,IAAI,WAAW,CAAC,yBAAyB,EAAE,CAAC;YAC1C,MAAM,CAAC,oBAAoB,GAAG,IAAI,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC;YACtE,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;gBACjC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,gBAAgB;QAChB,IAAI,WAAW,CAAC,yBAAyB,EAAE,CAAC;YAC1C,MAAM,CAAC,oBAAoB,GAAG,IAAI,CAAC,yBAAyB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAClF,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;gBACjC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,YAAY;QACZ,IAAI,WAAW,CAAC,4BAA4B,EAAE,CAAC;YAC7C,MAAM,CAAC,uBAAuB,GAAG,IAAI,CAAC,kBAAkB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC5E,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE,CAAC;gBACpC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,aAAa;QACb,IAAI,WAAW,CAAC,iBAAiB,EAAE,CAAC;YAClC,KAAK,MAAM,UAAU,IAAI,WAAW,CAAC,iBAAiB,EAAE,CAAC;gBACvD,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBACxB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;QACH,CAAC;QAED,gBAAgB;QAChB,MAAM,CAAC,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEnF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,2BAA2B,CACxC,MAAc,EACd,WAA8B;QAE9B,wBAAwB;QACxB,MAAM,UAAU,GAAG;YACjB,wCAAwC;YACxC,gDAAgD;YAChD,sCAAsC;YACtC,2BAA2B;SAC5B,CAAC;QAEF,OAAO,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,uBAAuB,CAAC,SAAyB;QAC9D,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QAEzC,qBAAqB;QACrB,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7D,MAAM,aAAa,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;QAExE,OAAO,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/B,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CACtE,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,yBAAyB,CACtC,SAAyB,EACzB,QAAkB;QAElB,MAAM,aAAa,GAAG,QAAQ,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACpF,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAEnE,aAAa;QACb,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAC3C,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAClD,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QAEhC,OAAO,QAAQ,IAAI,GAAG,CAAC,CAAC,gBAAgB;IAC1C,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,kBAAkB,CAAC,MAAc,EAAE,SAAyB;QACzE,MAAM,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACtD,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAEvD,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC,CAAC,YAAY;QAEzC,eAAe;QACf,MAAM,kBAAkB,GAAG;YACzB,kCAAkC;YAClC,qCAAqC;YACrC,kCAAkC;YAClC,mCAAmC;SACpC,CAAC;QAEF,OAAO,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAClE,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,2BAA2B,CACxC,MAAc,EACd,MAA4B;QAE5B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,aAAa;QACb,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC3C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;YAC9B,IAAI,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,eAAe;QACf,MAAM,MAAM,GAAG;YACb,MAAM,CAAC,oBAAoB;YAC3B,MAAM,CAAC,oBAAoB;YAC3B,MAAM,CAAC,uBAAuB;YAC9B,MAAM,CAAC,oBAAoB;SAC5B,CAAC;QAEF,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC;QACvE,MAAM,YAAY,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAEvE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,UAAU,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,8BAA8B,CAAC,MAA4B;QAChE,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;YACjC,WAAW,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,oBAAoB,EAAE,CAAC;YACjC,WAAW,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,uBAAuB,EAAE,CAAC;YACpC,WAAW,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QAC9C,CAAC;QAED,IAAI,MAAM,CAAC,wBAAwB,GAAG,GAAG,EAAE,CAAC;YAC1C,WAAW,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QACjD,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC"}
@@ -0,0 +1,45 @@
1
+ /**
2
+ * 工具调用分析器
3
+ *
4
+ * 分析和验证 AI Agent 调用的工具类型、参数、顺序等,
5
+ * 评估工具使用的准确性和合理性
6
+ */
7
+ import { ToolCallAnalysisResult, ExecutionInfo } from '../core/TestResult';
8
+ import { TestCase } from '../core/TestCase';
9
+ export interface ToolCallPattern {
10
+ tool: string;
11
+ pattern: RegExp;
12
+ extractParams?: (match: RegExpMatchArray) => Record<string, any>;
13
+ }
14
+ export declare class ToolCallAnalyzer {
15
+ private static readonly TOOL_PATTERNS;
16
+ /**
17
+ * 分析工具调用情况
18
+ */
19
+ static analyze(testCase: TestCase, executionInfo: ExecutionInfo): ToolCallAnalysisResult;
20
+ /**
21
+ * 从输出中提取工具调用信息
22
+ */
23
+ private static extractToolCalls;
24
+ /**
25
+ * 计算工具使用准确性
26
+ */
27
+ private static calculateToolAccuracy;
28
+ /**
29
+ * 分析工具调用顺序的正确性
30
+ */
31
+ private static analyzeCallSequence;
32
+ /**
33
+ * 分析参数使用的正确性
34
+ */
35
+ private static analyzeParameterCorrectness;
36
+ /**
37
+ * 识别意外和缺失的工具
38
+ */
39
+ private static identifyToolGaps;
40
+ /**
41
+ * 生成工具使用改进建议
42
+ */
43
+ static generateImprovementSuggestions(result: ToolCallAnalysisResult): string[];
44
+ }
45
+ //# sourceMappingURL=ToolCallAnalyzer.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ToolCallAnalyzer.d.ts","sourceRoot":"","sources":["../../../../src/test/framework/analyzers/ToolCallAnalyzer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,sBAAsB,EAAgB,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACzF,OAAO,EAAE,QAAQ,EAAuB,MAAM,kBAAkB,CAAC;AAEjE,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAClE;AAED,qBAAa,gBAAgB;IAE3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CA0BnC;IAEF;;OAEG;IACH,MAAM,CAAC,OAAO,CACZ,QAAQ,EAAE,QAAQ,EAClB,aAAa,EAAE,aAAa,GAC3B,sBAAsB;IAgCzB;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAgD/B;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,qBAAqB;IA2BpC;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,mBAAmB;IAgClC;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,2BAA2B;IAkC1C;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,gBAAgB;IAyB/B;;OAEG;IACH,MAAM,CAAC,8BAA8B,CAAC,MAAM,EAAE,sBAAsB,GAAG,MAAM,EAAE;CAyBhF"}
@@ -0,0 +1,215 @@
1
+ /**
2
+ * 工具调用分析器
3
+ *
4
+ * 分析和验证 AI Agent 调用的工具类型、参数、顺序等,
5
+ * 评估工具使用的准确性和合理性
6
+ */
7
+ export class ToolCallAnalyzer {
8
+ // 工具调用模式匹配规则
9
+ static TOOL_PATTERNS = [
10
+ {
11
+ tool: 'glob',
12
+ pattern: /● glob - ([^-]+) - ([^\n]+)/g,
13
+ extractParams: (match) => ({ description: match[1], action: match[2] })
14
+ },
15
+ {
16
+ tool: 'read-file',
17
+ pattern: /● read-file - ([^-]+) - ([^\n]+)/g,
18
+ extractParams: (match) => ({ description: match[1], action: match[2] })
19
+ },
20
+ {
21
+ tool: 'write-file',
22
+ pattern: /● write-file - ([^-]+) - ([^\n]+)/g,
23
+ extractParams: (match) => ({ description: match[1], action: match[2] })
24
+ },
25
+ {
26
+ tool: 'grep',
27
+ pattern: /● grep - ([^-]+) - ([^\n]+)/g,
28
+ extractParams: (match) => ({ description: match[1], action: match[2] })
29
+ },
30
+ {
31
+ tool: 'shell',
32
+ pattern: /● shell - ([^-]+) - ([^\n]+)/g,
33
+ extractParams: (match) => ({ description: match[1], action: match[2] })
34
+ }
35
+ ];
36
+ /**
37
+ * 分析工具调用情况
38
+ */
39
+ static analyze(testCase, executionInfo) {
40
+ const output = executionInfo.stdout + executionInfo.stderr;
41
+ // 1. 提取工具调用信息
42
+ const toolCallDetails = this.extractToolCalls(output);
43
+ // 2. 分析工具使用准确性
44
+ const toolAccuracy = this.calculateToolAccuracy(testCase.expectedToolCalls, toolCallDetails);
45
+ // 3. 分析调用顺序正确性
46
+ const sequenceCorrectness = this.analyzeCallSequence(testCase.expectedToolCalls, toolCallDetails);
47
+ // 4. 分析参数正确性
48
+ const parameterCorrectness = this.analyzeParameterCorrectness(testCase.expectedToolCalls, toolCallDetails);
49
+ // 5. 识别意外和缺失的工具
50
+ const { unexpectedTools, missingTools } = this.identifyToolGaps(testCase.expectedToolCalls, toolCallDetails);
51
+ const uniqueTools = [...new Set(toolCallDetails.map(call => call.tool))];
52
+ return {
53
+ totalCalls: toolCallDetails.length,
54
+ uniqueTools,
55
+ toolAccuracy,
56
+ sequenceCorrectness,
57
+ parameterCorrectness,
58
+ unexpectedTools,
59
+ missingTools,
60
+ toolCallDetails
61
+ };
62
+ }
63
+ /**
64
+ * 从输出中提取工具调用信息
65
+ */
66
+ static extractToolCalls(output) {
67
+ const toolCalls = [];
68
+ const lines = output.split('\n');
69
+ let currentTimestamp = new Date();
70
+ for (let i = 0; i < lines.length; i++) {
71
+ const line = lines[i];
72
+ // 尝试匹配各种工具调用模式
73
+ for (const pattern of this.TOOL_PATTERNS) {
74
+ const matches = [...line.matchAll(pattern.pattern)];
75
+ for (const match of matches) {
76
+ const toolCall = {
77
+ tool: pattern.tool,
78
+ timestamp: new Date(currentTimestamp.getTime() + i * 100), // 模拟时间戳
79
+ success: !line.includes('Error') && !line.includes('Failed'),
80
+ parameters: pattern.extractParams ? pattern.extractParams(match) : undefined
81
+ };
82
+ // 尝试提取输出信息
83
+ if (i + 1 < lines.length) {
84
+ const nextLine = lines[i + 1];
85
+ if (!nextLine.startsWith('●') && nextLine.trim()) {
86
+ toolCall.output = nextLine.trim();
87
+ }
88
+ }
89
+ toolCalls.push(toolCall);
90
+ }
91
+ }
92
+ // 也检查其他可能的工具调用指示器
93
+ if (line.includes('Tool:') || line.includes('Using tool:') || line.includes('Executing tool:')) {
94
+ const toolMatch = line.match(/(?:Tool:|Using tool:|Executing tool:)\s*([a-zA-Z-]+)/);
95
+ if (toolMatch) {
96
+ toolCalls.push({
97
+ tool: toolMatch[1],
98
+ timestamp: new Date(currentTimestamp.getTime() + i * 100),
99
+ success: !line.includes('Error') && !line.includes('Failed')
100
+ });
101
+ }
102
+ }
103
+ }
104
+ return toolCalls;
105
+ }
106
+ /**
107
+ * 计算工具使用准确性
108
+ */
109
+ static calculateToolAccuracy(expectedCalls, actualCalls) {
110
+ if (expectedCalls.length === 0)
111
+ return 1;
112
+ let correctCalls = 0;
113
+ const actualToolNames = actualCalls.map(call => call.tool.toLowerCase());
114
+ for (const expected of expectedCalls) {
115
+ const expectedTool = expected.tool.toLowerCase();
116
+ const actualCount = actualToolNames.filter(tool => tool.includes(expectedTool) || expectedTool.includes(tool)).length;
117
+ // 检查调用次数是否符合期望
118
+ const minCalls = expected.minCalls || (expected.required ? 1 : 0);
119
+ const maxCalls = expected.maxCalls || Infinity;
120
+ if (actualCount >= minCalls && actualCount <= maxCalls) {
121
+ correctCalls++;
122
+ }
123
+ }
124
+ return correctCalls / expectedCalls.length;
125
+ }
126
+ /**
127
+ * 分析工具调用顺序的正确性
128
+ */
129
+ static analyzeCallSequence(expectedCalls, actualCalls) {
130
+ // 获取有顺序要求的期望调用
131
+ const orderedExpected = expectedCalls
132
+ .filter(exp => exp.order !== undefined)
133
+ .sort((a, b) => (a.order || 0) - (b.order || 0));
134
+ if (orderedExpected.length === 0)
135
+ return 1; // 没有顺序要求
136
+ let correctSequences = 0;
137
+ let lastFoundIndex = -1;
138
+ for (const expected of orderedExpected) {
139
+ const expectedTool = expected.tool.toLowerCase();
140
+ // 在实际调用中查找这个工具
141
+ const foundIndex = actualCalls.findIndex((call, index) => index > lastFoundIndex &&
142
+ (call.tool.toLowerCase().includes(expectedTool) || expectedTool.includes(call.tool.toLowerCase())));
143
+ if (foundIndex > lastFoundIndex) {
144
+ correctSequences++;
145
+ lastFoundIndex = foundIndex;
146
+ }
147
+ }
148
+ return correctSequences / orderedExpected.length;
149
+ }
150
+ /**
151
+ * 分析参数使用的正确性
152
+ */
153
+ static analyzeParameterCorrectness(expectedCalls, actualCalls) {
154
+ const expectedWithParams = expectedCalls.filter(exp => exp.parameters);
155
+ if (expectedWithParams.length === 0)
156
+ return 1; // 没有参数要求
157
+ let correctParams = 0;
158
+ for (const expected of expectedWithParams) {
159
+ const matchingCalls = actualCalls.filter(call => call.tool.toLowerCase().includes(expected.tool.toLowerCase()) ||
160
+ expected.tool.toLowerCase().includes(call.tool.toLowerCase()));
161
+ for (const call of matchingCalls) {
162
+ if (call.parameters && expected.parameters) {
163
+ const paramMatches = Object.entries(expected.parameters).every(([key, value]) => {
164
+ const actualValue = call.parameters[key];
165
+ return actualValue === value ||
166
+ (typeof actualValue === 'string' && actualValue.includes(String(value)));
167
+ });
168
+ if (paramMatches) {
169
+ correctParams++;
170
+ break; // 找到一个匹配的就够了
171
+ }
172
+ }
173
+ }
174
+ }
175
+ return correctParams / expectedWithParams.length;
176
+ }
177
+ /**
178
+ * 识别意外和缺失的工具
179
+ */
180
+ static identifyToolGaps(expectedCalls, actualCalls) {
181
+ const expectedTools = expectedCalls.map(exp => exp.tool.toLowerCase());
182
+ const actualTools = [...new Set(actualCalls.map(call => call.tool.toLowerCase()))];
183
+ // 找出缺失的必需工具
184
+ const missingTools = expectedCalls
185
+ .filter(exp => exp.required)
186
+ .filter(exp => !actualTools.some(actual => actual.includes(exp.tool.toLowerCase()) || exp.tool.toLowerCase().includes(actual)))
187
+ .map(exp => exp.tool);
188
+ // 找出意外的工具(不在期望列表中的)
189
+ const unexpectedTools = actualTools.filter(actual => !expectedTools.some(expected => actual.includes(expected) || expected.includes(actual)));
190
+ return { unexpectedTools, missingTools };
191
+ }
192
+ /**
193
+ * 生成工具使用改进建议
194
+ */
195
+ static generateImprovementSuggestions(result) {
196
+ const suggestions = [];
197
+ if (result.toolAccuracy < 0.7) {
198
+ suggestions.push('工具选择准确性较低,建议改进系统提示词中的工具选择指导');
199
+ }
200
+ if (result.sequenceCorrectness < 0.8) {
201
+ suggestions.push('工具调用顺序不够合理,建议在提示词中强调逻辑顺序');
202
+ }
203
+ if (result.parameterCorrectness < 0.8) {
204
+ suggestions.push('工具参数使用不够准确,建议提供更详细的参数使用示例');
205
+ }
206
+ if (result.missingTools.length > 0) {
207
+ suggestions.push(`缺少必需的工具调用: ${result.missingTools.join(', ')}`);
208
+ }
209
+ if (result.unexpectedTools.length > 0) {
210
+ suggestions.push(`使用了意外的工具: ${result.unexpectedTools.join(', ')}`);
211
+ }
212
+ return suggestions;
213
+ }
214
+ }
215
+ //# sourceMappingURL=ToolCallAnalyzer.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ToolCallAnalyzer.js","sourceRoot":"","sources":["../../../../src/test/framework/analyzers/ToolCallAnalyzer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAWH,MAAM,OAAO,gBAAgB;IAC3B,aAAa;IACL,MAAM,CAAU,aAAa,GAAsB;QACzD;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,8BAA8B;YACvC,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE;QACD;YACE,IAAI,EAAE,WAAW;YACjB,OAAO,EAAE,mCAAmC;YAC5C,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE;QACD;YACE,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,oCAAoC;YAC7C,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE;QACD;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,8BAA8B;YACvC,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE;QACD;YACE,IAAI,EAAE,OAAO;YACb,OAAO,EAAE,+BAA+B;YACxC,aAAa,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;SACxE;KACF,CAAC;IAEF;;OAEG;IACH,MAAM,CAAC,OAAO,CACZ,QAAkB,EAClB,aAA4B;QAE5B,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC;QAE3D,cAAc;QACd,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAEtD,eAAe;QACf,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAE7F,eAAe;QACf,MAAM,mBAAmB,GAAG,IAAI,CAAC,mBAAmB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAElG,aAAa;QACb,MAAM,oBAAoB,GAAG,IAAI,CAAC,2BAA2B,CAAC,QAAQ,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAE3G,gBAAgB;QAChB,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAE7G,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEzE,OAAO;YACL,UAAU,EAAE,eAAe,CAAC,MAAM;YAClC,WAAW;YACX,YAAY;YACZ,mBAAmB;YACnB,oBAAoB;YACpB,eAAe;YACf,YAAY;YACZ,eAAe;SAChB,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,gBAAgB,CAAC,MAAc;QAC5C,MAAM,SAAS,GAAmB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,gBAAgB,GAAG,IAAI,IAAI,EAAE,CAAC;QAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,eAAe;YACf,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;gBAEpD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC5B,MAAM,QAAQ,GAAiB;wBAC7B,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,SAAS,EAAE,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,EAAE,QAAQ;wBACnE,OAAO,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBAC5D,UAAU,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS;qBAC7E,CAAC;oBAEF,WAAW;oBACX,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;wBACzB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;wBAC9B,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;4BACjD,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;wBACpC,CAAC;oBACH,CAAC;oBAED,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC3B,CAAC;YACH,CAAC;YAED,kBAAkB;YAClB,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC/F,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;gBACrF,IAAI,SAAS,EAAE,CAAC;oBACd,SAAS,CAAC,IAAI,CAAC;wBACb,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;wBAClB,SAAS,EAAE,IAAI,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC;wBACzD,OAAO,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;qBAC7D,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,qBAAqB,CAClC,aAAoC,EACpC,WAA2B;QAE3B,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAEzE,KAAK,MAAM,QAAQ,IAAI,aAAa,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjD,MAAM,WAAW,GAAG,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAChD,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAC3D,CAAC,MAAM,CAAC;YAET,eAAe;YACf,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClE,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,IAAI,QAAQ,CAAC;YAE/C,IAAI,WAAW,IAAI,QAAQ,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;gBACvD,YAAY,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QAED,OAAO,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC;IAC7C,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,mBAAmB,CAChC,aAAoC,EACpC,WAA2B;QAE3B,eAAe;QACf,MAAM,eAAe,GAAG,aAAa;aAClC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC;aACtC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC;QAEnD,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,SAAS;QAErD,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,cAAc,GAAG,CAAC,CAAC,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,MAAM,YAAY,GAAG,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YAEjD,eAAe;YACf,MAAM,UAAU,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACvD,KAAK,GAAG,cAAc;gBACtB,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CACnG,CAAC;YAEF,IAAI,UAAU,GAAG,cAAc,EAAE,CAAC;gBAChC,gBAAgB,EAAE,CAAC;gBACnB,cAAc,GAAG,UAAU,CAAC;YAC9B,CAAC;QACH,CAAC;QAED,OAAO,gBAAgB,GAAG,eAAe,CAAC,MAAM,CAAC;IACnD,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,2BAA2B,CACxC,aAAoC,EACpC,WAA2B;QAE3B,MAAM,kBAAkB,GAAG,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACvE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC,CAAC,SAAS;QAExD,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,QAAQ,IAAI,kBAAkB,EAAE,CAAC;YAC1C,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAC9C,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAC7D,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAC9D,CAAC;YAEF,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;gBACjC,IAAI,IAAI,CAAC,UAAU,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;oBAC3C,MAAM,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;wBAC9E,MAAM,WAAW,GAAG,IAAI,CAAC,UAAW,CAAC,GAAG,CAAC,CAAC;wBAC1C,OAAO,WAAW,KAAK,KAAK;4BACrB,CAAC,OAAO,WAAW,KAAK,QAAQ,IAAI,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBAClF,CAAC,CAAC,CAAC;oBAEH,IAAI,YAAY,EAAE,CAAC;wBACjB,aAAa,EAAE,CAAC;wBAChB,MAAM,CAAC,aAAa;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,aAAa,GAAG,kBAAkB,CAAC,MAAM,CAAC;IACnD,CAAC;IAED;;OAEG;IACK,MAAM,CAAC,gBAAgB,CAC7B,aAAoC,EACpC,WAA2B;QAE3B,MAAM,aAAa,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACvE,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QAEnF,YAAY;QACZ,MAAM,YAAY,GAAG,aAAa;aAC/B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC;aAC3B,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CACxC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CACnF,CAAC;aACD,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAExB,oBAAoB;QACpB,MAAM,eAAe,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAClD,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAC7B,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CACvD,CACF,CAAC;QAEF,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,8BAA8B,CAAC,MAA8B;QAClE,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,IAAI,MAAM,CAAC,YAAY,GAAG,GAAG,EAAE,CAAC;YAC9B,WAAW,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,MAAM,CAAC,mBAAmB,GAAG,GAAG,EAAE,CAAC;YACrC,WAAW,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC/C,CAAC;QAED,IAAI,MAAM,CAAC,oBAAoB,GAAG,GAAG,EAAE,CAAC;YACtC,WAAW,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,WAAW,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnE,CAAC;QAED,IAAI,MAAM,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,WAAW,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC"}
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * AI Agent 健壮性测试框架 CLI
4
+ *
5
+ * 命令行工具,用于运行各种测试场景
6
+ */
7
+ export {};
8
+ //# sourceMappingURL=cli.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../../src/test/framework/cli.ts"],"names":[],"mappings":";AAEA;;;;GAIG"}
@@ -0,0 +1,175 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * AI Agent 健壮性测试框架 CLI
4
+ *
5
+ * 命令行工具,用于运行各种测试场景
6
+ */
7
+ import { program } from 'commander';
8
+ import { createTestEngine, ScenarioBuilder, ConsoleReporter, FRAMEWORK_VERSION } from './index';
9
+ program
10
+ .name('agent-test')
11
+ .description('AI Agent 健壮性测试框架')
12
+ .version(FRAMEWORK_VERSION);
13
+ // 运行预定义测试套件
14
+ program
15
+ .command('suite <name>')
16
+ .description('运行预定义的测试套件 (basic|business|error-recovery|performance)')
17
+ .option('-a, --agent <path>', 'Agent 可执行文件路径', './dist/index.js')
18
+ .option('-o, --output <dir>', '输出目录', './test-results')
19
+ .option('-v, --verbose', '详细输出', false)
20
+ .option('-p, --parallel', '并行执行', false)
21
+ .option('-k, --keep-projects', '保留测试项目', false)
22
+ .action(async (suiteName, options) => {
23
+ console.log(`🚀 运行测试套件: ${suiteName}`);
24
+ console.log(`⚠️ 测试套件功能正在开发中,请使用 scenario 命令运行单个测试场景`);
25
+ process.exit(0);
26
+ });
27
+ // 运行单个测试场景
28
+ program
29
+ .command('scenario <template>')
30
+ .description('运行指定的测试场景模板')
31
+ .option('-a, --agent <path>', 'Agent 可执行文件路径', './dist/index.js')
32
+ .option('-p, --params <json>', '模板参数 (JSON 格式)', '{}')
33
+ .option('-v, --verbose', '详细输出', false)
34
+ .option('-k, --keep-projects', '保留测试项目', false)
35
+ .action(async (templateId, options) => {
36
+ console.log(`🧪 运行测试场景: ${templateId}`);
37
+ try {
38
+ const params = JSON.parse(options.params);
39
+ const testCase = ScenarioBuilder.generateFromTemplate(templateId, params);
40
+ const testEngine = createTestEngine({
41
+ agentPath: options.agent,
42
+ verbose: options.verbose,
43
+ keepTestProjects: options.keepProjects
44
+ });
45
+ const result = await testEngine.runTest(testCase);
46
+ console.log(ConsoleReporter.generateTestReport(result));
47
+ process.exit(result.overallScore >= 0.8 ? 0 : 1);
48
+ }
49
+ catch (error) {
50
+ console.error(`❌ 测试场景执行失败: ${error}`);
51
+ process.exit(1);
52
+ }
53
+ });
54
+ // 列出可用的测试模板
55
+ program
56
+ .command('list-templates')
57
+ .description('列出所有可用的测试场景模板')
58
+ .option('-c, --category <category>', '按类别筛选')
59
+ .option('-d, --difficulty <level>', '按难度筛选 (easy|medium|hard|expert)')
60
+ .action((options) => {
61
+ console.log('📋 可用的测试场景模板:\n');
62
+ const filter = {};
63
+ if (options.category) {
64
+ filter.category = options.category;
65
+ }
66
+ if (options.difficulty) {
67
+ filter.difficulty = [options.difficulty];
68
+ }
69
+ const templates = ScenarioBuilder.filterTemplates(filter);
70
+ templates.forEach(template => {
71
+ console.log(`🔧 ${template.id}`);
72
+ console.log(` 名称: ${template.name}`);
73
+ console.log(` 描述: ${template.description}`);
74
+ console.log(` 类别: ${template.category}`);
75
+ console.log(` 难度: ${template.difficulty}`);
76
+ console.log(` 预估时间: ${template.estimatedDuration}ms`);
77
+ if (Object.keys(template.parameters).length > 0) {
78
+ console.log(` 参数:`);
79
+ Object.entries(template.parameters).forEach(([key, param]) => {
80
+ const required = param.required ? ' (必需)' : '';
81
+ const defaultValue = param.default !== undefined ? ` [默认: ${param.default}]` : '';
82
+ console.log(` • ${key}: ${param.description}${required}${defaultValue}`);
83
+ });
84
+ }
85
+ console.log('');
86
+ });
87
+ console.log(`总计: ${templates.length} 个模板`);
88
+ });
89
+ // 运行自定义测试
90
+ program
91
+ .command('custom')
92
+ .description('运行自定义测试')
93
+ .requiredOption('-t, --task <task>', '测试任务描述')
94
+ .option('-n, --name <name>', '测试名称', '自定义测试')
95
+ .option('-a, --agent <path>', 'Agent 可执行文件路径', './dist/index.js')
96
+ .option('--project-type <type>', '项目类型 (gradle-spring-boot|maven-spring-boot|npm-node|empty)', 'gradle-spring-boot')
97
+ .option('--timeout <ms>', '超时时间 (毫秒)', '300000')
98
+ .option('--expected-tools <tools>', '期望的工具列表 (逗号分隔)', '')
99
+ .option('-v, --verbose', '详细输出', false)
100
+ .option('-k, --keep-projects', '保留测试项目', false)
101
+ .action(async (options) => {
102
+ console.log(`🎯 运行自定义测试: ${options.name}`);
103
+ try {
104
+ const { TestCaseBuilder, TestCategory, ProjectType } = await import('./index');
105
+ const builder = TestCaseBuilder.create(`custom-${Date.now()}`)
106
+ .withName(options.name)
107
+ .withDescription('自定义测试用例')
108
+ .withCategory(TestCategory.BASIC_ROBUSTNESS)
109
+ .withTask(options.task)
110
+ .withProjectType(options.projectType)
111
+ .withTimeout(parseInt(options.timeout));
112
+ // 添加期望的工具
113
+ if (options.expectedTools) {
114
+ const tools = options.expectedTools.split(',').map((t) => t.trim());
115
+ tools.forEach((tool) => {
116
+ builder.expectTool(tool, { required: true });
117
+ });
118
+ }
119
+ const testCase = builder.build();
120
+ const testEngine = createTestEngine({
121
+ agentPath: options.agent,
122
+ verbose: options.verbose,
123
+ keepTestProjects: options.keepProjects
124
+ });
125
+ const result = await testEngine.runTest(testCase);
126
+ console.log(ConsoleReporter.generateTestReport(result));
127
+ process.exit(result.overallScore >= 0.8 ? 0 : 1);
128
+ }
129
+ catch (error) {
130
+ console.error(`❌ 自定义测试执行失败: ${error}`);
131
+ process.exit(1);
132
+ }
133
+ });
134
+ // 显示框架信息
135
+ program
136
+ .command('info')
137
+ .description('显示框架信息')
138
+ .action(() => {
139
+ console.log(`
140
+ 🤖 AI Agent 健壮性测试框架 v${FRAMEWORK_VERSION}
141
+
142
+ 📋 功能特性:
143
+ • 提示词效果验证 - 测试系统提示词是否能正确引导 Agent 行为
144
+ • 工具调用分析 - 跟踪和验证 Agent 调用的工具类型、参数、顺序
145
+ • 结果变更分析 - 检测和评估 Agent 产生的代码变更和质量
146
+ • 场景扩展能力 - 支持不同复杂度和类型的测试场景
147
+ • 详细报告生成 - 提供全面的测试分析报告和改进建议
148
+
149
+ 🎯 测试类别:
150
+ • basic-robustness - 基础健壮性测试
151
+ • business-scenario - 业务场景测试
152
+ • error-recovery - 错误恢复测试
153
+ • performance - 性能测试
154
+ • boundary-conditions - 边界条件测试
155
+
156
+ 🔧 支持的项目类型:
157
+ • gradle-spring-boot - Gradle Spring Boot 项目
158
+ • maven-spring-boot - Maven Spring Boot 项目
159
+ • npm-node - NPM Node.js 项目
160
+ • empty - 空项目
161
+
162
+ 📖 使用示例:
163
+ agent-test suite basic # 运行基础测试套件
164
+ agent-test scenario basic-file-operations # 运行文件操作场景
165
+ agent-test custom -t "Create a REST API" # 运行自定义测试
166
+ agent-test list-templates # 列出所有模板
167
+ `);
168
+ });
169
+ // 解析命令行参数
170
+ program.parse();
171
+ // 如果没有提供命令,显示帮助
172
+ if (!process.argv.slice(2).length) {
173
+ program.outputHelp();
174
+ }
175
+ //# sourceMappingURL=cli.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.js","sourceRoot":"","sources":["../../../src/test/framework/cli.ts"],"names":[],"mappings":";AAEA;;;;GAIG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EACL,gBAAgB,EAChB,eAAe,EAEf,eAAe,EACf,iBAAiB,EAClB,MAAM,SAAS,CAAC;AAEjB,OAAO;KACJ,IAAI,CAAC,YAAY,CAAC;KAClB,WAAW,CAAC,kBAAkB,CAAC;KAC/B,OAAO,CAAC,iBAAiB,CAAC,CAAC;AAE9B,YAAY;AACZ,OAAO;KACJ,OAAO,CAAC,cAAc,CAAC;KACvB,WAAW,CAAC,wDAAwD,CAAC;KACrE,MAAM,CAAC,oBAAoB,EAAE,eAAe,EAAE,iBAAiB,CAAC;KAChE,MAAM,CAAC,oBAAoB,EAAE,MAAM,EAAE,gBAAgB,CAAC;KACtD,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,KAAK,CAAC;KACtC,MAAM,CAAC,gBAAgB,EAAE,MAAM,EAAE,KAAK,CAAC;KACvC,MAAM,CAAC,qBAAqB,EAAE,QAAQ,EAAE,KAAK,CAAC;KAC9C,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE;IACnC,OAAO,CAAC,GAAG,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC;IACvC,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;IACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEL,WAAW;AACX,OAAO;KACJ,OAAO,CAAC,qBAAqB,CAAC;KAC9B,WAAW,CAAC,aAAa,CAAC;KAC1B,MAAM,CAAC,oBAAoB,EAAE,eAAe,EAAE,iBAAiB,CAAC;KAChE,MAAM,CAAC,qBAAqB,EAAE,gBAAgB,EAAE,IAAI,CAAC;KACrD,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,KAAK,CAAC;KACtC,MAAM,CAAC,qBAAqB,EAAE,QAAQ,EAAE,KAAK,CAAC;KAC9C,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE;IACpC,OAAO,CAAC,GAAG,CAAC,cAAc,UAAU,EAAE,CAAC,CAAC;IAExC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC1C,MAAM,QAAQ,GAAG,eAAe,CAAC,oBAAoB,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAE1E,MAAM,UAAU,GAAG,gBAAgB,CAAC;YAClC,SAAS,EAAE,OAAO,CAAC,KAAK;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,gBAAgB,EAAE,OAAO,CAAC,YAAY;SACvC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;QAExD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,eAAe,KAAK,EAAE,CAAC,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,YAAY;AACZ,OAAO;KACJ,OAAO,CAAC,gBAAgB,CAAC;KACzB,WAAW,CAAC,eAAe,CAAC;KAC5B,MAAM,CAAC,2BAA2B,EAAE,OAAO,CAAC;KAC5C,MAAM,CAAC,0BAA0B,EAAE,iCAAiC,CAAC;KACrE,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE;IAClB,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAE/B,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAwB,CAAC;IACrD,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,MAAM,CAAC,UAAU,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAC3C,CAAC;IAED,MAAM,SAAS,GAAG,eAAe,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;IAE1D,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE;QAC3B,OAAO,CAAC,GAAG,CAAC,MAAM,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,UAAU,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,GAAG,CAAC,UAAU,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,UAAU,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,UAAU,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,CAAC,YAAY,QAAQ,CAAC,iBAAiB,IAAI,CAAC,CAAC;QAExD,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACtB,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;gBAC3D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClF,OAAO,CAAC,GAAG,CAAC,UAAU,GAAG,KAAK,KAAK,CAAC,WAAW,GAAG,QAAQ,GAAG,YAAY,EAAE,CAAC,CAAC;YAC/E,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,OAAO,SAAS,CAAC,MAAM,MAAM,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEL,UAAU;AACV,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,SAAS,CAAC;KACtB,cAAc,CAAC,mBAAmB,EAAE,QAAQ,CAAC;KAC7C,MAAM,CAAC,mBAAmB,EAAE,MAAM,EAAE,OAAO,CAAC;KAC5C,MAAM,CAAC,oBAAoB,EAAE,eAAe,EAAE,iBAAiB,CAAC;KAChE,MAAM,CAAC,uBAAuB,EAAE,4DAA4D,EAAE,oBAAoB,CAAC;KACnH,MAAM,CAAC,gBAAgB,EAAE,WAAW,EAAE,QAAQ,CAAC;KAC/C,MAAM,CAAC,0BAA0B,EAAE,gBAAgB,EAAE,EAAE,CAAC;KACxD,MAAM,CAAC,eAAe,EAAE,MAAM,EAAE,KAAK,CAAC;KACtC,MAAM,CAAC,qBAAqB,EAAE,QAAQ,EAAE,KAAK,CAAC;KAC9C,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE;IACxB,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IAE3C,IAAI,CAAC;QACH,MAAM,EAAE,eAAe,EAAE,YAAY,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;QAE/E,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;aAC3D,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;aACtB,eAAe,CAAC,SAAS,CAAC;aAC1B,YAAY,CAAC,YAAY,CAAC,gBAAgB,CAAC;aAC3C,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC;aACtB,eAAe,CAAC,OAAO,CAAC,WAAkB,CAAC;aAC3C,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAE1C,UAAU;QACV,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5E,KAAK,CAAC,OAAO,CAAC,CAAC,IAAY,EAAE,EAAE;gBAC7B,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/C,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;QAEjC,MAAM,UAAU,GAAG,gBAAgB,CAAC;YAClC,SAAS,EAAE,OAAO,CAAC,KAAK;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,gBAAgB,EAAE,OAAO,CAAC,YAAY;SACvC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;QAExD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,gBAAgB,KAAK,EAAE,CAAC,CAAC;QACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,SAAS;AACT,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,QAAQ,CAAC;KACrB,MAAM,CAAC,GAAG,EAAE;IACX,OAAO,CAAC,GAAG,CAAC;uBACO,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;KA2BnC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEL,UAAU;AACV,OAAO,CAAC,KAAK,EAAE,CAAC;AAEhB,gBAAgB;AAChB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;IAClC,OAAO,CAAC,UAAU,EAAE,CAAC;AACvB,CAAC"}
@@ -0,0 +1,100 @@
1
+ /**
2
+ * AI Agent 测试用例定义
3
+ *
4
+ * 定义了测试用例的完整结构,包括输入、期望行为、验证规则等
5
+ */
6
+ export declare enum TestCategory {
7
+ BASIC_ROBUSTNESS = "basic-robustness",
8
+ BUSINESS_SCENARIO = "business-scenario",
9
+ ERROR_RECOVERY = "error-recovery",
10
+ PERFORMANCE = "performance",
11
+ BOUNDARY_CONDITIONS = "boundary-conditions"
12
+ }
13
+ export declare enum ProjectType {
14
+ GRADLE_SPRING_BOOT = "gradle-spring-boot",
15
+ MAVEN_SPRING_BOOT = "maven-spring-boot",
16
+ NPM_NODE = "npm-node",
17
+ EMPTY = "empty"
18
+ }
19
+ export interface FileDefinition {
20
+ path: string;
21
+ content: string;
22
+ encoding?: string;
23
+ }
24
+ export interface PromptExpectation {
25
+ shouldFollowSystemPrompt: boolean;
26
+ shouldExploreProjectFirst?: boolean;
27
+ shouldUseAppropriateTools?: boolean;
28
+ shouldHandleErrorsGracefully?: boolean;
29
+ customValidations?: ((output: string) => boolean)[];
30
+ }
31
+ export interface ToolCallExpectation {
32
+ tool: string;
33
+ required: boolean;
34
+ minCalls?: number;
35
+ maxCalls?: number;
36
+ pattern?: RegExp;
37
+ parameters?: Record<string, any>;
38
+ order?: number;
39
+ }
40
+ export interface ChangeExpectation {
41
+ type: 'file-created' | 'file-modified' | 'file-deleted' | 'dependency-added';
42
+ path?: string;
43
+ pattern?: RegExp;
44
+ content?: string | RegExp;
45
+ required: boolean;
46
+ }
47
+ export interface TestCase {
48
+ id: string;
49
+ name: string;
50
+ description: string;
51
+ category: TestCategory;
52
+ tags?: string[];
53
+ task: string;
54
+ projectType: ProjectType;
55
+ initialFiles?: FileDefinition[];
56
+ expectedPromptBehavior: PromptExpectation;
57
+ expectedToolCalls: ToolCallExpectation[];
58
+ expectedChanges: ChangeExpectation[];
59
+ qualityThresholds: {
60
+ minToolAccuracy: number;
61
+ maxExecutionTime: number;
62
+ minTaskCompletion: number;
63
+ maxCodeIssues: number;
64
+ };
65
+ config: {
66
+ timeout: number;
67
+ maxIterations: number;
68
+ retryCount: number;
69
+ keepTestProject: boolean;
70
+ quiet: boolean;
71
+ };
72
+ customValidators?: {
73
+ name: string;
74
+ validator: (result: any) => Promise<boolean>;
75
+ description: string;
76
+ }[];
77
+ }
78
+ export interface TestSuite {
79
+ id: string;
80
+ name: string;
81
+ description: string;
82
+ testCases: TestCase[];
83
+ setup?: () => Promise<void>;
84
+ teardown?: () => Promise<void>;
85
+ }
86
+ export declare class TestCaseBuilder {
87
+ private testCase;
88
+ static create(id: string): TestCaseBuilder;
89
+ withId(id: string): TestCaseBuilder;
90
+ withName(name: string): TestCaseBuilder;
91
+ withDescription(description: string): TestCaseBuilder;
92
+ withCategory(category: TestCategory): TestCaseBuilder;
93
+ withTask(task: string): TestCaseBuilder;
94
+ withProjectType(projectType: ProjectType): TestCaseBuilder;
95
+ expectTool(tool: string, options?: Partial<ToolCallExpectation>): TestCaseBuilder;
96
+ expectChange(type: ChangeExpectation['type'], options?: Partial<ChangeExpectation>): TestCaseBuilder;
97
+ withTimeout(timeout: number): TestCaseBuilder;
98
+ build(): TestCase;
99
+ }
100
+ //# sourceMappingURL=TestCase.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"TestCase.d.ts","sourceRoot":"","sources":["../../../../src/test/framework/core/TestCase.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,oBAAY,YAAY;IACtB,gBAAgB,qBAAqB;IACrC,iBAAiB,sBAAsB;IACvC,cAAc,mBAAmB;IACjC,WAAW,gBAAgB;IAC3B,mBAAmB,wBAAwB;CAC5C;AAED,oBAAY,WAAW;IACrB,kBAAkB,uBAAuB;IACzC,iBAAiB,sBAAsB;IACvC,QAAQ,aAAa;IACrB,KAAK,UAAU;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAiB;IAChC,wBAAwB,EAAE,OAAO,CAAC;IAClC,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,4BAA4B,CAAC,EAAE,OAAO,CAAC;IACvC,iBAAiB,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,EAAE,CAAC;CACrD;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,cAAc,GAAG,eAAe,GAAG,cAAc,GAAG,kBAAkB,CAAC;IAC7E,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,QAAQ;IAEvB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,YAAY,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAGhB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,WAAW,CAAC;IACzB,YAAY,CAAC,EAAE,cAAc,EAAE,CAAC;IAGhC,sBAAsB,EAAE,iBAAiB,CAAC;IAC1C,iBAAiB,EAAE,mBAAmB,EAAE,CAAC;IACzC,eAAe,EAAE,iBAAiB,EAAE,CAAC;IAGrC,iBAAiB,EAAE;QACjB,eAAe,EAAE,MAAM,CAAC;QACxB,gBAAgB,EAAE,MAAM,CAAC;QACzB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,aAAa,EAAE,MAAM,CAAC;KACvB,CAAC;IAGF,MAAM,EAAE;QACN,OAAO,EAAE,MAAM,CAAC;QAChB,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,eAAe,EAAE,OAAO,CAAC;QACzB,KAAK,EAAE,OAAO,CAAC;KAChB,CAAC;IAGF,gBAAgB,CAAC,EAAE;QACjB,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,EAAE,CAAC,MAAM,EAAE,GAAG,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;QAC7C,WAAW,EAAE,MAAM,CAAC;KACrB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAChC;AAGD,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAuBd;IAEF,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe;IAI1C,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,eAAe;IAKnC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe;IAKvC,eAAe,CAAC,WAAW,EAAE,MAAM,GAAG,eAAe;IAKrD,YAAY,CAAC,QAAQ,EAAE,YAAY,GAAG,eAAe;IAKrD,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe;IAKvC,eAAe,CAAC,WAAW,EAAE,WAAW,GAAG,eAAe;IAK1D,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,OAAO,CAAC,mBAAmB,CAAM,GAAG,eAAe;IASrF,YAAY,CAAC,IAAI,EAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,OAAO,GAAE,OAAO,CAAC,iBAAiB,CAAM,GAAG,eAAe;IASxG,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe;IAK7C,KAAK,IAAI,QAAQ;CAOlB"}