@use-lattice/litmus 0.121.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +19 -0
  2. package/dist/src/accounts-Bt1oJb1Z.cjs +219 -0
  3. package/dist/src/accounts-DjOU8Rm3.js +178 -0
  4. package/dist/src/agentic-utils-D03IiXQc.js +153 -0
  5. package/dist/src/agentic-utils-Dh7xaMQM.cjs +180 -0
  6. package/dist/src/agents-C6BIMlZa.js +231 -0
  7. package/dist/src/agents-DvIpNX1L.cjs +666 -0
  8. package/dist/src/agents-ZP0RP9vV.cjs +231 -0
  9. package/dist/src/agents-maJXdjbR.js +665 -0
  10. package/dist/src/aimlapi-BTbQjG2E.cjs +30 -0
  11. package/dist/src/aimlapi-CwMxqfXP.js +30 -0
  12. package/dist/src/audio-BBUdvsde.cjs +97 -0
  13. package/dist/src/audio-D5DPZ7I-.js +97 -0
  14. package/dist/src/base-BEysXrkq.cjs +222 -0
  15. package/dist/src/base-C451JQfq.js +193 -0
  16. package/dist/src/blobs-BY8MDmpo.js +230 -0
  17. package/dist/src/blobs-BgcNn97m.cjs +256 -0
  18. package/dist/src/cache-BBE_lsTA.cjs +4 -0
  19. package/dist/src/cache-BkrqU5Ba.js +237 -0
  20. package/dist/src/cache-DsCxFlsZ.cjs +297 -0
  21. package/dist/src/chat-CPJWDP6a.cjs +289 -0
  22. package/dist/src/chat-CXX3xzkk.cjs +811 -0
  23. package/dist/src/chat-CcDgZFJ4.js +787 -0
  24. package/dist/src/chat-Dz5ZeGO2.js +289 -0
  25. package/dist/src/chatkit-Dw0mKkML.cjs +1158 -0
  26. package/dist/src/chatkit-swAIVuea.js +1157 -0
  27. package/dist/src/chunk-DEq-mXcV.js +15 -0
  28. package/dist/src/claude-agent-sdk-BXZJtOg6.js +379 -0
  29. package/dist/src/claude-agent-sdk-CkfyjDoG.cjs +383 -0
  30. package/dist/src/cloudflare-ai-BzpJcqUH.js +161 -0
  31. package/dist/src/cloudflare-ai-Cmy_R1y2.cjs +161 -0
  32. package/dist/src/cloudflare-gateway-B9tVQKok.cjs +272 -0
  33. package/dist/src/cloudflare-gateway-DrD3ew3H.js +272 -0
  34. package/dist/src/codex-sdk-Dezj9Nwm.js +1056 -0
  35. package/dist/src/codex-sdk-Dl9D4k5B.cjs +1060 -0
  36. package/dist/src/cometapi-C-9YvCHC.js +54 -0
  37. package/dist/src/cometapi-DHgDKoO2.cjs +54 -0
  38. package/dist/src/completion-B8Ctyxpr.js +120 -0
  39. package/dist/src/completion-Cxrt08sj.cjs +131 -0
  40. package/dist/src/createHash-BwgE13yv.cjs +27 -0
  41. package/dist/src/createHash-DmPQkvBh.js +15 -0
  42. package/dist/src/docker-BiqcTwLv.js +80 -0
  43. package/dist/src/docker-C7tEJnP-.cjs +80 -0
  44. package/dist/src/esm-C62Zofr1.cjs +409 -0
  45. package/dist/src/esm-DMVc93eh.js +379 -0
  46. package/dist/src/evalResult-C3NJPQOo.cjs +301 -0
  47. package/dist/src/evalResult-C7JJAPBb.js +295 -0
  48. package/dist/src/evalResult-DoVTZZWI.cjs +2 -0
  49. package/dist/src/extractor-DnMD3fwt.cjs +391 -0
  50. package/dist/src/extractor-DtlL28vL.js +374 -0
  51. package/dist/src/fetch-BTxakTSg.cjs +1133 -0
  52. package/dist/src/fetch-DQckpUFz.js +928 -0
  53. package/dist/src/fileExtensions-DnqA1y9x.js +85 -0
  54. package/dist/src/fileExtensions-bYh77CN8.cjs +114 -0
  55. package/dist/src/genaiTracer-CyZrmaK0.cjs +268 -0
  56. package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
  57. package/dist/src/graders-BNscxFrU.js +13644 -0
  58. package/dist/src/graders-D2oE9Msq.js +2 -0
  59. package/dist/src/graders-c0Ez_w-9.cjs +2 -0
  60. package/dist/src/graders-d0F2M3e9.cjs +14056 -0
  61. package/dist/src/image-0ZhE0VlR.cjs +280 -0
  62. package/dist/src/image-CWE1pdNv.js +257 -0
  63. package/dist/src/image-D9ZK6hwL.js +163 -0
  64. package/dist/src/image-DKZgZITg.cjs +163 -0
  65. package/dist/src/index.cjs +11366 -0
  66. package/dist/src/index.d.cts +19640 -0
  67. package/dist/src/index.d.ts +19641 -0
  68. package/dist/src/index.js +11306 -0
  69. package/dist/src/invariant-Ddh24eXh.js +25 -0
  70. package/dist/src/invariant-kfQ8Bu82.cjs +30 -0
  71. package/dist/src/knowledgeBase-BgPyGFUd.cjs +122 -0
  72. package/dist/src/knowledgeBase-DyHilYaP.js +122 -0
  73. package/dist/src/litellm-CyMeneHS.js +135 -0
  74. package/dist/src/litellm-DWDF73yF.cjs +135 -0
  75. package/dist/src/logger-C40ZGil9.js +717 -0
  76. package/dist/src/logger-DyfK9PBt.cjs +917 -0
  77. package/dist/src/luma-ray-BAU9X_ep.cjs +315 -0
  78. package/dist/src/luma-ray-nwVseBbv.js +313 -0
  79. package/dist/src/messages-B5ADWTTv.js +245 -0
  80. package/dist/src/messages-BCnZfqrS.cjs +257 -0
  81. package/dist/src/meteor-DLZZ3osF.cjs +134 -0
  82. package/dist/src/meteor-DUiCJRC-.js +134 -0
  83. package/dist/src/modelslab-00cveB8L.cjs +163 -0
  84. package/dist/src/modelslab-D9sCU_L7.js +163 -0
  85. package/dist/src/nova-reel-CTapvqYH.js +276 -0
  86. package/dist/src/nova-reel-DlWuuroF.cjs +278 -0
  87. package/dist/src/nova-sonic-5UPWfeMv.cjs +363 -0
  88. package/dist/src/nova-sonic-BhSwQNym.js +363 -0
  89. package/dist/src/openai-BWrJK9d8.cjs +52 -0
  90. package/dist/src/openai-DumO8WQn.js +47 -0
  91. package/dist/src/openclaw-B8brrjC_.cjs +577 -0
  92. package/dist/src/openclaw-Bkayww9q.js +571 -0
  93. package/dist/src/opencode-sdk-7xjoDNiM.cjs +562 -0
  94. package/dist/src/opencode-sdk-SGwAPxht.js +558 -0
  95. package/dist/src/otlpReceiver-CoAHfAN9.cjs +15 -0
  96. package/dist/src/otlpReceiver-oO3EQwI9.js +14 -0
  97. package/dist/src/providerRegistry-4yjhaEM8.js +45 -0
  98. package/dist/src/providerRegistry-DhV4rJIc.cjs +50 -0
  99. package/dist/src/providers-B5RJVG-7.cjs +33609 -0
  100. package/dist/src/providers-BdmZCLzV.js +33262 -0
  101. package/dist/src/providers-CxtRxn8e.js +2 -0
  102. package/dist/src/providers-DnQLNbx1.cjs +3 -0
  103. package/dist/src/pythonUtils-BD0druiM.cjs +275 -0
  104. package/dist/src/pythonUtils-IBhn5YGR.js +249 -0
  105. package/dist/src/quiverai-BDOwZBsM.cjs +213 -0
  106. package/dist/src/quiverai-D3JTF5lD.js +213 -0
  107. package/dist/src/responses-B2LCDCXZ.js +667 -0
  108. package/dist/src/responses-BvNm4Xv9.cjs +685 -0
  109. package/dist/src/rubyUtils-B0NwnfpY.cjs +245 -0
  110. package/dist/src/rubyUtils-BroxzZ7c.cjs +2 -0
  111. package/dist/src/rubyUtils-hqVw5UvJ.js +222 -0
  112. package/dist/src/sagemaker-Cno2V-Sx.js +689 -0
  113. package/dist/src/sagemaker-fV_KUgs5.cjs +691 -0
  114. package/dist/src/server-BOuAXb06.cjs +238 -0
  115. package/dist/src/server-CtI-EWzm.cjs +2 -0
  116. package/dist/src/server-Cy3DZymt.js +189 -0
  117. package/dist/src/slack-CP8xBePa.js +135 -0
  118. package/dist/src/slack-DSQ1yXVb.cjs +135 -0
  119. package/dist/src/store-BwDDaBjb.cjs +246 -0
  120. package/dist/src/store-DcbLC593.cjs +2 -0
  121. package/dist/src/store-IGpqMIkv.js +240 -0
  122. package/dist/src/tables-3Q2cL7So.cjs +373 -0
  123. package/dist/src/tables-Bi2fjr4W.js +288 -0
  124. package/dist/src/telemetry-Bg2WqF79.js +161 -0
  125. package/dist/src/telemetry-D0x6u5kX.cjs +166 -0
  126. package/dist/src/telemetry-DXNimrI0.cjs +2 -0
  127. package/dist/src/text-B_UCRPp2.js +22 -0
  128. package/dist/src/text-CW1cyrwj.cjs +33 -0
  129. package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
  130. package/dist/src/tokenUsageUtils-bVa1ga6f.cjs +173 -0
  131. package/dist/src/transcription-Cl_W16Pr.js +122 -0
  132. package/dist/src/transcription-yt1EecY8.cjs +124 -0
  133. package/dist/src/transform-BCtGrl_W.cjs +228 -0
  134. package/dist/src/transform-Bv6gG2MJ.cjs +1688 -0
  135. package/dist/src/transform-CY1wbpRy.js +1507 -0
  136. package/dist/src/transform-DU8rUL9P.cjs +2 -0
  137. package/dist/src/transform-yWaShiKr.js +216 -0
  138. package/dist/src/transformersAvailability-BGkzavwb.js +35 -0
  139. package/dist/src/transformersAvailability-DKoRtQLy.cjs +35 -0
  140. package/dist/src/types-5aqHpBwE.cjs +3769 -0
  141. package/dist/src/types-Bn6D9c4U.js +3300 -0
  142. package/dist/src/util-BkKlTkI2.js +293 -0
  143. package/dist/src/util-CTh0bfOm.cjs +1119 -0
  144. package/dist/src/util-D17oBwo7.cjs +328 -0
  145. package/dist/src/util-DsS_-v4p.js +613 -0
  146. package/dist/src/util-DuntT1Ga.js +951 -0
  147. package/dist/src/util-aWjdCYMI.cjs +667 -0
  148. package/dist/src/utils-CisQwpjA.js +94 -0
  149. package/dist/src/utils-yWamDvmz.cjs +123 -0
  150. package/dist/tsconfig.tsbuildinfo +1 -0
  151. package/drizzle/0000_lush_hellion.sql +36 -0
  152. package/drizzle/0001_wide_calypso.sql +3 -0
  153. package/drizzle/0002_tidy_juggernaut.sql +1 -0
  154. package/drizzle/0003_lively_naoko.sql +8 -0
  155. package/drizzle/0004_minor_peter_quill.sql +19 -0
  156. package/drizzle/0005_silky_millenium_guard.sql +2 -0
  157. package/drizzle/0006_harsh_caretaker.sql +42 -0
  158. package/drizzle/0007_cloudy_wong.sql +1 -0
  159. package/drizzle/0008_broad_boomer.sql +2 -0
  160. package/drizzle/0009_strong_marten_broadcloak.sql +19 -0
  161. package/drizzle/0010_needy_bishop.sql +11 -0
  162. package/drizzle/0011_moaning_millenium_guard.sql +1 -0
  163. package/drizzle/0012_late_marten_broadcloak.sql +2 -0
  164. package/drizzle/0013_previous_dormammu.sql +9 -0
  165. package/drizzle/0014_lazy_captain_universe.sql +2 -0
  166. package/drizzle/0015_zippy_wallop.sql +29 -0
  167. package/drizzle/0016_jazzy_zemo.sql +2 -0
  168. package/drizzle/0017_reflective_praxagora.sql +4 -0
  169. package/drizzle/0018_fat_vanisher.sql +22 -0
  170. package/drizzle/0019_new_clint_barton.sql +8 -0
  171. package/drizzle/0020_skinny_maverick.sql +1 -0
  172. package/drizzle/0021_mysterious_madelyne_pryor.sql +13 -0
  173. package/drizzle/0022_sleepy_ultimo.sql +25 -0
  174. package/drizzle/0023_wooden_mandrill.sql +2 -0
  175. package/drizzle/AGENTS.md +68 -0
  176. package/drizzle/CLAUDE.md +1 -0
  177. package/drizzle/meta/0000_snapshot.json +221 -0
  178. package/drizzle/meta/0001_snapshot.json +214 -0
  179. package/drizzle/meta/0002_snapshot.json +221 -0
  180. package/drizzle/meta/0005_snapshot.json +369 -0
  181. package/drizzle/meta/0006_snapshot.json +638 -0
  182. package/drizzle/meta/0007_snapshot.json +640 -0
  183. package/drizzle/meta/0008_snapshot.json +649 -0
  184. package/drizzle/meta/0009_snapshot.json +554 -0
  185. package/drizzle/meta/0010_snapshot.json +619 -0
  186. package/drizzle/meta/0011_snapshot.json +627 -0
  187. package/drizzle/meta/0012_snapshot.json +639 -0
  188. package/drizzle/meta/0013_snapshot.json +717 -0
  189. package/drizzle/meta/0014_snapshot.json +717 -0
  190. package/drizzle/meta/0015_snapshot.json +897 -0
  191. package/drizzle/meta/0016_snapshot.json +1031 -0
  192. package/drizzle/meta/0018_snapshot.json +1210 -0
  193. package/drizzle/meta/0019_snapshot.json +1165 -0
  194. package/drizzle/meta/0020_snapshot.json +1232 -0
  195. package/drizzle/meta/0021_snapshot.json +1311 -0
  196. package/drizzle/meta/0022_snapshot.json +1481 -0
  197. package/drizzle/meta/0023_snapshot.json +1496 -0
  198. package/drizzle/meta/_journal.json +174 -0
  199. package/package.json +240 -0
@@ -0,0 +1,3769 @@
1
+ const require_logger = require("./logger-DyfK9PBt.cjs");
2
+ const require_fileExtensions = require("./fileExtensions-bYh77CN8.cjs");
3
+ let dedent = require("dedent");
4
+ dedent = require_logger.__toESM(dedent);
5
+ let zod = require("zod");
6
+ //#region src/types/env.ts
7
+ const ProviderEnvOverridesSchema = zod.z.object({
8
+ AI21_API_BASE_URL: zod.z.string().optional(),
9
+ AI21_API_KEY: zod.z.string().optional(),
10
+ AIML_API_KEY: zod.z.string().optional(),
11
+ ANTHROPIC_API_KEY: zod.z.string().optional(),
12
+ ANTHROPIC_BASE_URL: zod.z.string().optional(),
13
+ AWS_BEDROCK_REGION: zod.z.string().optional(),
14
+ AZURE_API_BASE_URL: zod.z.string().optional(),
15
+ AZURE_API_HOST: zod.z.string().optional(),
16
+ AZURE_API_KEY: zod.z.string().optional(),
17
+ AZURE_AUTHORITY_HOST: zod.z.string().optional(),
18
+ AZURE_CLIENT_ID: zod.z.string().optional(),
19
+ AZURE_CLIENT_SECRET: zod.z.string().optional(),
20
+ AZURE_DEPLOYMENT_NAME: zod.z.string().optional(),
21
+ AZURE_EMBEDDING_DEPLOYMENT_NAME: zod.z.string().optional(),
22
+ AZURE_OPENAI_API_BASE_URL: zod.z.string().optional(),
23
+ AZURE_OPENAI_API_HOST: zod.z.string().optional(),
24
+ AZURE_OPENAI_API_KEY: zod.z.string().optional(),
25
+ AZURE_OPENAI_BASE_URL: zod.z.string().optional(),
26
+ AZURE_OPENAI_DEPLOYMENT_NAME: zod.z.string().optional(),
27
+ AZURE_OPENAI_EMBEDDING_DEPLOYMENT_NAME: zod.z.string().optional(),
28
+ AZURE_TENANT_ID: zod.z.string().optional(),
29
+ AZURE_TOKEN_SCOPE: zod.z.string().optional(),
30
+ CLAUDE_CODE_USE_BEDROCK: zod.z.string().optional(),
31
+ CLAUDE_CODE_USE_VERTEX: zod.z.string().optional(),
32
+ CLOUDFLARE_ACCOUNT_ID: zod.z.string().optional(),
33
+ CLOUDFLARE_API_KEY: zod.z.string().optional(),
34
+ CLOUDFLARE_GATEWAY_ID: zod.z.string().optional(),
35
+ CF_AIG_TOKEN: zod.z.string().optional(),
36
+ COMETAPI_KEY: zod.z.string().optional(),
37
+ COHERE_API_KEY: zod.z.string().optional(),
38
+ COHERE_CLIENT_NAME: zod.z.string().optional(),
39
+ DATABRICKS_TOKEN: zod.z.string().optional(),
40
+ DATABRICKS_WORKSPACE_URL: zod.z.string().optional(),
41
+ DOCKER_MODEL_RUNNER_BASE_URL: zod.z.string().optional(),
42
+ DOCKER_MODEL_RUNNER_API_KEY: zod.z.string().optional(),
43
+ ELEVENLABS_API_KEY: zod.z.string().optional(),
44
+ FAL_KEY: zod.z.string().optional(),
45
+ GITHUB_TOKEN: zod.z.string().optional(),
46
+ GOOGLE_API_HOST: zod.z.string().optional(),
47
+ GOOGLE_API_BASE_URL: zod.z.string().optional(),
48
+ GOOGLE_API_KEY: zod.z.string().optional(),
49
+ GOOGLE_PROJECT_ID: zod.z.string().optional(),
50
+ GOOGLE_LOCATION: zod.z.string().optional(),
51
+ GOOGLE_GENERATIVE_AI_API_KEY: zod.z.string().optional(),
52
+ GEMINI_API_KEY: zod.z.string().optional(),
53
+ GROQ_API_KEY: zod.z.string().optional(),
54
+ HELICONE_API_KEY: zod.z.string().optional(),
55
+ HF_API_TOKEN: zod.z.string().optional(),
56
+ HF_TOKEN: zod.z.string().optional(),
57
+ HYPERBOLIC_API_KEY: zod.z.string().optional(),
58
+ HUGGING_FACE_HUB_TOKEN: zod.z.string().optional(),
59
+ JFROG_API_KEY: zod.z.string().optional(),
60
+ LANGFUSE_HOST: zod.z.string().optional(),
61
+ LANGFUSE_PUBLIC_KEY: zod.z.string().optional(),
62
+ LANGFUSE_SECRET_KEY: zod.z.string().optional(),
63
+ LITELLM_API_BASE: zod.z.string().optional(),
64
+ LLAMA_BASE_URL: zod.z.string().optional(),
65
+ LOCALAI_BASE_URL: zod.z.string().optional(),
66
+ MISTRAL_API_BASE_URL: zod.z.string().optional(),
67
+ MISTRAL_API_HOST: zod.z.string().optional(),
68
+ MISTRAL_API_KEY: zod.z.string().optional(),
69
+ MODELSLAB_API_KEY: zod.z.string().optional(),
70
+ NSCALE_SERVICE_TOKEN: zod.z.string().optional(),
71
+ NSCALE_API_KEY: zod.z.string().optional(),
72
+ OLLAMA_API_KEY: zod.z.string().optional(),
73
+ OLLAMA_BASE_URL: zod.z.string().optional(),
74
+ OPENAI_API_BASE_URL: zod.z.string().optional(),
75
+ OPENAI_API_HOST: zod.z.string().optional(),
76
+ OPENAI_API_KEY: zod.z.string().optional(),
77
+ OPENAI_BASE_URL: zod.z.string().optional(),
78
+ OPENAI_ORGANIZATION: zod.z.string().optional(),
79
+ CLAWDBOT_GATEWAY_PASSWORD: zod.z.string().optional(),
80
+ CLAWDBOT_GATEWAY_TOKEN: zod.z.string().optional(),
81
+ CLAWDBOT_GATEWAY_URL: zod.z.string().optional(),
82
+ CODEX_API_KEY: zod.z.string().optional(),
83
+ OPENCLAW_CONFIG_PATH: zod.z.string().optional(),
84
+ OPENCLAW_GATEWAY_PASSWORD: zod.z.string().optional(),
85
+ OPENCLAW_GATEWAY_TOKEN: zod.z.string().optional(),
86
+ OPENCLAW_GATEWAY_URL: zod.z.string().optional(),
87
+ PALM_API_HOST: zod.z.string().optional(),
88
+ PALM_API_KEY: zod.z.string().optional(),
89
+ PORTKEY_API_KEY: zod.z.string().optional(),
90
+ PROMPTFOO_CA_CERT_PATH: zod.z.string().optional(),
91
+ PROMPTFOO_PFX_CERT_PATH: zod.z.string().optional(),
92
+ PROMPTFOO_PFX_PASSWORD: zod.z.string().optional(),
93
+ PROMPTFOO_JKS_CERT_PATH: zod.z.string().optional(),
94
+ PROMPTFOO_JKS_PASSWORD: zod.z.string().optional(),
95
+ PROMPTFOO_JKS_ALIAS: zod.z.string().optional(),
96
+ PROMPTFOO_INSECURE_SSL: zod.z.string().optional(),
97
+ QUIVERAI_API_KEY: zod.z.string().optional(),
98
+ REPLICATE_API_KEY: zod.z.string().optional(),
99
+ REPLICATE_API_TOKEN: zod.z.string().optional(),
100
+ SHAREPOINT_BASE_URL: zod.z.string().optional(),
101
+ SHAREPOINT_CERT_PATH: zod.z.string().optional(),
102
+ SHAREPOINT_CLIENT_ID: zod.z.string().optional(),
103
+ SHAREPOINT_TENANT_ID: zod.z.string().optional(),
104
+ VERCEL_AI_GATEWAY_API_KEY: zod.z.string().optional(),
105
+ VERCEL_AI_GATEWAY_BASE_URL: zod.z.string().optional(),
106
+ VERTEX_API_HOST: zod.z.string().optional(),
107
+ VERTEX_API_KEY: zod.z.string().optional(),
108
+ VERTEX_API_VERSION: zod.z.string().optional(),
109
+ VERTEX_PROJECT_ID: zod.z.string().optional(),
110
+ VERTEX_PUBLISHER: zod.z.string().optional(),
111
+ VERTEX_REGION: zod.z.string().optional(),
112
+ VOYAGE_API_BASE_URL: zod.z.string().optional(),
113
+ VOYAGE_API_KEY: zod.z.string().optional(),
114
+ WATSONX_AI_APIKEY: zod.z.string().optional(),
115
+ WATSONX_AI_AUTH_TYPE: zod.z.string().optional(),
116
+ WATSONX_AI_BEARER_TOKEN: zod.z.string().optional(),
117
+ WATSONX_AI_PROJECT_ID: zod.z.string().optional(),
118
+ XAI_API_BASE_URL: zod.z.string().optional(),
119
+ XAI_API_KEY: zod.z.string().optional(),
120
+ AZURE_CONTENT_SAFETY_ENDPOINT: zod.z.string().optional(),
121
+ AZURE_CONTENT_SAFETY_API_KEY: zod.z.string().optional(),
122
+ AZURE_CONTENT_SAFETY_API_VERSION: zod.z.string().optional(),
123
+ AWS_REGION: zod.z.string().optional(),
124
+ AWS_DEFAULT_REGION: zod.z.string().optional(),
125
+ AWS_SAGEMAKER_MAX_TOKENS: zod.z.string().optional(),
126
+ AWS_SAGEMAKER_TEMPERATURE: zod.z.string().optional(),
127
+ AWS_SAGEMAKER_TOP_P: zod.z.string().optional(),
128
+ AWS_SAGEMAKER_MAX_RETRIES: zod.z.string().optional(),
129
+ PROMPTFOO_EVAL_TIMEOUT_MS: zod.z.string().optional()
130
+ });
131
+ //#endregion
132
+ //#region src/types/shared.ts
133
+ const CompletionTokenDetailsSchema = zod.z.object({
134
+ reasoning: zod.z.number().optional(),
135
+ acceptedPrediction: zod.z.number().optional(),
136
+ rejectedPrediction: zod.z.number().optional()
137
+ });
138
+ /**
139
+ * Base schema for token usage statistics with all fields optional
140
+ */
141
+ const BaseTokenUsageSchema = zod.z.object({
142
+ prompt: zod.z.number().optional(),
143
+ completion: zod.z.number().optional(),
144
+ cached: zod.z.number().optional(),
145
+ total: zod.z.number().optional(),
146
+ numRequests: zod.z.number().optional(),
147
+ completionDetails: CompletionTokenDetailsSchema.optional(),
148
+ assertions: zod.z.object({
149
+ total: zod.z.number().optional(),
150
+ prompt: zod.z.number().optional(),
151
+ completion: zod.z.number().optional(),
152
+ cached: zod.z.number().optional(),
153
+ numRequests: zod.z.number().optional(),
154
+ completionDetails: CompletionTokenDetailsSchema.optional()
155
+ }).optional()
156
+ });
157
+ const InputsSchema = zod.z.record(zod.z.string().regex(/^[a-zA-Z_][a-zA-Z0-9_]*$/, { error: "Input variable names must be valid identifiers (start with letter or underscore)" }), zod.z.string().min(1, { error: "Input descriptions must be non-empty strings" }));
158
+ //#endregion
159
+ //#region src/validators/prompts.ts
160
+ const PromptConfigSchema = zod.z.object({
161
+ prefix: zod.z.string().optional(),
162
+ suffix: zod.z.string().optional()
163
+ });
164
+ const PromptFunctionSchema = zod.z.custom((v) => typeof v === "function");
165
+ const PromptSchema = zod.z.object({
166
+ id: zod.z.string().optional(),
167
+ raw: zod.z.string(),
168
+ template: zod.z.string().optional(),
169
+ display: zod.z.string().optional(),
170
+ label: zod.z.string(),
171
+ function: PromptFunctionSchema.optional(),
172
+ config: zod.z.any().optional()
173
+ });
174
+ function assert$1() {}
175
+ assert$1();
176
+ assert$1();
177
+ assert$1();
178
+ //#endregion
179
+ //#region src/redteam/constants/plugins.ts
180
+ const MULTI_INPUT_VAR = "__prompt";
181
+ const LLAMA_GUARD_REPLICATE_PROVIDER = "replicate:moderation:meta/llama-guard-4-12b";
182
+ const LLAMA_GUARD_ENABLED_CATEGORIES = [
183
+ "S1",
184
+ "S2",
185
+ "S3",
186
+ "S4",
187
+ "S5",
188
+ "S6",
189
+ "S8",
190
+ "S9",
191
+ "S10",
192
+ "S11",
193
+ "S12",
194
+ "S13"
195
+ ];
196
+ const FOUNDATION_PLUGINS = [
197
+ "ascii-smuggling",
198
+ "beavertails",
199
+ "bias:age",
200
+ "bias:disability",
201
+ "bias:gender",
202
+ "bias:race",
203
+ "contracts",
204
+ "cyberseceval",
205
+ "donotanswer",
206
+ "divergent-repetition",
207
+ "excessive-agency",
208
+ "hallucination",
209
+ "harmful:chemical-biological-weapons",
210
+ "harmful:child-exploitation",
211
+ "harmful:copyright-violations",
212
+ "harmful:cybercrime",
213
+ "harmful:cybercrime:malicious-code",
214
+ "harmful:graphic-content",
215
+ "harmful:harassment-bullying",
216
+ "harmful:hate",
217
+ "harmful:illegal-activities",
218
+ "harmful:illegal-drugs",
219
+ "harmful:illegal-drugs:meth",
220
+ "harmful:indiscriminate-weapons",
221
+ "harmful:insults",
222
+ "harmful:intellectual-property",
223
+ "harmful:misinformation-disinformation",
224
+ "harmful:non-violent-crime",
225
+ "harmful:profanity",
226
+ "harmful:radicalization",
227
+ "harmful:self-harm",
228
+ "harmful:sex-crime",
229
+ "harmful:sexual-content",
230
+ "harmful:specialized-advice",
231
+ "harmful:unsafe-practices",
232
+ "harmful:violent-crime",
233
+ "harmful:weapons:ied",
234
+ "hijacking",
235
+ "imitation",
236
+ "overreliance",
237
+ "pii:direct",
238
+ "pliny",
239
+ "politics",
240
+ "religion"
241
+ ];
242
+ const GUARDRAILS_EVALUATION_PLUGINS = [
243
+ "ascii-smuggling",
244
+ "indirect-prompt-injection",
245
+ "cca",
246
+ "hijacking",
247
+ "system-prompt-override",
248
+ "beavertails",
249
+ "harmbench",
250
+ "pliny",
251
+ "donotanswer",
252
+ "prompt-extraction",
253
+ "harmful:chemical-biological-weapons",
254
+ "harmful:indiscriminate-weapons",
255
+ "harmful:weapons:ied",
256
+ "harmful:violent-crime",
257
+ "harmful:sex-crime",
258
+ "harmful:non-violent-crime",
259
+ "harmful:graphic-content",
260
+ "harmful:unsafe-practices",
261
+ "harmful:child-exploitation",
262
+ "harmful:harassment-bullying",
263
+ "harmful:hate",
264
+ "harmful:self-harm",
265
+ "harmful:sexual-content",
266
+ "harmful:insults",
267
+ "harmful:profanity",
268
+ "harmful:radicalization",
269
+ "harmful:cybercrime",
270
+ "harmful:cybercrime:malicious-code",
271
+ "harmful:illegal-activities",
272
+ "harmful:illegal-drugs",
273
+ "harmful:illegal-drugs:meth",
274
+ "harmful:misinformation-disinformation",
275
+ "harmful:specialized-advice",
276
+ "harmful:copyright-violations",
277
+ "harmful:intellectual-property",
278
+ "cyberseceval",
279
+ "excessive-agency",
280
+ "hallucination",
281
+ "overreliance",
282
+ "divergent-repetition",
283
+ "reasoning-dos",
284
+ "harmful:privacy"
285
+ ];
286
+ const AGENTIC_PLUGINS = ["agentic:memory-poisoning"];
287
+ const COLLECTIONS = [
288
+ "default",
289
+ "foundation",
290
+ "harmful",
291
+ "pii",
292
+ "bias",
293
+ "medical",
294
+ "pharmacy",
295
+ "insurance",
296
+ "financial",
297
+ "ecommerce",
298
+ "telecom",
299
+ "realestate",
300
+ "guardrails-eval"
301
+ ];
302
+ const UNALIGNED_PROVIDER_HARM_PLUGINS = {
303
+ "harmful:child-exploitation": "Child Exploitation",
304
+ "harmful:hate": "Hate",
305
+ "harmful:indiscriminate-weapons": "Indiscriminate Weapons",
306
+ "harmful:non-violent-crime": "Non-Violent Crimes",
307
+ "harmful:self-harm": "Self-Harm",
308
+ "harmful:sex-crime": "Sex Crimes",
309
+ "harmful:sexual-content": "Sexual Content",
310
+ "harmful:violent-crime": "Violent Crimes",
311
+ "harmful:chemical-biological-weapons": "Chemical & Biological Weapons",
312
+ "harmful:copyright-violations": "Copyright Violations - Copyrighted text",
313
+ "harmful:cybercrime": "Cybercrime & Unauthorized Intrusion - Hacking and Malware",
314
+ "harmful:graphic-content": "Graphic & age-restricted content",
315
+ "harmful:harassment-bullying": "Harassment & Bullying",
316
+ "harmful:illegal-activities": "Illegal Activities - Fraud & scams",
317
+ "harmful:illegal-drugs": "Illegal Drugs",
318
+ "harmful:unsafe-practices": "Promotion of unsafe practices",
319
+ "harmful:insults": "Insults and personal attacks",
320
+ "harmful:profanity": "Requests containing profanity",
321
+ "harmful:radicalization": "Radicalization",
322
+ "harmful:illegal-drugs:meth": "Methamphetamine",
323
+ "harmful:weapons:ied": "Improvised Explosive Devices",
324
+ "harmful:cybercrime:malicious-code": "Malicious Code"
325
+ };
326
+ const REDTEAM_PROVIDER_HARM_PLUGINS = {
327
+ "harmful:intellectual-property": "Intellectual Property violation",
328
+ "harmful:privacy": "Privacy violations"
329
+ };
330
+ const HARM_PLUGINS = {
331
+ "harmful:misinformation-disinformation": "Misinformation & Disinformation - Harmful lies and propaganda",
332
+ "harmful:specialized-advice": "Specialized Advice - Financial",
333
+ ...UNALIGNED_PROVIDER_HARM_PLUGINS,
334
+ ...REDTEAM_PROVIDER_HARM_PLUGINS
335
+ };
336
+ const PII_PLUGINS = [
337
+ "pii:api-db",
338
+ "pii:direct",
339
+ "pii:session",
340
+ "pii:social"
341
+ ];
342
+ const BIAS_PLUGINS = [
343
+ "bias:age",
344
+ "bias:disability",
345
+ "bias:gender",
346
+ "bias:race"
347
+ ];
348
+ const MEDICAL_PLUGINS = [
349
+ "medical:anchoring-bias",
350
+ "medical:hallucination",
351
+ "medical:incorrect-knowledge",
352
+ "medical:off-label-use",
353
+ "medical:prioritization-error",
354
+ "medical:sycophancy"
355
+ ];
356
+ const FINANCIAL_PLUGINS = [
357
+ "financial:calculation-error",
358
+ "financial:compliance-violation",
359
+ "financial:confidential-disclosure",
360
+ "financial:counterfactual",
361
+ "financial:data-leakage",
362
+ "financial:defamation",
363
+ "financial:hallucination",
364
+ "financial:impartiality",
365
+ "financial:japan-fiea-suitability",
366
+ "financial:misconduct",
367
+ "financial:sox-compliance",
368
+ "financial:sycophancy"
369
+ ];
370
+ const PHARMACY_PLUGINS = [
371
+ "pharmacy:controlled-substance-compliance",
372
+ "pharmacy:dosage-calculation",
373
+ "pharmacy:drug-interaction"
374
+ ];
375
+ const INSURANCE_PLUGINS = [
376
+ "insurance:coverage-discrimination",
377
+ "insurance:data-disclosure",
378
+ "insurance:network-misinformation",
379
+ "insurance:phi-disclosure"
380
+ ];
381
+ const ECOMMERCE_PLUGINS = [
382
+ "ecommerce:compliance-bypass",
383
+ "ecommerce:order-fraud",
384
+ "ecommerce:pci-dss",
385
+ "ecommerce:price-manipulation"
386
+ ];
387
+ const TELECOM_PLUGINS = [
388
+ "telecom:cpni-disclosure",
389
+ "telecom:location-disclosure",
390
+ "telecom:account-takeover",
391
+ "telecom:e911-misinformation",
392
+ "telecom:tcpa-violation",
393
+ "telecom:unauthorized-changes",
394
+ "telecom:fraud-enablement",
395
+ "telecom:porting-misinformation",
396
+ "telecom:billing-misinformation",
397
+ "telecom:coverage-misinformation",
398
+ "telecom:law-enforcement-request-handling",
399
+ "telecom:accessibility-violation"
400
+ ];
401
+ const REALESTATE_PLUGINS = [
402
+ "realestate:fair-housing-discrimination",
403
+ "realestate:steering",
404
+ "realestate:discriminatory-listings",
405
+ "realestate:lending-discrimination",
406
+ "realestate:valuation-bias",
407
+ "realestate:accessibility-discrimination",
408
+ "realestate:advertising-discrimination",
409
+ "realestate:source-of-income"
410
+ ];
411
+ const BASE_PLUGINS = [
412
+ "contracts",
413
+ "excessive-agency",
414
+ "hallucination",
415
+ "hijacking",
416
+ "politics"
417
+ ];
418
+ const ADDITIONAL_PLUGINS = [
419
+ "aegis",
420
+ "ascii-smuggling",
421
+ "beavertails",
422
+ "bfla",
423
+ "bola",
424
+ "cca",
425
+ "competitors",
426
+ "coppa",
427
+ "cross-session-leak",
428
+ "cyberseceval",
429
+ "data-exfil",
430
+ "debug-access",
431
+ "divergent-repetition",
432
+ "donotanswer",
433
+ "ferpa",
434
+ "harmbench",
435
+ "toxic-chat",
436
+ "imitation",
437
+ "indirect-prompt-injection",
438
+ "mcp",
439
+ "model-identification",
440
+ "medical:anchoring-bias",
441
+ "medical:hallucination",
442
+ "medical:incorrect-knowledge",
443
+ "medical:off-label-use",
444
+ "medical:prioritization-error",
445
+ "medical:sycophancy",
446
+ "financial:calculation-error",
447
+ "financial:compliance-violation",
448
+ "financial:confidential-disclosure",
449
+ "financial:counterfactual",
450
+ "financial:data-leakage",
451
+ "financial:defamation",
452
+ "financial:hallucination",
453
+ "financial:impartiality",
454
+ "financial:japan-fiea-suitability",
455
+ "financial:misconduct",
456
+ "financial:sox-compliance",
457
+ "financial:sycophancy",
458
+ "ecommerce:compliance-bypass",
459
+ "ecommerce:order-fraud",
460
+ "ecommerce:pci-dss",
461
+ "ecommerce:price-manipulation",
462
+ "goal-misalignment",
463
+ "insurance:coverage-discrimination",
464
+ "insurance:data-disclosure",
465
+ "insurance:network-misinformation",
466
+ "insurance:phi-disclosure",
467
+ "off-topic",
468
+ "overreliance",
469
+ "pharmacy:controlled-substance-compliance",
470
+ "pharmacy:dosage-calculation",
471
+ "pharmacy:drug-interaction",
472
+ "telecom:cpni-disclosure",
473
+ "telecom:location-disclosure",
474
+ "telecom:account-takeover",
475
+ "telecom:e911-misinformation",
476
+ "telecom:tcpa-violation",
477
+ "telecom:unauthorized-changes",
478
+ "telecom:fraud-enablement",
479
+ "telecom:porting-misinformation",
480
+ "telecom:billing-misinformation",
481
+ "telecom:coverage-misinformation",
482
+ "telecom:law-enforcement-request-handling",
483
+ "telecom:accessibility-violation",
484
+ "realestate:fair-housing-discrimination",
485
+ "realestate:steering",
486
+ "realestate:discriminatory-listings",
487
+ "realestate:lending-discrimination",
488
+ "realestate:valuation-bias",
489
+ "realestate:accessibility-discrimination",
490
+ "realestate:advertising-discrimination",
491
+ "realestate:source-of-income",
492
+ "pliny",
493
+ "prompt-extraction",
494
+ "rag-document-exfiltration",
495
+ "rag-poisoning",
496
+ "rag-source-attribution",
497
+ "rbac",
498
+ "reasoning-dos",
499
+ "religion",
500
+ "shell-injection",
501
+ "special-token-injection",
502
+ "sql-injection",
503
+ "ssrf",
504
+ "system-prompt-override",
505
+ "tool-discovery",
506
+ "unsafebench",
507
+ "unverifiable-claims",
508
+ "vlguard",
509
+ "vlsu",
510
+ "wordplay",
511
+ "xstest"
512
+ ];
513
+ const CONFIG_REQUIRED_PLUGINS = ["intent", "policy"];
514
+ const AGENTIC_EXEMPT_PLUGINS = ["system-prompt-override", "agentic:memory-poisoning"];
515
+ const DATASET_EXEMPT_PLUGINS = [
516
+ "aegis",
517
+ "beavertails",
518
+ "cyberseceval",
519
+ "donotanswer",
520
+ "harmbench",
521
+ "pliny",
522
+ "toxic-chat",
523
+ "unsafebench",
524
+ "vlguard",
525
+ "vlsu",
526
+ "xstest"
527
+ ];
528
+ const MULTI_INPUT_EXCLUDED_PLUGINS = [
529
+ "cca",
530
+ "cross-session-leak",
531
+ "special-token-injection",
532
+ "system-prompt-override",
533
+ "ascii-smuggling"
534
+ ];
535
+ const STRATEGY_EXEMPT_PLUGINS = [...AGENTIC_EXEMPT_PLUGINS, ...DATASET_EXEMPT_PLUGINS];
536
+ const DEFAULT_PLUGINS = new Set([...[
537
+ ...BASE_PLUGINS,
538
+ ...Object.keys(HARM_PLUGINS),
539
+ ...PII_PLUGINS,
540
+ ...BIAS_PLUGINS
541
+ ].sort()]);
542
+ new Set([
543
+ ...DEFAULT_PLUGINS,
544
+ "bola",
545
+ "bfla",
546
+ "rbac",
547
+ "rag-source-attribution"
548
+ ]);
549
+ const ALL_PLUGINS = [...new Set([
550
+ ...DEFAULT_PLUGINS,
551
+ ...ADDITIONAL_PLUGINS,
552
+ ...CONFIG_REQUIRED_PLUGINS,
553
+ ...AGENTIC_PLUGINS
554
+ ])].sort();
555
+ const PLUGIN_CATEGORIES = {
556
+ bias: BIAS_PLUGINS,
557
+ ecommerce: ECOMMERCE_PLUGINS,
558
+ financial: FINANCIAL_PLUGINS,
559
+ harmful: Object.keys(HARM_PLUGINS),
560
+ pii: PII_PLUGINS,
561
+ medical: MEDICAL_PLUGINS,
562
+ pharmacy: PHARMACY_PLUGINS,
563
+ insurance: INSURANCE_PLUGINS,
564
+ telecom: TELECOM_PLUGINS,
565
+ realestate: REALESTATE_PLUGINS
566
+ };
567
+ const REMOTE_ONLY_PLUGIN_IDS = [
568
+ "agentic:memory-poisoning",
569
+ "ascii-smuggling",
570
+ "bfla",
571
+ "bola",
572
+ "cca",
573
+ "competitors",
574
+ "coppa",
575
+ "data-exfil",
576
+ "ferpa",
577
+ "goal-misalignment",
578
+ "harmful:misinformation-disinformation",
579
+ "harmful:specialized-advice",
580
+ "hijacking",
581
+ "indirect-prompt-injection",
582
+ "mcp",
583
+ "model-identification",
584
+ "off-topic",
585
+ "rag-document-exfiltration",
586
+ "rag-poisoning",
587
+ "rag-source-attribution",
588
+ "reasoning-dos",
589
+ "religion",
590
+ "special-token-injection",
591
+ "ssrf",
592
+ "system-prompt-override",
593
+ "wordplay",
594
+ ...MEDICAL_PLUGINS,
595
+ ...FINANCIAL_PLUGINS,
596
+ ...PHARMACY_PLUGINS,
597
+ ...INSURANCE_PLUGINS,
598
+ ...ECOMMERCE_PLUGINS,
599
+ ...TELECOM_PLUGINS,
600
+ ...REALESTATE_PLUGINS
601
+ ];
602
+ [
603
+ ...Object.keys(UNALIGNED_PROVIDER_HARM_PLUGINS),
604
+ ...BIAS_PLUGINS,
605
+ ...REMOTE_ONLY_PLUGIN_IDS
606
+ ];
607
+ //#endregion
608
+ //#region src/redteam/constants/frameworks.ts
609
+ const OWASP_LLM_TOP_10_MAPPING = {
610
+ "owasp:llm:01": {
611
+ plugins: [
612
+ "ascii-smuggling",
613
+ "indirect-prompt-injection",
614
+ "prompt-extraction",
615
+ "harmful"
616
+ ],
617
+ strategies: [
618
+ "jailbreak",
619
+ "jailbreak-templates",
620
+ "jailbreak:composite"
621
+ ]
622
+ },
623
+ "owasp:llm:02": {
624
+ plugins: [
625
+ "pii:api-db",
626
+ "pii:direct",
627
+ "pii:session",
628
+ "pii:social",
629
+ "harmful:privacy",
630
+ "cross-session-leak",
631
+ "prompt-extraction"
632
+ ],
633
+ strategies: [
634
+ "jailbreak",
635
+ "jailbreak-templates",
636
+ "jailbreak:composite"
637
+ ]
638
+ },
639
+ "owasp:llm:03": {
640
+ plugins: [],
641
+ strategies: []
642
+ },
643
+ "owasp:llm:04": {
644
+ plugins: [
645
+ "harmful:misinformation-disinformation",
646
+ "harmful:hate",
647
+ "bias:age",
648
+ "bias:disability",
649
+ "bias:gender",
650
+ "bias:race",
651
+ "harmful:radicalization",
652
+ "harmful:specialized-advice"
653
+ ],
654
+ strategies: [
655
+ "jailbreak",
656
+ "jailbreak-templates",
657
+ "jailbreak:composite"
658
+ ]
659
+ },
660
+ "owasp:llm:05": {
661
+ plugins: [
662
+ "shell-injection",
663
+ "sql-injection",
664
+ "ssrf",
665
+ "debug-access"
666
+ ],
667
+ strategies: ["jailbreak", "jailbreak-templates"]
668
+ },
669
+ "owasp:llm:06": {
670
+ plugins: [
671
+ "excessive-agency",
672
+ "rbac",
673
+ "bfla",
674
+ "bola",
675
+ "shell-injection",
676
+ "sql-injection",
677
+ "ssrf"
678
+ ],
679
+ strategies: [
680
+ "jailbreak",
681
+ "jailbreak-templates",
682
+ "jailbreak:composite"
683
+ ]
684
+ },
685
+ "owasp:llm:07": {
686
+ plugins: [
687
+ "prompt-extraction",
688
+ "rbac",
689
+ "harmful:privacy",
690
+ "pii:api-db",
691
+ "pii:direct",
692
+ "pii:session",
693
+ "pii:social"
694
+ ],
695
+ strategies: [
696
+ "jailbreak",
697
+ "jailbreak-templates",
698
+ "jailbreak:composite"
699
+ ]
700
+ },
701
+ "owasp:llm:08": {
702
+ plugins: [
703
+ "cross-session-leak",
704
+ "harmful:privacy",
705
+ "pii:api-db",
706
+ "pii:direct",
707
+ "pii:session",
708
+ "pii:social"
709
+ ],
710
+ strategies: [
711
+ "jailbreak",
712
+ "jailbreak-templates",
713
+ "jailbreak:composite"
714
+ ]
715
+ },
716
+ "owasp:llm:09": {
717
+ plugins: [
718
+ "hallucination",
719
+ "overreliance",
720
+ "harmful:misinformation-disinformation",
721
+ "harmful:specialized-advice"
722
+ ],
723
+ strategies: [
724
+ "jailbreak",
725
+ "jailbreak-templates",
726
+ "jailbreak:composite"
727
+ ]
728
+ },
729
+ "owasp:llm:10": {
730
+ plugins: ["divergent-repetition", "reasoning-dos"],
731
+ strategies: []
732
+ }
733
+ };
734
+ const OWASP_API_TOP_10_MAPPING = {
735
+ "owasp:api:01": {
736
+ plugins: ["bola", "rbac"],
737
+ strategies: []
738
+ },
739
+ "owasp:api:02": {
740
+ plugins: ["bfla", "rbac"],
741
+ strategies: []
742
+ },
743
+ "owasp:api:03": {
744
+ plugins: ["excessive-agency", "overreliance"],
745
+ strategies: []
746
+ },
747
+ "owasp:api:04": {
748
+ plugins: [
749
+ "harmful:privacy",
750
+ "pii:api-db",
751
+ "pii:session"
752
+ ],
753
+ strategies: []
754
+ },
755
+ "owasp:api:05": {
756
+ plugins: [
757
+ "bfla",
758
+ "bola",
759
+ "rbac"
760
+ ],
761
+ strategies: []
762
+ },
763
+ "owasp:api:06": {
764
+ plugins: ["harmful:misinformation-disinformation", "overreliance"],
765
+ strategies: []
766
+ },
767
+ "owasp:api:07": {
768
+ plugins: ["shell-injection", "sql-injection"],
769
+ strategies: []
770
+ },
771
+ "owasp:api:08": {
772
+ plugins: [
773
+ "harmful:privacy",
774
+ "pii:api-db",
775
+ "pii:session"
776
+ ],
777
+ strategies: []
778
+ },
779
+ "owasp:api:09": {
780
+ plugins: ["harmful:specialized-advice", "overreliance"],
781
+ strategies: []
782
+ },
783
+ "owasp:api:10": {
784
+ plugins: ["debug-access", "harmful:privacy"],
785
+ strategies: []
786
+ }
787
+ };
788
+ /**
789
+ * OWASP Top 10 for Agentic Applications (December 2025)
790
+ * The official OWASP Top 10 list for AI agent security risks.
791
+ * Announced during Black Hat Europe 2025 and the OWASP Agentic Security Summit.
792
+ *
793
+ * @see https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications/
794
+ */
795
+ const OWASP_AGENTIC_TOP_10_MAPPING = {
796
+ "owasp:agentic:asi01": {
797
+ plugins: [
798
+ "hijacking",
799
+ "system-prompt-override",
800
+ "indirect-prompt-injection",
801
+ "intent"
802
+ ],
803
+ strategies: [
804
+ "jailbreak",
805
+ "jailbreak-templates",
806
+ "jailbreak:composite"
807
+ ]
808
+ },
809
+ "owasp:agentic:asi02": {
810
+ plugins: [
811
+ "excessive-agency",
812
+ "mcp",
813
+ "tool-discovery"
814
+ ],
815
+ strategies: ["jailbreak", "jailbreak-templates"]
816
+ },
817
+ "owasp:agentic:asi03": {
818
+ plugins: [
819
+ "rbac",
820
+ "bfla",
821
+ "bola",
822
+ "imitation"
823
+ ],
824
+ strategies: ["jailbreak", "jailbreak-templates"]
825
+ },
826
+ "owasp:agentic:asi04": {
827
+ plugins: ["indirect-prompt-injection", "mcp"],
828
+ strategies: ["jailbreak-templates"]
829
+ },
830
+ "owasp:agentic:asi05": {
831
+ plugins: [
832
+ "shell-injection",
833
+ "sql-injection",
834
+ "harmful:cybercrime:malicious-code",
835
+ "ssrf"
836
+ ],
837
+ strategies: ["jailbreak", "jailbreak-templates"]
838
+ },
839
+ "owasp:agentic:asi06": {
840
+ plugins: [
841
+ "agentic:memory-poisoning",
842
+ "cross-session-leak",
843
+ "indirect-prompt-injection"
844
+ ],
845
+ strategies: ["jailbreak", "crescendo"]
846
+ },
847
+ "owasp:agentic:asi07": {
848
+ plugins: [
849
+ "indirect-prompt-injection",
850
+ "hijacking",
851
+ "imitation"
852
+ ],
853
+ strategies: ["jailbreak-templates"]
854
+ },
855
+ "owasp:agentic:asi08": {
856
+ plugins: [
857
+ "hallucination",
858
+ "harmful:misinformation-disinformation",
859
+ "divergent-repetition"
860
+ ],
861
+ strategies: ["jailbreak", "jailbreak-templates"]
862
+ },
863
+ "owasp:agentic:asi09": {
864
+ plugins: [
865
+ "overreliance",
866
+ "imitation",
867
+ "harmful:misinformation-disinformation"
868
+ ],
869
+ strategies: ["crescendo"]
870
+ },
871
+ "owasp:agentic:asi10": {
872
+ plugins: [
873
+ "excessive-agency",
874
+ "hijacking",
875
+ "rbac",
876
+ "goal-misalignment"
877
+ ],
878
+ strategies: ["jailbreak", "crescendo"]
879
+ }
880
+ };
881
+ /**
882
+ * Maps each major phase of the OWASP GenAI Red Teaming Blueprint
883
+ * to relevant Promptfoo plugins and strategies for automated testing.
884
+ */
885
+ const OWASP_LLM_RED_TEAM_MAPPING = {
886
+ "owasp:llm:redteam:model": {
887
+ plugins: [...FOUNDATION_PLUGINS],
888
+ strategies: [
889
+ "jailbreak",
890
+ "jailbreak:tree",
891
+ "jailbreak:composite",
892
+ "crescendo",
893
+ "goat",
894
+ "jailbreak-templates",
895
+ "best-of-n"
896
+ ]
897
+ },
898
+ "owasp:llm:redteam:implementation": {
899
+ plugins: [
900
+ ...PII_PLUGINS,
901
+ "prompt-extraction",
902
+ "harmful:privacy",
903
+ "rbac",
904
+ "bfla",
905
+ "bola",
906
+ "ascii-smuggling"
907
+ ],
908
+ strategies: [
909
+ "jailbreak",
910
+ "jailbreak:tree",
911
+ "jailbreak:composite",
912
+ "jailbreak-templates",
913
+ "hex",
914
+ "base64",
915
+ "homoglyph",
916
+ "leetspeak",
917
+ "morse",
918
+ "piglatin",
919
+ "rot13"
920
+ ]
921
+ },
922
+ "owasp:llm:redteam:system": {
923
+ plugins: [
924
+ "shell-injection",
925
+ "sql-injection",
926
+ "ssrf",
927
+ "debug-access",
928
+ "tool-discovery",
929
+ "indirect-prompt-injection",
930
+ "hijacking"
931
+ ],
932
+ strategies: [
933
+ "jailbreak",
934
+ "jailbreak:tree",
935
+ "jailbreak:composite",
936
+ "crescendo",
937
+ "goat",
938
+ "gcg"
939
+ ]
940
+ },
941
+ "owasp:llm:redteam:runtime": {
942
+ plugins: [
943
+ "excessive-agency",
944
+ "overreliance",
945
+ "pliny",
946
+ "competitors",
947
+ "imitation",
948
+ "politics",
949
+ "religion",
950
+ "harmful:radicalization",
951
+ "harmful:self-harm",
952
+ "harmful:hate"
953
+ ],
954
+ strategies: [
955
+ "crescendo",
956
+ "goat",
957
+ "jailbreak:tree",
958
+ "jailbreak:composite",
959
+ "jailbreak-templates"
960
+ ]
961
+ }
962
+ };
963
+ const NIST_AI_RMF_MAPPING = {
964
+ "nist:ai:measure:1.1": {
965
+ plugins: ["excessive-agency", "harmful:misinformation-disinformation"],
966
+ strategies: ["jailbreak", "jailbreak-templates"]
967
+ },
968
+ "nist:ai:measure:1.2": {
969
+ plugins: ["excessive-agency", "harmful:misinformation-disinformation"],
970
+ strategies: ["jailbreak", "jailbreak-templates"]
971
+ },
972
+ "nist:ai:measure:2.1": {
973
+ plugins: [
974
+ "harmful:privacy",
975
+ "pii:api-db",
976
+ "pii:direct",
977
+ "pii:session",
978
+ "pii:social"
979
+ ],
980
+ strategies: []
981
+ },
982
+ "nist:ai:measure:2.2": {
983
+ plugins: [
984
+ "harmful:privacy",
985
+ "pii:api-db",
986
+ "pii:direct",
987
+ "pii:session",
988
+ "pii:social"
989
+ ],
990
+ strategies: []
991
+ },
992
+ "nist:ai:measure:2.3": {
993
+ plugins: ["excessive-agency"],
994
+ strategies: []
995
+ },
996
+ "nist:ai:measure:2.4": {
997
+ plugins: ["excessive-agency", "harmful:misinformation-disinformation"],
998
+ strategies: ["jailbreak", "jailbreak-templates"]
999
+ },
1000
+ "nist:ai:measure:2.5": {
1001
+ plugins: ["excessive-agency"],
1002
+ strategies: []
1003
+ },
1004
+ "nist:ai:measure:2.6": {
1005
+ plugins: [
1006
+ "harmful:chemical-biological-weapons",
1007
+ "harmful:indiscriminate-weapons",
1008
+ "harmful:unsafe-practices"
1009
+ ],
1010
+ strategies: []
1011
+ },
1012
+ "nist:ai:measure:2.7": {
1013
+ plugins: [
1014
+ "harmful:cybercrime",
1015
+ "shell-injection",
1016
+ "sql-injection"
1017
+ ],
1018
+ strategies: ["jailbreak", "jailbreak-templates"]
1019
+ },
1020
+ "nist:ai:measure:2.8": {
1021
+ plugins: [
1022
+ "bfla",
1023
+ "bola",
1024
+ "rbac"
1025
+ ],
1026
+ strategies: []
1027
+ },
1028
+ "nist:ai:measure:2.9": {
1029
+ plugins: ["excessive-agency"],
1030
+ strategies: []
1031
+ },
1032
+ "nist:ai:measure:2.10": {
1033
+ plugins: [
1034
+ "harmful:privacy",
1035
+ "pii:api-db",
1036
+ "pii:direct",
1037
+ "pii:session",
1038
+ "pii:social"
1039
+ ],
1040
+ strategies: []
1041
+ },
1042
+ "nist:ai:measure:2.11": {
1043
+ plugins: [
1044
+ "harmful:harassment-bullying",
1045
+ "harmful:hate",
1046
+ "harmful:insults"
1047
+ ],
1048
+ strategies: []
1049
+ },
1050
+ "nist:ai:measure:2.12": {
1051
+ plugins: [],
1052
+ strategies: []
1053
+ },
1054
+ "nist:ai:measure:2.13": {
1055
+ plugins: ["excessive-agency"],
1056
+ strategies: []
1057
+ },
1058
+ "nist:ai:measure:3.1": {
1059
+ plugins: ["excessive-agency", "harmful:misinformation-disinformation"],
1060
+ strategies: ["jailbreak", "jailbreak-templates"]
1061
+ },
1062
+ "nist:ai:measure:3.2": {
1063
+ plugins: ["excessive-agency"],
1064
+ strategies: []
1065
+ },
1066
+ "nist:ai:measure:3.3": {
1067
+ plugins: ["excessive-agency"],
1068
+ strategies: []
1069
+ },
1070
+ "nist:ai:measure:4.1": {
1071
+ plugins: ["excessive-agency"],
1072
+ strategies: []
1073
+ },
1074
+ "nist:ai:measure:4.2": {
1075
+ plugins: ["excessive-agency", "harmful:misinformation-disinformation"],
1076
+ strategies: []
1077
+ },
1078
+ "nist:ai:measure:4.3": {
1079
+ plugins: ["excessive-agency"],
1080
+ strategies: []
1081
+ }
1082
+ };
1083
+ const MITRE_ATLAS_MAPPING = {
1084
+ "mitre:atlas:exfiltration": {
1085
+ plugins: [
1086
+ "ascii-smuggling",
1087
+ "harmful:privacy",
1088
+ "indirect-prompt-injection",
1089
+ "pii:api-db",
1090
+ "pii:direct",
1091
+ "pii:session",
1092
+ "pii:social",
1093
+ "prompt-extraction"
1094
+ ],
1095
+ strategies: []
1096
+ },
1097
+ "mitre:atlas:impact": {
1098
+ plugins: [
1099
+ "excessive-agency",
1100
+ "harmful",
1101
+ "hijacking",
1102
+ "imitation"
1103
+ ],
1104
+ strategies: ["crescendo"]
1105
+ },
1106
+ "mitre:atlas:initial-access": {
1107
+ plugins: [
1108
+ "debug-access",
1109
+ "harmful:cybercrime",
1110
+ "shell-injection",
1111
+ "sql-injection",
1112
+ "ssrf"
1113
+ ],
1114
+ strategies: [
1115
+ "base64",
1116
+ "jailbreak",
1117
+ "leetspeak",
1118
+ "jailbreak-templates",
1119
+ "rot13"
1120
+ ]
1121
+ },
1122
+ "mitre:atlas:ml-attack-staging": {
1123
+ plugins: [
1124
+ "ascii-smuggling",
1125
+ "excessive-agency",
1126
+ "hallucination",
1127
+ "indirect-prompt-injection"
1128
+ ],
1129
+ strategies: ["jailbreak", "jailbreak:tree"]
1130
+ },
1131
+ "mitre:atlas:reconnaissance": {
1132
+ plugins: [
1133
+ "competitors",
1134
+ "policy",
1135
+ "prompt-extraction",
1136
+ "rbac"
1137
+ ],
1138
+ strategies: []
1139
+ },
1140
+ "mitre:atlas:resource-development": {
1141
+ plugins: [
1142
+ "harmful:cybercrime",
1143
+ "harmful:illegal-drugs",
1144
+ "harmful:indiscriminate-weapons"
1145
+ ],
1146
+ strategies: []
1147
+ }
1148
+ };
1149
+ /**
1150
+ * EU Artificial Intelligence Act
1151
+ * ▸ Art. 5 (Prohibited AI practices) – unacceptable-risk
1152
+ * ▸ Annex III (High-risk AI systems, Art. 6(2)) – high-risk
1153
+ *
1154
+ * Sources:
1155
+ * * Art. 5 list of prohibitions [oai_citation:0‡Artificial Intelligence Act](https://artificialintelligenceact.eu/article/5/?utm_source=chatgpt.com)
1156
+ * * Annex III high-risk categories [oai_citation:1‡Lexology](https://www.lexology.com/library/detail.aspx?g=ec2aab25-67aa-4635-87a0-fc43d9fd1f51&utm_source=chatgpt.com)
1157
+ */
1158
+ const EU_AI_ACT_MAPPING = {
1159
+ "eu:ai-act:art5:subliminal-manipulation": {
1160
+ plugins: [
1161
+ "hijacking",
1162
+ "intent",
1163
+ "excessive-agency",
1164
+ "harmful:misinformation-disinformation"
1165
+ ],
1166
+ strategies: [
1167
+ "jailbreak",
1168
+ "jailbreak:tree",
1169
+ "jailbreak:composite",
1170
+ "jailbreak-templates"
1171
+ ]
1172
+ },
1173
+ "eu:ai-act:art5:exploitation-of-vulnerabilities": {
1174
+ plugins: [
1175
+ "hijacking",
1176
+ "imitation",
1177
+ "harmful:misinformation-disinformation",
1178
+ "bias:age",
1179
+ "bias:disability"
1180
+ ],
1181
+ strategies: ["jailbreak"]
1182
+ },
1183
+ "eu:ai-act:art5:social-scoring": {
1184
+ plugins: [
1185
+ "harmful:misinformation-disinformation",
1186
+ "overreliance",
1187
+ "bias:age",
1188
+ "bias:gender",
1189
+ "bias:race",
1190
+ "bias:disability"
1191
+ ],
1192
+ strategies: []
1193
+ },
1194
+ "eu:ai-act:art5:biometric-categorisation": {
1195
+ plugins: [
1196
+ "pii:direct",
1197
+ "pii:session",
1198
+ "harmful:privacy"
1199
+ ],
1200
+ strategies: []
1201
+ },
1202
+ "eu:ai-act:art5:remote-biometric-id-live": {
1203
+ plugins: [
1204
+ "pii:session",
1205
+ "pii:direct",
1206
+ "harmful:privacy"
1207
+ ],
1208
+ strategies: []
1209
+ },
1210
+ "eu:ai-act:art5:remote-biometric-id-post": {
1211
+ plugins: [
1212
+ "pii:api-db",
1213
+ "pii:direct",
1214
+ "harmful:privacy"
1215
+ ],
1216
+ strategies: []
1217
+ },
1218
+ "eu:ai-act:annex3:biometric-id": {
1219
+ plugins: [
1220
+ "pii:direct",
1221
+ "pii:session",
1222
+ "harmful:privacy"
1223
+ ],
1224
+ strategies: []
1225
+ },
1226
+ "eu:ai-act:annex3:critical-infrastructure": {
1227
+ plugins: [
1228
+ "shell-injection",
1229
+ "sql-injection",
1230
+ "ssrf",
1231
+ "excessive-agency"
1232
+ ],
1233
+ strategies: ["jailbreak", "jailbreak-templates"]
1234
+ },
1235
+ "eu:ai-act:annex3:education": {
1236
+ plugins: [
1237
+ "harmful:misinformation-disinformation",
1238
+ "hallucination",
1239
+ "overreliance",
1240
+ "bias:race",
1241
+ "bias:gender",
1242
+ "bias:disability"
1243
+ ],
1244
+ strategies: []
1245
+ },
1246
+ "eu:ai-act:annex3:employment": {
1247
+ plugins: [
1248
+ "imitation",
1249
+ "pii:direct",
1250
+ "overreliance",
1251
+ "bias:gender",
1252
+ "bias:race",
1253
+ "bias:age",
1254
+ "bias:disability"
1255
+ ],
1256
+ strategies: []
1257
+ },
1258
+ "eu:ai-act:annex3:essential-services": {
1259
+ plugins: [
1260
+ "pii:direct",
1261
+ "pii:session",
1262
+ "excessive-agency",
1263
+ "bias:race",
1264
+ "bias:gender"
1265
+ ],
1266
+ strategies: []
1267
+ },
1268
+ "eu:ai-act:annex3:law-enforcement": {
1269
+ plugins: [
1270
+ "pii:direct",
1271
+ "pii:api-db",
1272
+ "harmful:privacy",
1273
+ "bias:race"
1274
+ ],
1275
+ strategies: []
1276
+ },
1277
+ "eu:ai-act:annex3:migration-border": {
1278
+ plugins: [
1279
+ "pii:direct",
1280
+ "harmful:hate",
1281
+ "harmful:privacy",
1282
+ "bias:race"
1283
+ ],
1284
+ strategies: []
1285
+ },
1286
+ "eu:ai-act:annex3:justice-democracy": {
1287
+ plugins: [
1288
+ "hallucination",
1289
+ "harmful:misinformation-disinformation",
1290
+ "pii:direct",
1291
+ "bias:race",
1292
+ "bias:gender"
1293
+ ],
1294
+ strategies: []
1295
+ }
1296
+ };
1297
+ /**
1298
+ * ISO/IEC 42001 – AI Management System (AIMS) framework risk areas
1299
+ * Covers key risk domains: human oversight, fairness, privacy, robustness, security, ethics, transparency.
1300
+ */
1301
+ const ISO_42001_MAPPING = {
1302
+ "iso:42001:accountability": {
1303
+ plugins: [
1304
+ "excessive-agency",
1305
+ "overreliance",
1306
+ "hijacking"
1307
+ ],
1308
+ strategies: []
1309
+ },
1310
+ "iso:42001:fairness": {
1311
+ plugins: [
1312
+ "bias:age",
1313
+ "bias:disability",
1314
+ "bias:gender",
1315
+ "bias:race",
1316
+ "harmful:hate"
1317
+ ],
1318
+ strategies: []
1319
+ },
1320
+ "iso:42001:privacy": {
1321
+ plugins: [
1322
+ "harmful:privacy",
1323
+ "pii:api-db",
1324
+ "pii:direct",
1325
+ "pii:session",
1326
+ "pii:social"
1327
+ ],
1328
+ strategies: []
1329
+ },
1330
+ "iso:42001:robustness": {
1331
+ plugins: ["ascii-smuggling", "prompt-extraction"],
1332
+ strategies: [
1333
+ "jailbreak",
1334
+ "jailbreak:composite",
1335
+ "jailbreak:tree"
1336
+ ]
1337
+ },
1338
+ "iso:42001:security": {
1339
+ plugins: [
1340
+ "shell-injection",
1341
+ "sql-injection",
1342
+ "ssrf",
1343
+ "debug-access"
1344
+ ],
1345
+ strategies: [
1346
+ "jailbreak",
1347
+ "jailbreak:composite",
1348
+ "base64",
1349
+ "rot13"
1350
+ ]
1351
+ },
1352
+ "iso:42001:safety": {
1353
+ plugins: [
1354
+ "harmful:chemical-biological-weapons",
1355
+ "harmful:child-exploitation",
1356
+ "harmful:violent-crime",
1357
+ "harmful:cybercrime",
1358
+ "harmful:cybercrime:malicious-code"
1359
+ ],
1360
+ strategies: [
1361
+ "jailbreak",
1362
+ "jailbreak:composite",
1363
+ "jailbreak:tree"
1364
+ ]
1365
+ },
1366
+ "iso:42001:transparency": {
1367
+ plugins: [
1368
+ "harmful:misinformation-disinformation",
1369
+ "hallucination",
1370
+ "imitation",
1371
+ "unverifiable-claims",
1372
+ "politics",
1373
+ "religion"
1374
+ ],
1375
+ strategies: []
1376
+ }
1377
+ };
1378
+ /**
1379
+ * EU General Data Protection Regulation (GDPR)
1380
+ * Maps key GDPR articles to relevant Promptfoo plugins for testing AI/LLM systems
1381
+ * for compliance with data protection requirements.
1382
+ *
1383
+ * Sources:
1384
+ * * GDPR full text: https://gdpr-info.eu/
1385
+ */
1386
+ const GDPR_MAPPING = {
1387
+ "gdpr:art5": {
1388
+ plugins: [
1389
+ "harmful:privacy",
1390
+ "pii:api-db",
1391
+ "pii:direct",
1392
+ "pii:session",
1393
+ "pii:social",
1394
+ "hallucination",
1395
+ "harmful:misinformation-disinformation"
1396
+ ],
1397
+ strategies: []
1398
+ },
1399
+ "gdpr:art9": {
1400
+ plugins: [
1401
+ "pii:direct",
1402
+ "pii:social",
1403
+ "harmful:privacy",
1404
+ "bias:age",
1405
+ "bias:disability",
1406
+ "bias:gender",
1407
+ "bias:race"
1408
+ ],
1409
+ strategies: []
1410
+ },
1411
+ "gdpr:art15": {
1412
+ plugins: [
1413
+ "pii:api-db",
1414
+ "pii:session",
1415
+ "rbac",
1416
+ "bola",
1417
+ "bfla"
1418
+ ],
1419
+ strategies: []
1420
+ },
1421
+ "gdpr:art17": {
1422
+ plugins: [
1423
+ "pii:api-db",
1424
+ "pii:direct",
1425
+ "pii:session",
1426
+ "harmful:privacy",
1427
+ "cross-session-leak"
1428
+ ],
1429
+ strategies: []
1430
+ },
1431
+ "gdpr:art22": {
1432
+ plugins: [
1433
+ "bias:age",
1434
+ "bias:disability",
1435
+ "bias:gender",
1436
+ "bias:race",
1437
+ "harmful:hate",
1438
+ "overreliance",
1439
+ "hallucination"
1440
+ ],
1441
+ strategies: []
1442
+ },
1443
+ "gdpr:art25": {
1444
+ plugins: [
1445
+ "harmful:privacy",
1446
+ "pii:api-db",
1447
+ "pii:direct",
1448
+ "pii:session",
1449
+ "pii:social",
1450
+ "prompt-extraction"
1451
+ ],
1452
+ strategies: []
1453
+ },
1454
+ "gdpr:art32": {
1455
+ plugins: [
1456
+ "shell-injection",
1457
+ "sql-injection",
1458
+ "ssrf",
1459
+ "debug-access",
1460
+ "harmful:cybercrime",
1461
+ "rbac",
1462
+ "bfla",
1463
+ "bola"
1464
+ ],
1465
+ strategies: []
1466
+ }
1467
+ };
1468
+ /**
1469
+ * U.S. Department of Defense (DoD) AI ethical principles.
1470
+ *
1471
+ * Source:
1472
+ * * https://www.defense.gov/News/News-Stories/Article/Article/2094085/dod-adopts-5-principles-of-artificial-intelligence-ethics/
1473
+ */
1474
+ const DOD_AI_ETHICS_MAPPING = {
1475
+ "dod:ai:ethics:01": {
1476
+ plugins: [
1477
+ "excessive-agency",
1478
+ "goal-misalignment",
1479
+ "overreliance",
1480
+ "hijacking"
1481
+ ],
1482
+ strategies: ["jailbreak", "jailbreak-templates"]
1483
+ },
1484
+ "dod:ai:ethics:02": {
1485
+ plugins: [
1486
+ "bias:age",
1487
+ "bias:disability",
1488
+ "bias:gender",
1489
+ "bias:race",
1490
+ "harmful:hate"
1491
+ ],
1492
+ strategies: []
1493
+ },
1494
+ "dod:ai:ethics:03": {
1495
+ plugins: [
1496
+ "hallucination",
1497
+ "harmful:misinformation-disinformation",
1498
+ "rag-source-attribution",
1499
+ "unverifiable-claims"
1500
+ ],
1501
+ strategies: []
1502
+ },
1503
+ "dod:ai:ethics:04": {
1504
+ plugins: [
1505
+ "harmful:misinformation-disinformation",
1506
+ "harmful:unsafe-practices",
1507
+ "shell-injection",
1508
+ "sql-injection",
1509
+ "ssrf",
1510
+ "debug-access",
1511
+ "reasoning-dos"
1512
+ ],
1513
+ strategies: ["jailbreak", "jailbreak-templates"]
1514
+ },
1515
+ "dod:ai:ethics:05": {
1516
+ plugins: [
1517
+ "excessive-agency",
1518
+ "hijacking",
1519
+ "indirect-prompt-injection",
1520
+ "system-prompt-override",
1521
+ "rbac",
1522
+ "bfla",
1523
+ "bola",
1524
+ "tool-discovery"
1525
+ ],
1526
+ strategies: [
1527
+ "jailbreak",
1528
+ "jailbreak-templates",
1529
+ "jailbreak:composite"
1530
+ ]
1531
+ }
1532
+ };
1533
+ const ALIASED_PLUGINS = [
1534
+ "dod:ai:ethics",
1535
+ "mitre:atlas",
1536
+ "nist:ai",
1537
+ "nist:ai:measure",
1538
+ "owasp:api",
1539
+ "owasp:llm",
1540
+ "owasp:llm:redteam:model",
1541
+ "owasp:llm:redteam:implementation",
1542
+ "owasp:llm:redteam:system",
1543
+ "owasp:llm:redteam:runtime",
1544
+ "owasp:agentic",
1545
+ "toxicity",
1546
+ "bias",
1547
+ "misinformation",
1548
+ "illegal-activity",
1549
+ "personal-safety",
1550
+ "tool-discovery:multi-turn",
1551
+ "eu:ai-act",
1552
+ "iso:42001",
1553
+ "gdpr",
1554
+ ...Object.keys(MITRE_ATLAS_MAPPING),
1555
+ ...Object.keys(NIST_AI_RMF_MAPPING),
1556
+ ...Object.keys(OWASP_API_TOP_10_MAPPING),
1557
+ ...Object.keys(OWASP_LLM_TOP_10_MAPPING),
1558
+ ...Object.keys(OWASP_AGENTIC_TOP_10_MAPPING),
1559
+ ...Object.keys(EU_AI_ACT_MAPPING),
1560
+ ...Object.keys(ISO_42001_MAPPING),
1561
+ ...Object.keys(GDPR_MAPPING),
1562
+ ...Object.keys(DOD_AI_ETHICS_MAPPING)
1563
+ ];
1564
+ const ALIASED_PLUGIN_MAPPINGS = {
1565
+ "dod:ai:ethics": DOD_AI_ETHICS_MAPPING,
1566
+ "mitre:atlas": MITRE_ATLAS_MAPPING,
1567
+ "nist:ai:measure": NIST_AI_RMF_MAPPING,
1568
+ "owasp:api": OWASP_API_TOP_10_MAPPING,
1569
+ "owasp:llm": OWASP_LLM_TOP_10_MAPPING,
1570
+ "owasp:llm:redteam": OWASP_LLM_RED_TEAM_MAPPING,
1571
+ "owasp:agentic": OWASP_AGENTIC_TOP_10_MAPPING,
1572
+ "eu:ai-act": EU_AI_ACT_MAPPING,
1573
+ "iso:42001": ISO_42001_MAPPING,
1574
+ gdpr: GDPR_MAPPING,
1575
+ "tool-discovery:multi-turn": { "tool-discovery:multi-turn": {
1576
+ plugins: ["tool-discovery"],
1577
+ strategies: []
1578
+ } },
1579
+ toxicity: { toxicity: {
1580
+ plugins: [
1581
+ "harmful:hate",
1582
+ "harmful:harassment-bullying",
1583
+ "harmful:insults",
1584
+ "harmful:profanity",
1585
+ "harmful:graphic-content",
1586
+ "harmful:sexual-content"
1587
+ ],
1588
+ strategies: []
1589
+ } },
1590
+ bias: { bias: {
1591
+ plugins: [
1592
+ "politics",
1593
+ "religion",
1594
+ "bias:age",
1595
+ "bias:disability",
1596
+ "bias:gender",
1597
+ "bias:race"
1598
+ ],
1599
+ strategies: []
1600
+ } },
1601
+ misinformation: { misinformation: {
1602
+ plugins: [
1603
+ "harmful:misinformation-disinformation",
1604
+ "hallucination",
1605
+ "harmful:radicalization",
1606
+ "imitation"
1607
+ ],
1608
+ strategies: []
1609
+ } },
1610
+ "illegal-activity": { "illegal-activity": {
1611
+ plugins: [
1612
+ "harmful:violent-crime",
1613
+ "harmful:non-violent-crime",
1614
+ "harmful:sex-crime",
1615
+ "harmful:cybercrime",
1616
+ "harmful:illegal-activities",
1617
+ "harmful:illegal-drugs",
1618
+ "harmful:illegal-drugs:meth",
1619
+ "harmful:chemical-biological-weapons",
1620
+ "harmful:indiscriminate-weapons",
1621
+ "harmful:weapons:ied"
1622
+ ],
1623
+ strategies: []
1624
+ } }
1625
+ };
1626
+ //#endregion
1627
+ //#region src/redteam/constants/metadata.ts
1628
+ const Severity = {
1629
+ Critical: "critical",
1630
+ High: "high",
1631
+ Medium: "medium",
1632
+ Low: "low",
1633
+ Informational: "informational"
1634
+ };
1635
+ const SeveritySchema = zod.z.enum([
1636
+ "critical",
1637
+ "high",
1638
+ "medium",
1639
+ "low",
1640
+ "informational"
1641
+ ]);
1642
+ Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
1643
+ Severity.Critical, Severity.High, Severity.Medium, Severity.Low, Severity.Informational;
1644
+ const riskCategorySeverityMap = {
1645
+ ["agentic:memory-poisoning"]: Severity.High,
1646
+ aegis: Severity.Medium,
1647
+ "ascii-smuggling": Severity.Low,
1648
+ beavertails: Severity.Low,
1649
+ bfla: Severity.High,
1650
+ bola: Severity.High,
1651
+ cca: Severity.High,
1652
+ ferpa: Severity.Medium,
1653
+ "financial:calculation-error": Severity.Low,
1654
+ "financial:compliance-violation": Severity.Medium,
1655
+ "financial:confidential-disclosure": Severity.High,
1656
+ "financial:counterfactual": Severity.Medium,
1657
+ "financial:data-leakage": Severity.Medium,
1658
+ "financial:defamation": Severity.Medium,
1659
+ "financial:hallucination": Severity.Low,
1660
+ "financial:impartiality": Severity.Medium,
1661
+ "financial:japan-fiea-suitability": Severity.High,
1662
+ "financial:misconduct": Severity.High,
1663
+ "financial:sox-compliance": Severity.High,
1664
+ "financial:sycophancy": Severity.Low,
1665
+ "goal-misalignment": Severity.Low,
1666
+ competitors: Severity.Low,
1667
+ contracts: Severity.Medium,
1668
+ coppa: Severity.High,
1669
+ "cross-session-leak": Severity.Medium,
1670
+ cyberseceval: Severity.Medium,
1671
+ "data-exfil": Severity.High,
1672
+ donotanswer: Severity.Medium,
1673
+ "debug-access": Severity.High,
1674
+ default: Severity.Low,
1675
+ mcp: Severity.High,
1676
+ "medical:anchoring-bias": Severity.High,
1677
+ "medical:hallucination": Severity.Critical,
1678
+ "medical:incorrect-knowledge": Severity.Critical,
1679
+ "medical:off-label-use": Severity.High,
1680
+ "medical:prioritization-error": Severity.High,
1681
+ "medical:sycophancy": Severity.Medium,
1682
+ "off-topic": Severity.Medium,
1683
+ "divergent-repetition": Severity.Medium,
1684
+ "excessive-agency": Severity.Medium,
1685
+ "model-identification": Severity.Medium,
1686
+ "tool-discovery": Severity.Low,
1687
+ foundation: Severity.Medium,
1688
+ "guardrails-eval": Severity.Medium,
1689
+ hallucination: Severity.Medium,
1690
+ harmbench: Severity.Medium,
1691
+ "toxic-chat": Severity.Medium,
1692
+ harmful: Severity.Medium,
1693
+ "bias:age": Severity.Low,
1694
+ "bias:disability": Severity.Low,
1695
+ "bias:gender": Severity.Low,
1696
+ "bias:race": Severity.Low,
1697
+ bias: Severity.Low,
1698
+ medical: Severity.High,
1699
+ pharmacy: Severity.High,
1700
+ insurance: Severity.High,
1701
+ financial: Severity.High,
1702
+ ecommerce: Severity.High,
1703
+ "harmful:chemical-biological-weapons": Severity.High,
1704
+ "harmful:child-exploitation": Severity.Critical,
1705
+ "harmful:copyright-violations": Severity.Low,
1706
+ "harmful:cybercrime": Severity.Low,
1707
+ "harmful:cybercrime:malicious-code": Severity.Low,
1708
+ "harmful:graphic-content": Severity.Medium,
1709
+ "harmful:harassment-bullying": Severity.Low,
1710
+ "harmful:hate": Severity.Critical,
1711
+ "harmful:illegal-activities": Severity.Medium,
1712
+ "harmful:illegal-drugs": Severity.Medium,
1713
+ "harmful:illegal-drugs:meth": Severity.Low,
1714
+ "harmful:indiscriminate-weapons": Severity.Medium,
1715
+ "harmful:insults": Severity.Low,
1716
+ "harmful:intellectual-property": Severity.Medium,
1717
+ "harmful:misinformation-disinformation": Severity.Medium,
1718
+ "harmful:non-violent-crime": Severity.Medium,
1719
+ "harmful:privacy": Severity.High,
1720
+ "harmful:profanity": Severity.Low,
1721
+ "harmful:radicalization": Severity.Low,
1722
+ "harmful:self-harm": Severity.Critical,
1723
+ "harmful:sex-crime": Severity.High,
1724
+ "harmful:sexual-content": Severity.Medium,
1725
+ "harmful:specialized-advice": Severity.Medium,
1726
+ "harmful:unsafe-practices": Severity.Low,
1727
+ "harmful:violent-crime": Severity.High,
1728
+ "harmful:weapons:ied": Severity.Low,
1729
+ hijacking: Severity.High,
1730
+ imitation: Severity.Low,
1731
+ "indirect-prompt-injection": Severity.High,
1732
+ "insurance:coverage-discrimination": Severity.Critical,
1733
+ "insurance:data-disclosure": Severity.Critical,
1734
+ "insurance:network-misinformation": Severity.High,
1735
+ "insurance:phi-disclosure": Severity.Critical,
1736
+ "ecommerce:pci-dss": Severity.Critical,
1737
+ "ecommerce:compliance-bypass": Severity.High,
1738
+ "ecommerce:order-fraud": Severity.High,
1739
+ "ecommerce:price-manipulation": Severity.High,
1740
+ telecom: Severity.Critical,
1741
+ "telecom:cpni-disclosure": Severity.Critical,
1742
+ "telecom:location-disclosure": Severity.Critical,
1743
+ "telecom:account-takeover": Severity.Critical,
1744
+ "telecom:e911-misinformation": Severity.Critical,
1745
+ "telecom:tcpa-violation": Severity.High,
1746
+ "telecom:unauthorized-changes": Severity.High,
1747
+ "telecom:fraud-enablement": Severity.High,
1748
+ "telecom:porting-misinformation": Severity.High,
1749
+ "telecom:billing-misinformation": Severity.Medium,
1750
+ "telecom:coverage-misinformation": Severity.Medium,
1751
+ "telecom:law-enforcement-request-handling": Severity.Medium,
1752
+ "telecom:accessibility-violation": Severity.Medium,
1753
+ realestate: Severity.Critical,
1754
+ "realestate:fair-housing-discrimination": Severity.Critical,
1755
+ "realestate:steering": Severity.Critical,
1756
+ "realestate:discriminatory-listings": Severity.High,
1757
+ "realestate:lending-discrimination": Severity.Critical,
1758
+ "realestate:valuation-bias": Severity.High,
1759
+ "realestate:accessibility-discrimination": Severity.High,
1760
+ "realestate:advertising-discrimination": Severity.High,
1761
+ "realestate:source-of-income": Severity.High,
1762
+ intent: Severity.High,
1763
+ overreliance: Severity.Low,
1764
+ "pharmacy:controlled-substance-compliance": Severity.High,
1765
+ "pharmacy:dosage-calculation": Severity.Critical,
1766
+ "pharmacy:drug-interaction": Severity.Critical,
1767
+ pii: Severity.High,
1768
+ "pii:api-db": Severity.High,
1769
+ "pii:direct": Severity.High,
1770
+ "pii:session": Severity.High,
1771
+ "pii:social": Severity.High,
1772
+ pliny: Severity.Medium,
1773
+ policy: Severity.High,
1774
+ politics: Severity.Low,
1775
+ "prompt-extraction": Severity.Medium,
1776
+ "rag-document-exfiltration": Severity.Medium,
1777
+ "rag-poisoning": Severity.Medium,
1778
+ "rag-source-attribution": Severity.High,
1779
+ rbac: Severity.High,
1780
+ "reasoning-dos": Severity.Low,
1781
+ religion: Severity.Low,
1782
+ "shell-injection": Severity.High,
1783
+ "special-token-injection": Severity.Medium,
1784
+ "sql-injection": Severity.High,
1785
+ ssrf: Severity.High,
1786
+ "system-prompt-override": Severity.High,
1787
+ unsafebench: Severity.Medium,
1788
+ "unverifiable-claims": Severity.Medium,
1789
+ vlguard: Severity.Medium,
1790
+ vlsu: Severity.Medium,
1791
+ wordplay: Severity.Low,
1792
+ xstest: Severity.Low
1793
+ };
1794
+ Object.entries({
1795
+ "Security & Access Control": [
1796
+ "ascii-smuggling",
1797
+ "bfla",
1798
+ "bola",
1799
+ "cca",
1800
+ "debug-access",
1801
+ "model-identification",
1802
+ "hijacking",
1803
+ "indirect-prompt-injection",
1804
+ "rbac",
1805
+ "reasoning-dos",
1806
+ "shell-injection",
1807
+ "special-token-injection",
1808
+ "sql-injection",
1809
+ "ssrf",
1810
+ "system-prompt-override",
1811
+ "tool-discovery",
1812
+ "mcp",
1813
+ "cross-session-leak",
1814
+ "data-exfil",
1815
+ "divergent-repetition",
1816
+ "harmful:privacy",
1817
+ "insurance:data-disclosure",
1818
+ "insurance:phi-disclosure",
1819
+ "pii:api-db",
1820
+ "pii:direct",
1821
+ "pii:session",
1822
+ "pii:social",
1823
+ "pii",
1824
+ "prompt-extraction",
1825
+ "rag-document-exfiltration",
1826
+ "rag-poisoning",
1827
+ "rag-source-attribution",
1828
+ "agentic:memory-poisoning"
1829
+ ],
1830
+ "Compliance & Legal": [
1831
+ "contracts",
1832
+ "coppa",
1833
+ "ferpa",
1834
+ "harmful:chemical-biological-weapons",
1835
+ "harmful:copyright-violations",
1836
+ "harmful:cybercrime:malicious-code",
1837
+ "harmful:cybercrime",
1838
+ "harmful:illegal-activities",
1839
+ "harmful:illegal-drugs:meth",
1840
+ "harmful:illegal-drugs",
1841
+ "harmful:indiscriminate-weapons",
1842
+ "harmful:intellectual-property",
1843
+ "harmful:non-violent-crime",
1844
+ "harmful:sex-crime",
1845
+ "harmful:specialized-advice",
1846
+ "harmful:unsafe-practices",
1847
+ "harmful:violent-crime",
1848
+ "harmful:weapons:ied",
1849
+ "insurance:coverage-discrimination",
1850
+ "insurance:network-misinformation"
1851
+ ],
1852
+ "Trust & Safety": [
1853
+ "bias:age",
1854
+ "bias:disability",
1855
+ "bias:gender",
1856
+ "bias:race",
1857
+ "harmful:child-exploitation",
1858
+ "harmful:graphic-content",
1859
+ "harmful:harassment-bullying",
1860
+ "harmful:hate",
1861
+ "harmful:insults",
1862
+ "harmful:profanity",
1863
+ "harmful:radicalization",
1864
+ "harmful:self-harm",
1865
+ "harmful:sexual-content",
1866
+ "wordplay"
1867
+ ],
1868
+ Brand: [
1869
+ "competitors",
1870
+ "excessive-agency",
1871
+ "goal-misalignment",
1872
+ "hallucination",
1873
+ "harmful:misinformation-disinformation",
1874
+ "hijacking",
1875
+ "imitation",
1876
+ "intent",
1877
+ "off-topic",
1878
+ "overreliance",
1879
+ "policy",
1880
+ "politics",
1881
+ "religion",
1882
+ "unverifiable-claims"
1883
+ ],
1884
+ "Domain-Specific Risks": [
1885
+ "ecommerce:pci-dss",
1886
+ "ecommerce:compliance-bypass",
1887
+ "ecommerce:order-fraud",
1888
+ "ecommerce:price-manipulation",
1889
+ "financial:calculation-error",
1890
+ "financial:compliance-violation",
1891
+ "financial:confidential-disclosure",
1892
+ "financial:counterfactual",
1893
+ "financial:data-leakage",
1894
+ "financial:defamation",
1895
+ "financial:hallucination",
1896
+ "financial:impartiality",
1897
+ "financial:japan-fiea-suitability",
1898
+ "financial:misconduct",
1899
+ "financial:sox-compliance",
1900
+ "financial:sycophancy",
1901
+ "medical:hallucination",
1902
+ "medical:anchoring-bias",
1903
+ "medical:incorrect-knowledge",
1904
+ "medical:off-label-use",
1905
+ "medical:prioritization-error",
1906
+ "medical:sycophancy",
1907
+ "pharmacy:controlled-substance-compliance",
1908
+ "pharmacy:dosage-calculation",
1909
+ "pharmacy:drug-interaction",
1910
+ "telecom:cpni-disclosure",
1911
+ "telecom:location-disclosure",
1912
+ "telecom:account-takeover",
1913
+ "telecom:e911-misinformation",
1914
+ "telecom:tcpa-violation",
1915
+ "telecom:unauthorized-changes",
1916
+ "telecom:fraud-enablement",
1917
+ "telecom:porting-misinformation",
1918
+ "telecom:billing-misinformation",
1919
+ "telecom:coverage-misinformation",
1920
+ "telecom:law-enforcement-request-handling",
1921
+ "telecom:accessibility-violation",
1922
+ "realestate:fair-housing-discrimination",
1923
+ "realestate:steering",
1924
+ "realestate:discriminatory-listings",
1925
+ "realestate:lending-discrimination",
1926
+ "realestate:valuation-bias",
1927
+ "realestate:accessibility-discrimination",
1928
+ "realestate:advertising-discrimination",
1929
+ "realestate:source-of-income"
1930
+ ],
1931
+ Datasets: [
1932
+ "aegis",
1933
+ "beavertails",
1934
+ "cyberseceval",
1935
+ "donotanswer",
1936
+ "harmbench",
1937
+ "toxic-chat",
1938
+ "pliny",
1939
+ "unsafebench",
1940
+ "vlguard",
1941
+ "vlsu",
1942
+ "xstest"
1943
+ ]
1944
+ }).reduce((acc, [category, harms]) => {
1945
+ harms.forEach((harm) => {
1946
+ acc[harm] = category;
1947
+ });
1948
+ return acc;
1949
+ }, {});
1950
+ const categoryAliases = {
1951
+ ["agentic:memory-poisoning"]: "AgenticMemoryPoisoning",
1952
+ aegis: "Aegis",
1953
+ "ascii-smuggling": "AsciiSmuggling",
1954
+ beavertails: "BeaverTails",
1955
+ bfla: "BFLAEnforcement",
1956
+ bola: "BOLAEnforcement",
1957
+ cca: "CCAEnforcement",
1958
+ competitors: "CompetitorEndorsement",
1959
+ contracts: "ContractualCommitment",
1960
+ coppa: "COPPACompliance",
1961
+ "cross-session-leak": "CrossSessionLeak",
1962
+ cyberseceval: "CyberSecEval",
1963
+ "data-exfil": "DataExfil",
1964
+ donotanswer: "DoNotAnswer",
1965
+ "debug-access": "DebugAccess",
1966
+ default: "Default",
1967
+ ferpa: "FERPACompliance",
1968
+ mcp: "MCP",
1969
+ "medical:anchoring-bias": "MedicalAnchoringBias",
1970
+ "medical:hallucination": "Medical Hallucination",
1971
+ "medical:incorrect-knowledge": "MedicalIncorrectKnowledge",
1972
+ "medical:off-label-use": "MedicalOffLabelUse",
1973
+ "medical:prioritization-error": "MedicalPrioritizationError",
1974
+ "medical:sycophancy": "MedicalSycophancy",
1975
+ "ecommerce:compliance-bypass": "EcommerceComplianceBypass",
1976
+ "ecommerce:order-fraud": "EcommerceOrderFraud",
1977
+ "ecommerce:pci-dss": "EcommercePciDss",
1978
+ "ecommerce:price-manipulation": "EcommercePriceManipulation",
1979
+ "financial:calculation-error": "FinancialCalculationError",
1980
+ "financial:compliance-violation": "FinancialComplianceViolation",
1981
+ "financial:confidential-disclosure": "FinancialConfidentialDisclosure",
1982
+ "financial:counterfactual": "FinancialCounterfactual",
1983
+ "financial:data-leakage": "FinancialDataLeakage",
1984
+ "financial:defamation": "FinancialDefamation",
1985
+ "financial:hallucination": "FinancialHallucination",
1986
+ "financial:impartiality": "FinancialImpartiality",
1987
+ "financial:japan-fiea-suitability": "FinancialJapanFieaSuitability",
1988
+ "financial:misconduct": "FinancialMisconduct",
1989
+ "financial:sox-compliance": "FinancialSoxCompliance",
1990
+ "financial:sycophancy": "FinancialSycophancy",
1991
+ "goal-misalignment": "GoalMisalignment",
1992
+ "off-topic": "OffTopic",
1993
+ "pharmacy:controlled-substance-compliance": "PharmacyControlledSubstanceCompliance",
1994
+ "pharmacy:dosage-calculation": "PharmacyDosageCalculation",
1995
+ "pharmacy:drug-interaction": "PharmacyDrugInteraction",
1996
+ "divergent-repetition": "DivergentRepetition",
1997
+ "excessive-agency": "ExcessiveAgency",
1998
+ "model-identification": "ModelIdentification",
1999
+ "tool-discovery": "ToolDiscovery",
2000
+ foundation: "Foundation",
2001
+ "guardrails-eval": "GuardrailsEvaluation",
2002
+ hallucination: "Hallucination",
2003
+ harmbench: "Harmbench",
2004
+ "toxic-chat": "ToxicChat",
2005
+ harmful: "Harmful",
2006
+ "bias:age": "Age Bias",
2007
+ "bias:disability": "Disability Bias",
2008
+ "bias:gender": "Gender Bias",
2009
+ "bias:race": "Race Bias",
2010
+ bias: "Bias Detection",
2011
+ medical: "Medical Safety",
2012
+ pharmacy: "Pharmacy Safety",
2013
+ insurance: "Insurance Safety",
2014
+ financial: "Financial Safety",
2015
+ ecommerce: "E-commerce Safety",
2016
+ telecom: "Telecommunications Safety",
2017
+ "telecom:cpni-disclosure": "TelecomCpniDisclosure",
2018
+ "telecom:location-disclosure": "TelecomLocationDisclosure",
2019
+ "telecom:account-takeover": "TelecomAccountTakeover",
2020
+ "telecom:e911-misinformation": "TelecomE911Misinformation",
2021
+ "telecom:tcpa-violation": "TelecomTcpaViolation",
2022
+ "telecom:unauthorized-changes": "TelecomUnauthorizedChanges",
2023
+ "telecom:fraud-enablement": "TelecomFraudEnablement",
2024
+ "telecom:porting-misinformation": "TelecomPortingMisinformation",
2025
+ "telecom:billing-misinformation": "TelecomBillingMisinformation",
2026
+ "telecom:coverage-misinformation": "TelecomCoverageMisinformation",
2027
+ "telecom:law-enforcement-request-handling": "TelecomLawEnforcementRequestHandling",
2028
+ "telecom:accessibility-violation": "TelecomAccessibilityViolation",
2029
+ realestate: "Real Estate Safety",
2030
+ "realestate:fair-housing-discrimination": "RealEstateFairHousingDiscrimination",
2031
+ "realestate:steering": "RealEstateSteering",
2032
+ "realestate:discriminatory-listings": "RealEstateDiscriminatoryListings",
2033
+ "realestate:lending-discrimination": "RealEstateLendingDiscrimination",
2034
+ "realestate:valuation-bias": "RealEstateValuationBias",
2035
+ "realestate:accessibility-discrimination": "RealEstateAccessibilityDiscrimination",
2036
+ "realestate:advertising-discrimination": "RealEstateAdvertisingDiscrimination",
2037
+ "realestate:source-of-income": "RealEstateSourceOfIncome",
2038
+ "harmful:chemical-biological-weapons": "Chemical & Biological Weapons",
2039
+ "harmful:child-exploitation": "Child Exploitation",
2040
+ "harmful:copyright-violations": "Copyright Violations - Copyrighted text",
2041
+ "harmful:cybercrime": "Cybercrime",
2042
+ "harmful:cybercrime:malicious-code": "Malicious Code",
2043
+ "harmful:graphic-content": "Graphic Content",
2044
+ "harmful:harassment-bullying": "Harassment",
2045
+ "harmful:hate": "Hate",
2046
+ "harmful:illegal-activities": "Illegal Activities - Fraud & scams",
2047
+ "harmful:illegal-drugs": "Illegal Drugs",
2048
+ "harmful:illegal-drugs:meth": "Methamphetamine",
2049
+ "harmful:indiscriminate-weapons": "Indiscriminate Weapons",
2050
+ "harmful:insults": "Insults and personal attacks",
2051
+ "harmful:intellectual-property": "Intellectual Property violation",
2052
+ "harmful:misinformation-disinformation": "Misinformation & Disinformation - Harmful lies and propaganda",
2053
+ "harmful:non-violent-crime": "Non-Violent Crimes",
2054
+ "harmful:privacy": "Privacy violations",
2055
+ "harmful:profanity": "Requests containing profanity",
2056
+ "harmful:radicalization": "Radicalization",
2057
+ "harmful:self-harm": "Self-Harm",
2058
+ "harmful:sex-crime": "Sex Crimes",
2059
+ "harmful:sexual-content": "Sexual Content",
2060
+ "harmful:specialized-advice": "Specialized Advice - Financial",
2061
+ "harmful:unsafe-practices": "Promotion of unsafe practices",
2062
+ "harmful:violent-crime": "Violent Crimes",
2063
+ "harmful:weapons:ied": "Improvised Explosive Devices",
2064
+ hijacking: "Hijacking",
2065
+ imitation: "Imitation",
2066
+ "indirect-prompt-injection": "Indirect Prompt Injection",
2067
+ "insurance:coverage-discrimination": "InsuranceCoverageDiscrimination",
2068
+ "insurance:data-disclosure": "InsuranceDataDisclosure",
2069
+ "insurance:network-misinformation": "InsuranceNetworkMisinformation",
2070
+ "insurance:phi-disclosure": "InsurancePhiDisclosure",
2071
+ intent: "Intent",
2072
+ overreliance: "Overreliance",
2073
+ pii: "PIILeak",
2074
+ "pii:api-db": "PIILeak",
2075
+ "pii:direct": "PIILeak",
2076
+ "pii:session": "PIILeak",
2077
+ "pii:social": "PIILeak",
2078
+ pliny: "Pliny",
2079
+ policy: "PolicyViolation",
2080
+ politics: "PoliticalStatement",
2081
+ "prompt-extraction": "PromptExtraction",
2082
+ "rag-document-exfiltration": "RAG Document Exfiltration",
2083
+ "rag-poisoning": "RAG Poisoning",
2084
+ "rag-source-attribution": "RAGSourceAttribution",
2085
+ rbac: "RbacEnforcement",
2086
+ "reasoning-dos": "Reasoning DoS",
2087
+ religion: "Religion",
2088
+ "shell-injection": "ShellInjection",
2089
+ "special-token-injection": "SpecialTokenInjection",
2090
+ "sql-injection": "SqlInjection",
2091
+ ssrf: "SSRFEnforcement",
2092
+ "system-prompt-override": "System Prompt Override",
2093
+ unsafebench: "UnsafeBench",
2094
+ "unverifiable-claims": "Unverifiable Claims",
2095
+ vlguard: "VLGuard",
2096
+ vlsu: "VLSU",
2097
+ wordplay: "Wordplay",
2098
+ xstest: "XSTest"
2099
+ };
2100
+ Object.entries(categoryAliases).reduce((acc, [key, value]) => {
2101
+ acc[value] = key;
2102
+ return acc;
2103
+ }, {});
2104
+ const pluginDescriptions = {
2105
+ ["agentic:memory-poisoning"]: "Tests whether an agent is vulnerable to memory poisoning attacks",
2106
+ aegis: "Tests content safety handling using NVIDIA's Aegis dataset",
2107
+ "ascii-smuggling": "Tests for ASCII-based prompt smuggling vulnerabilities",
2108
+ beavertails: "Tests handling of malicious prompts from the BeaverTails dataset",
2109
+ bfla: "Evaluates function-level authorization controls to identify privilege escalation vulnerabilities (OWASP API Security Top 10 #5)",
2110
+ bola: "Tests object-level authorization mechanisms to detect unauthorized data access vulnerabilities (OWASP API Security Top 10 #1)",
2111
+ cca: "Tests for vulnerability to Context Compliance Attacks using fabricated conversation history",
2112
+ competitors: "Assesses system protection against unauthorized competitor endorsements and brand security risks",
2113
+ contracts: "Evaluates safeguards against unauthorized contractual commitments and legal exposure",
2114
+ coppa: "Tests for COPPA (Children's Online Privacy Protection Act) compliance violations, ensuring proper protection of children's data and privacy",
2115
+ "cross-session-leak": "Tests for information leakage vulnerabilities between different user sessions",
2116
+ cyberseceval: "Tests prompt injection attacks from Meta's CyberSecEval dataset",
2117
+ "data-exfil": "Tests whether the AI can be tricked into exfiltrating sensitive data via URL parameters, images, or markdown links",
2118
+ donotanswer: "Tests for vulnerabilities to Do Not Answer attacks",
2119
+ "debug-access": "Identifies exposed debugging interfaces and unauthorized command execution vectors",
2120
+ default: "Executes comprehensive baseline security testing across multiple risk categories",
2121
+ "divergent-repetition": "Tests repetitive patterns that can cause the model to diverge from normal behavior and leak training data",
2122
+ "excessive-agency": "Evaluates system boundary enforcement and unauthorized action prevention",
2123
+ "model-identification": "Tests whether an AI system reveals its underlying model identity, version, or creator when probed",
2124
+ "tool-discovery": "Tests for enumeration of available tools and function calls",
2125
+ ferpa: "Tests for Family Educational Rights and Privacy Act compliance",
2126
+ foundation: "Tests a collection of plugins designed to run against foundation models",
2127
+ hallucination: "Tests system resilience against false information generation and propagation",
2128
+ harmbench: "Tests for harmful content, illegal activities, misinformation using the HarmBench dataset from the Center for AI Safety (CAIS)",
2129
+ "toxic-chat": "Tests handling of toxic user prompts from the ToxicChat dataset",
2130
+ harmful: "Comprehensive suite for testing handling of various categories of malicious content",
2131
+ "bias:age": "Tests handling of age bias in responses, stereotypes, and discrimination",
2132
+ "bias:disability": "Tests handling of disability bias in responses, stereotypes, and discrimination",
2133
+ "bias:gender": "Tests handling of gender bias in responses, stereotypes, and discrimination",
2134
+ "bias:race": "Tests handling of race bias in responses, stereotypes, and discrimination",
2135
+ bias: "Unified bias detection across protected characteristics including age, disability, gender, and race",
2136
+ medical: "Comprehensive medical AI safety testing for healthcare-specific vulnerabilities including hallucination, anchoring bias, incorrect knowledge, off-label use, prioritization errors, and sycophancy",
2137
+ pharmacy: "Comprehensive pharmacy AI safety testing for pharmaceutical vulnerabilities including drug interactions, dosage calculations, and controlled substance compliance",
2138
+ insurance: "Comprehensive insurance AI safety testing across all insurance types including coverage discrimination, network misinformation, PHI disclosure, and policyholder data disclosure",
2139
+ financial: "Comprehensive financial AI safety testing for financial services vulnerabilities including calculation errors, compliance violations, hallucination, and data leakage",
2140
+ ecommerce: "Comprehensive e-commerce AI safety testing for payment security and transaction integrity vulnerabilities including PCI DSS compliance, fraud detection, and price manipulation",
2141
+ telecom: "Comprehensive telecommunications AI safety testing across CPNI protection, account security, regulatory compliance (FCC, TCPA, CALEA), and telecom-specific fraud prevention",
2142
+ "telecom:cpni-disclosure": "Tests for unauthorized disclosure of Customer Proprietary Network Information (CPNI) including call records, service details, and billing information in violation of FCC 47 U.S.C. Section 222",
2143
+ "telecom:location-disclosure": "Tests for unauthorized disclosure of customer location data from cell tower connections, GPS, or network information",
2144
+ "telecom:account-takeover": "Tests for SIM swap vulnerabilities, authentication bypass, and account hijacking that could enable fraud or identity theft",
2145
+ "telecom:e911-misinformation": "Tests for incorrect or unsafe information about emergency calling, location accuracy, VoIP limitations, and E911 regulations (Kari's Law, RAY BAUM's Act)",
2146
+ "telecom:tcpa-violation": "Tests for TCPA violations including improper consent handling, Do Not Call list compliance, and illegal robocall/text facilitation",
2147
+ "telecom:unauthorized-changes": "Tests for slamming (unauthorized carrier switches) and cramming (unauthorized charges) vulnerabilities that violate FCC Section 258",
2148
+ "telecom:fraud-enablement": "Tests for telecom-specific fraud facilitation including caller ID spoofing, SIM box fraud, and service arbitrage schemes",
2149
+ "telecom:porting-misinformation": "Tests for incorrect number portability information, unauthorized port facilitation, or port-blocking that violates FCC LNP rules",
2150
+ "telecom:billing-misinformation": "Tests for incorrect billing information, hidden fees, or misleading price quotes that violate FCC Truth-in-Billing requirements",
2151
+ "telecom:coverage-misinformation": "Tests for false or misleading claims about network coverage, 5G availability, or roaming capabilities",
2152
+ "telecom:law-enforcement-request-handling": "Tests for improper handling of law enforcement data requests in violation of CALEA requirements and privacy laws",
2153
+ "telecom:accessibility-violation": "Tests for accessibility compliance violations under Section 255, Section 508, ADA, and 21st Century CVAA",
2154
+ realestate: "Comprehensive real estate AI safety testing for Fair Housing Act compliance, steering prevention, lending discrimination (ECOA), valuation bias, and housing discrimination vulnerabilities",
2155
+ "realestate:fair-housing-discrimination": "Tests for Fair Housing Act violations across all 7 protected classes: race, color, religion, national origin, sex (including gender identity and sexual orientation), familial status, and disability",
2156
+ "realestate:steering": "Tests for illegal steering practices where systems direct buyers toward or away from neighborhoods based on protected class characteristics rather than objective housing criteria",
2157
+ "realestate:discriminatory-listings": "Tests for discriminatory language in property listings that indicates preferences, limitations, or discrimination based on protected classes including code words and exclusionary language",
2158
+ "realestate:lending-discrimination": "Tests for Equal Credit Opportunity Act (ECOA) and Fair Housing Act violations in mortgage lending including redlining, disparate treatment, and discriminatory underwriting",
2159
+ "realestate:valuation-bias": "Tests for algorithmic bias in property appraisals and automated valuation models that could result in discriminatory property valuations based on neighborhood demographics",
2160
+ "realestate:accessibility-discrimination": "Tests for Americans with Disabilities Act (ADA) and Fair Housing Act violations in disability accommodations, service animal policies, and reasonable modification requests",
2161
+ "realestate:advertising-discrimination": "Tests for Fair Housing Act violations in housing advertising including discriminatory targeting, imagery, and language in digital and traditional marketing",
2162
+ "realestate:source-of-income": "Tests for Section 8 voucher and housing assistance discrimination in jurisdictions with source of income protections",
2163
+ "harmful:chemical-biological-weapons": "Tests detection and response to WMD-related content",
2164
+ "harmful:child-exploitation": "Tests handling of child exploitation content",
2165
+ "harmful:copyright-violations": "Tests handling of intellectual property violations",
2166
+ "harmful:cybercrime": "Tests detection of cybercrime-related content",
2167
+ "harmful:cybercrime:malicious-code": "Tests creation of malicious code",
2168
+ "harmful:graphic-content": "Tests handling of graphic or violent content",
2169
+ "harmful:harassment-bullying": "Tests handling of harassment and bullying content",
2170
+ "harmful:hate": "Tests handling of hate speech and discriminatory content",
2171
+ "harmful:illegal-activities": "Tests handling of illegal activity content",
2172
+ "harmful:illegal-drugs": "Tests handling of drug-related content",
2173
+ "harmful:illegal-drugs:meth": "Tests handling of methamphetamine-related content",
2174
+ "harmful:indiscriminate-weapons": "Tests handling of weapons-related content",
2175
+ "harmful:insults": "Tests handling of personal attacks and insults",
2176
+ "harmful:intellectual-property": "Tests handling of IP theft attempts",
2177
+ "harmful:misinformation-disinformation": "Tests handling of disinformation campaigns",
2178
+ "harmful:non-violent-crime": "Tests handling of non-violent criminal content",
2179
+ "harmful:privacy": "Tests handling of privacy violation attempts",
2180
+ "harmful:profanity": "Tests handling of profane content",
2181
+ "harmful:radicalization": "Tests handling of extremist content",
2182
+ "harmful:self-harm": "Tests handling of self-harm content",
2183
+ "harmful:sex-crime": "Tests handling of sexual crime content",
2184
+ "harmful:sexual-content": "Tests handling of explicit sexual content",
2185
+ "harmful:specialized-advice": "Tests handling of unauthorized professional advice",
2186
+ "harmful:unsafe-practices": "Tests handling of dangerous activity instructions",
2187
+ "harmful:violent-crime": "Tests handling of violent crime content",
2188
+ "harmful:weapons:ied": "Tests handling of Improvised Explosive Devices",
2189
+ hijacking: "Assesses protection against unauthorized resource usage and purpose deviation",
2190
+ imitation: "Tests safeguards against unauthorized entity impersonation attempts",
2191
+ "insurance:coverage-discrimination": "Tests for discriminatory coverage, underwriting, or claims determinations based on protected characteristics (age, disability, race, genetic information, sex) in violation of federal civil rights laws including ADA, Section 1557, GINA, Fair Housing Act, ECOA, and state unfair trade practices acts",
2192
+ "insurance:data-disclosure": "Tests for unauthorized disclosure of protected policyholder data including claims history, property details, driving records, beneficiary information, and business data in violation of GLBA, FCRA, DPPA, and state insurance privacy laws",
2193
+ "insurance:network-misinformation": "Tests for inaccurate provider or vendor network information including wrong network status, terminated contracts, ghost networks, and outdated provider data that could expose policyholders to unexpected costs, voided warranties, or inadequate service",
2194
+ "insurance:phi-disclosure": "Tests for unauthorized disclosure of Protected Health Information in violation of HIPAA privacy regulations including cross-member PHI leakage, inadequate authentication, and disclosure to unauthorized third parties",
2195
+ "ecommerce:pci-dss": "Tests for exposure of payment card data including credit card numbers, CVVs, expiration dates, cardholder names, bank account details, and payment tokens in violation of PCI DSS requirements for secure payment processing",
2196
+ "ecommerce:compliance-bypass": "Tests for attempts to bypass security rules, terms of service, age restrictions, geographic limitations, or compliance policies in e-commerce systems",
2197
+ "ecommerce:order-fraud": "Tests for detection of fraudulent order patterns, unauthorized transactions, account takeover attempts, or payment manipulation in e-commerce checkout and order processing",
2198
+ "ecommerce:price-manipulation": "Tests for price manipulation through discount code abuse, inventory exploits, cart manipulation, unauthorized price modifications, or coupon stacking vulnerabilities",
2199
+ mcp: "Tests for vulnerabilities to Model Context Protocol (MCP) attacks",
2200
+ "medical:anchoring-bias": "Tests for medical anchoring bias where AI fixates on irrelevant information in medical contexts",
2201
+ "medical:hallucination": "Tests for fabricated medical facts, non-existent studies, made-up drug interactions, or other false medical information",
2202
+ "medical:incorrect-knowledge": "Tests for factually wrong medical information that could be harmful if followed",
2203
+ "medical:off-label-use": "Tests for inappropriate off-label medication recommendations without proper disclaimers, warnings, or professional consultation requirements",
2204
+ "medical:prioritization-error": "Tests for poor medical prioritization and triage decisions that could delay critical care",
2205
+ "medical:sycophancy": "Tests for agreeing with incorrect medical assumptions or providing responses aligned with user desires at the expense of medical accuracy",
2206
+ "financial:calculation-error": "Tests for errors in option pricing, risk calculations, portfolio optimization, or other quantitative financial analysis that could lead to significant losses",
2207
+ "financial:compliance-violation": "Tests for suggestions that enable insider trading, market manipulation, money laundering, or other violations of securities laws and financial regulations",
2208
+ "financial:confidential-disclosure": "Tests for inappropriate disclosure of material non-public information (MNPI), proprietary trading strategies, client portfolios, deal information, or other confidential financial data",
2209
+ "financial:counterfactual": "Tests for false financial narratives including fabricated market events, fictional corporate actions, misleading causation, or manipulated timelines that could mislead investors",
2210
+ "financial:data-leakage": "Tests for exposure of proprietary trading algorithms, customer account information, M&A details, or other confidential financial data",
2211
+ "financial:defamation": "Tests for false statements that damage the reputation of financial entities, including unfounded fraud accusations, fabricated scandals, or false insolvency claims",
2212
+ "financial:hallucination": "Tests for fabricated market data, non-existent financial instruments, fictional company financials, or invented market events",
2213
+ "financial:impartiality": "Tests for inappropriate provision of financial advice, investment recommendations, broker suggestions, or transaction facilitation without proper authorization or disclaimers",
2214
+ "financial:japan-fiea-suitability": "Tests whether AI recommends unsuitable financial products, downplays risks, or uses sales framing that conflicts with Japan's FIEA suitability and explanation duties",
2215
+ "financial:misconduct": "Tests for facilitation of financial crimes including insider trading, market manipulation, front-running, spoofing, bribery, or circumventing financial regulations",
2216
+ "financial:sox-compliance": "Tests for violations of Sarbanes-Oxley requirements including internal controls bypass, financial statement manipulation, CEO/CFO certification fraud, audit obstruction, document retention violations, and whistleblower retaliation",
2217
+ "financial:sycophancy": "Tests for agreeing with risky trading strategies, unrealistic return expectations, or get-rich-quick schemes",
2218
+ "goal-misalignment": "Tests whether AI systems recognize when optimizing measurable proxy metrics might not align with true underlying objectives (Goodhart's Law: \"When a measure becomes a target, it ceases to be a good measure\")",
2219
+ "off-topic": "Tests whether AI systems can be manipulated to go off-topic by performing tasks outside their domain",
2220
+ "pharmacy:controlled-substance-compliance": "Tests adherence to DEA regulations for controlled substance prescribing, dispensing, and monitoring, including schedule restrictions, quantity limits, early refill detection, PDMP requirements, and red flags for prescription fraud or diversion",
2221
+ "pharmacy:dosage-calculation": "Tests accuracy of pharmaceutical dosage calculations including weight-based dosing, renal and hepatic adjustments, unit conversions, IV drip rates, concentration calculations, and pediatric dosing to prevent over/underdosing errors",
2222
+ "pharmacy:drug-interaction": "Tests identification of dangerous drug-drug, drug-food, and drug-supplement interactions including major contraindications, CYP450 interactions, QT prolongation risks, serotonin syndrome, and additive effects to prevent serious adverse drug events",
2223
+ "indirect-prompt-injection": "Evaluates vulnerability to injection attacks via untrusted variables",
2224
+ intent: "Tests system resilience against specific manipulation attempts",
2225
+ overreliance: "Assesses overreliance on system assumptions",
2226
+ pii: "Comprehensive evaluation of personal identifiable information protection",
2227
+ "pii:api-db": "Tests for PII exposure through API and database access vectors",
2228
+ "pii:direct": "Evaluates direct PII exposure vulnerabilities",
2229
+ "pii:session": "Tests for PII leakage in session data",
2230
+ "pii:social": "Assesses PII exposure through social engineering vectors",
2231
+ pliny: "Tests handling of Pliny prompt injections",
2232
+ policy: "Validates compliance with custom security policies and controls",
2233
+ politics: "Tests handling of political content and bias mitigation",
2234
+ "prompt-extraction": "Evaluates protection against system prompt disclosure attempts",
2235
+ "rag-document-exfiltration": "Tests handling of RAG document exfiltration",
2236
+ "rag-poisoning": "Tests resistance against poisoning attacks on RAG retrieval systems",
2237
+ "rag-source-attribution": "Tests whether RAG systems fabricate document citations, policy references, or source attributions that don't exist in the knowledge base",
2238
+ rbac: "Tests implementation of role-based access control mechanisms",
2239
+ "reasoning-dos": "Tests for computational resource exhaustion through excessive reasoning patterns",
2240
+ religion: "Assesses handling of religious content and bias mitigation",
2241
+ "shell-injection": "Tests protection against command injection vulnerabilities",
2242
+ "special-token-injection": "Tests for token injection vulnerabilities using conversation format delimiters",
2243
+ "sql-injection": "Evaluates resilience against SQL injection attacks",
2244
+ ssrf: "Tests for server-side request forgery vulnerabilities",
2245
+ "system-prompt-override": "Tests for system prompt override vulnerabilities",
2246
+ unsafebench: "Tests handling of unsafe image content through multi-modal model evaluation and safety filters",
2247
+ "unverifiable-claims": "Tests whether an AI system makes claims that cannot be verified, including future predictions, unsourced statistics, and unknowable information",
2248
+ vlguard: "Tests handling of potentially unsafe image content using the VLGuard dataset",
2249
+ vlsu: "Tests compositional safety where individually safe images and text combine to produce harmful outputs using Apple VLSU dataset",
2250
+ wordplay: "Tests whether AI systems can be tricked into generating profanity or offensive language through innocent-seeming wordplay like riddles and rhyming games",
2251
+ xstest: "Tests how models handle ambiguous terms related to potentially harmful topics like violence and drugs",
2252
+ "guardrails-eval": "Evaluate guardrail effectiveness against common risks"
2253
+ };
2254
+ //#endregion
2255
+ //#region src/redteam/constants/strategies.ts
2256
+ const FRAMEWORK_COMPLIANCE_IDS = [
2257
+ "mitre:atlas",
2258
+ "nist:ai:measure",
2259
+ "owasp:api",
2260
+ "owasp:llm",
2261
+ "owasp:agentic",
2262
+ "eu:ai-act",
2263
+ "iso:42001",
2264
+ "gdpr",
2265
+ "dod:ai:ethics"
2266
+ ];
2267
+ const DEFAULT_STRATEGIES = [
2268
+ "basic",
2269
+ "jailbreak:meta",
2270
+ "jailbreak:composite"
2271
+ ];
2272
+ new Set(DEFAULT_STRATEGIES);
2273
+ const MULTI_TURN_STRATEGIES = [
2274
+ "crescendo",
2275
+ "goat",
2276
+ "jailbreak:hydra",
2277
+ "custom",
2278
+ "mischievous-user"
2279
+ ];
2280
+ new Set(MULTI_TURN_STRATEGIES);
2281
+ const isCustomStrategy = (strategyId) => {
2282
+ return strategyId === "custom" || strategyId.startsWith("custom:");
2283
+ };
2284
+ const AGENTIC_STRATEGIES = [
2285
+ "crescendo",
2286
+ "goat",
2287
+ "indirect-web-pwn",
2288
+ "custom",
2289
+ "jailbreak",
2290
+ "jailbreak:hydra",
2291
+ "jailbreak:meta",
2292
+ "jailbreak:tree",
2293
+ "mischievous-user"
2294
+ ];
2295
+ new Set(AGENTIC_STRATEGIES);
2296
+ const DATASET_PLUGINS = [
2297
+ "beavertails",
2298
+ "cyberseceval",
2299
+ "donotanswer",
2300
+ "harmbench",
2301
+ "toxic-chat",
2302
+ "aegis",
2303
+ "pliny",
2304
+ "unsafebench",
2305
+ "vlguard",
2306
+ "xstest"
2307
+ ];
2308
+ const ADDITIONAL_STRATEGIES = [
2309
+ "audio",
2310
+ "authoritative-markup-injection",
2311
+ "base64",
2312
+ "best-of-n",
2313
+ "camelcase",
2314
+ "citation",
2315
+ "crescendo",
2316
+ "custom",
2317
+ "emoji",
2318
+ "gcg",
2319
+ "goat",
2320
+ "hex",
2321
+ "homoglyph",
2322
+ "image",
2323
+ "indirect-web-pwn",
2324
+ "jailbreak:hydra",
2325
+ "jailbreak",
2326
+ "jailbreak:likert",
2327
+ "jailbreak:meta",
2328
+ "jailbreak:tree",
2329
+ "jailbreak-templates",
2330
+ "layer",
2331
+ "leetspeak",
2332
+ "math-prompt",
2333
+ "mischievous-user",
2334
+ "morse",
2335
+ "multilingual",
2336
+ "piglatin",
2337
+ "prompt-injection",
2338
+ "retry",
2339
+ "rot13",
2340
+ "video"
2341
+ ];
2342
+ const STRATEGY_COLLECTIONS = ["other-encodings"];
2343
+ const STRATEGY_COLLECTION_MAPPINGS = { "other-encodings": [
2344
+ "camelcase",
2345
+ "morse",
2346
+ "piglatin",
2347
+ "emoji"
2348
+ ] };
2349
+ const _ALL_STRATEGIES = [
2350
+ "default",
2351
+ ...DEFAULT_STRATEGIES,
2352
+ ...ADDITIONAL_STRATEGIES,
2353
+ ...STRATEGY_COLLECTIONS,
2354
+ ...AGENTIC_STRATEGIES
2355
+ ];
2356
+ const ALL_STRATEGIES = Array.from(new Set(_ALL_STRATEGIES)).sort();
2357
+ /**
2358
+ * Default 'n' fan out for strategies that can add additional test cases during generation
2359
+ */
2360
+ const DEFAULT_N_FAN_OUT_BY_STRATEGY = {
2361
+ "jailbreak:composite": 5,
2362
+ gcg: 1
2363
+ };
2364
+ for (const strategyId in DEFAULT_N_FAN_OUT_BY_STRATEGY) if (!ALL_STRATEGIES.includes(strategyId)) throw new Error(`Default fan out strategy ${strategyId} is not in ALL_STRATEGIES`);
2365
+ function getDefaultNFanout(strategyId) {
2366
+ return DEFAULT_N_FAN_OUT_BY_STRATEGY[strategyId] ?? 1;
2367
+ }
2368
+ function isFanoutStrategy(strategyId) {
2369
+ return strategyId in DEFAULT_N_FAN_OUT_BY_STRATEGY;
2370
+ }
2371
+ //#endregion
2372
+ //#region src/util/uuid.ts
2373
+ /**
2374
+ * UUID validation regex pattern.
2375
+ * Matches UUID v1-v5 format: xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx
2376
+ * where M is the version (1-5) and N is the variant (8, 9, a, or b).
2377
+ */
2378
+ const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
2379
+ /**
2380
+ * Validates whether a string is a valid UUID (v1-v5).
2381
+ * @param value - The string to validate
2382
+ * @returns true if the string is a valid UUID, false otherwise
2383
+ */
2384
+ function isUuid(value) {
2385
+ return UUID_REGEX.test(value);
2386
+ }
2387
+ //#endregion
2388
+ //#region src/redteam/plugins/policy/validators.ts
2389
+ /**
2390
+ * @fileoverview This module contains pure validation functions – those without external dependencies
2391
+ * e.g. `PolicyObjectSchema` (which would otherwise introduce circular dependencies).
2392
+ *
2393
+ * TODO:
2394
+ *
2395
+ * - PolicyObjectSchema could be moved into this module along w/ `isPolicyMetric` and `isValidPolicyObject`,
2396
+ * to co-locate all of the policy validation logic.
2397
+ */
2398
+ /**
2399
+ * Checks whether a policy ID is a valid reusable policy ID.
2400
+ * @param id - The policy ID to check.
2401
+ * @returns True if the policy ID is a valid reusable policy ID, false otherwise.
2402
+ */
2403
+ function isValidReusablePolicyId(id) {
2404
+ return isUuid(id);
2405
+ }
2406
+ /**
2407
+ * Checks whether a policy ID is a valid inline policy ID.
2408
+ * @param id - The policy ID to check.
2409
+ * @returns True if the policy ID is a valid inline policy ID, false otherwise.
2410
+ */
2411
+ function isValidInlinePolicyId(id) {
2412
+ return /^[0-9a-f]{12}$/i.test(id);
2413
+ }
2414
+ /**
2415
+ * Checks whether a policy ID is a valid policy ID.
2416
+ * @param id - The policy ID to check.
2417
+ * @returns True if the policy ID is a valid policy ID, false otherwise.
2418
+ */
2419
+ function isValidPolicyId(id) {
2420
+ return isValidReusablePolicyId(id) || isValidInlinePolicyId(id);
2421
+ }
2422
+ //#endregion
2423
+ //#region src/redteam/types.ts
2424
+ const PolicyObjectSchema = zod.z.object({
2425
+ id: zod.z.string().refine(isValidPolicyId, { message: "ID must be either a UUID or a 12-character hex string" }),
2426
+ text: zod.z.string().optional(),
2427
+ name: zod.z.string().optional()
2428
+ });
2429
+ const PluginConfigSchema = zod.z.object({
2430
+ examples: zod.z.array(zod.z.string()).optional(),
2431
+ graderExamples: zod.z.array(zod.z.object({
2432
+ output: zod.z.string(),
2433
+ pass: zod.z.boolean(),
2434
+ score: zod.z.number(),
2435
+ reason: zod.z.string()
2436
+ })).optional(),
2437
+ graderGuidance: zod.z.string().optional(),
2438
+ severity: SeveritySchema.optional(),
2439
+ language: zod.z.union([zod.z.string(), zod.z.array(zod.z.string())]).optional(),
2440
+ prompt: zod.z.string().optional(),
2441
+ purpose: zod.z.string().optional(),
2442
+ modifiers: zod.z.record(zod.z.string(), zod.z.unknown()).optional(),
2443
+ targetIdentifiers: zod.z.array(zod.z.string()).optional(),
2444
+ targetSystems: zod.z.array(zod.z.string()).optional(),
2445
+ mentions: zod.z.boolean().optional(),
2446
+ targetUrls: zod.z.array(zod.z.string()).optional(),
2447
+ ssrfFailThreshold: zod.z.enum([
2448
+ "low",
2449
+ "medium",
2450
+ "high",
2451
+ "critical"
2452
+ ]).optional(),
2453
+ name: zod.z.string().optional(),
2454
+ multilingual: zod.z.boolean().optional(),
2455
+ indirectInjectionVar: zod.z.string().optional(),
2456
+ intendedResults: zod.z.array(zod.z.string()).optional(),
2457
+ intent: zod.z.union([zod.z.string(), zod.z.array(zod.z.union([zod.z.string(), zod.z.array(zod.z.string())]))]).optional(),
2458
+ policy: zod.z.union([zod.z.string(), PolicyObjectSchema]).optional(),
2459
+ systemPrompt: zod.z.string().optional(),
2460
+ excludeStrategies: zod.z.array(zod.z.string()).optional(),
2461
+ inputs: InputsSchema.optional(),
2462
+ __nonce: zod.z.number().optional()
2463
+ });
2464
+ const StrategyConfigSchema = zod.z.object({
2465
+ enabled: zod.z.boolean().optional(),
2466
+ plugins: zod.z.array(zod.z.string()).optional(),
2467
+ numTests: zod.z.number().int().min(0).finite().optional()
2468
+ }).catchall(zod.z.unknown());
2469
+ const ConversationMessageSchema = zod.z.object({
2470
+ role: zod.z.enum(["assistant", "user"]),
2471
+ content: zod.z.string()
2472
+ });
2473
+ /**
2474
+ * Custom error class for partial test generation failures.
2475
+ * Thrown when some plugins completely fail to generate any test cases,
2476
+ * which would significantly impact scan quality and completeness.
2477
+ */
2478
+ var PartialGenerationError = class extends Error {
2479
+ failedPlugins;
2480
+ constructor(failedPlugins) {
2481
+ const pluginList = failedPlugins.map((p) => ` - ${p.pluginId} (0/${p.requested} tests)`);
2482
+ const message = `Test case generation failed for ${failedPlugins.length} plugin(s):\n${pluginList.join("\n")}\n\nThe scan has been stopped because missing test cases would significantly decrease scan quality and completeness.\n\nPossible causes:\n - API rate limiting or connectivity issues\n - Invalid plugin configuration\n - Provider errors during generation\n\nTo troubleshoot:\n - Run with --verbose flag to see detailed error messages\n - Check API keys and provider configuration\n - Retry the scan after resolving any reported errors`;
2483
+ super(message);
2484
+ this.name = "PartialGenerationError";
2485
+ this.failedPlugins = failedPlugins;
2486
+ }
2487
+ };
2488
+ //#endregion
2489
+ //#region src/validators/providers.ts
2490
+ const ProviderOptionsSchema = zod.z.object({
2491
+ id: zod.z.custom().optional(),
2492
+ label: zod.z.custom().optional(),
2493
+ config: zod.z.any().optional(),
2494
+ prompts: zod.z.array(zod.z.string()).optional(),
2495
+ transform: zod.z.string().optional(),
2496
+ delay: zod.z.number().optional(),
2497
+ env: ProviderEnvOverridesSchema.optional(),
2498
+ inputs: InputsSchema.optional()
2499
+ });
2500
+ const CallApiFunctionSchema = zod.z.custom((v) => typeof v === "function");
2501
+ const ApiProviderSchema = zod.z.object({
2502
+ id: zod.z.custom((v) => typeof v === "function"),
2503
+ callApi: zod.z.custom((v) => typeof v === "function"),
2504
+ callEmbeddingApi: zod.z.custom((v) => typeof v === "function").optional(),
2505
+ callClassificationApi: zod.z.custom((v) => typeof v === "function").optional(),
2506
+ label: zod.z.custom().optional(),
2507
+ transform: zod.z.string().optional(),
2508
+ delay: zod.z.number().optional(),
2509
+ config: zod.z.any().optional(),
2510
+ inputs: InputsSchema.optional()
2511
+ });
2512
+ zod.z.object({
2513
+ cached: zod.z.boolean().optional(),
2514
+ cost: zod.z.number().optional(),
2515
+ error: zod.z.string().optional(),
2516
+ logProbs: zod.z.array(zod.z.number()).optional(),
2517
+ metadata: zod.z.object({ redteamFinalPrompt: zod.z.string().optional() }).catchall(zod.z.any()).optional(),
2518
+ output: zod.z.union([zod.z.string(), zod.z.any()]).optional(),
2519
+ tokenUsage: BaseTokenUsageSchema.optional()
2520
+ });
2521
+ zod.z.object({
2522
+ error: zod.z.string().optional(),
2523
+ embedding: zod.z.array(zod.z.number()).optional(),
2524
+ tokenUsage: BaseTokenUsageSchema.partial().optional()
2525
+ });
2526
+ zod.z.object({
2527
+ error: zod.z.string().optional(),
2528
+ similarity: zod.z.number().optional(),
2529
+ tokenUsage: BaseTokenUsageSchema.partial().optional()
2530
+ });
2531
+ zod.z.object({
2532
+ error: zod.z.string().optional(),
2533
+ classification: zod.z.record(zod.z.string(), zod.z.number()).optional()
2534
+ });
2535
+ const ProvidersSchema = zod.z.union([
2536
+ zod.z.string(),
2537
+ CallApiFunctionSchema,
2538
+ zod.z.array(zod.z.union([
2539
+ zod.z.string(),
2540
+ CallApiFunctionSchema,
2541
+ zod.z.record(zod.z.string(), ProviderOptionsSchema),
2542
+ ProviderOptionsSchema
2543
+ ]))
2544
+ ]);
2545
+ const ProviderSchema = zod.z.union([
2546
+ zod.z.string(),
2547
+ ApiProviderSchema,
2548
+ ProviderOptionsSchema
2549
+ ]);
2550
+ //#endregion
2551
+ //#region src/validators/redteam.ts
2552
+ const TracingConfigSchema = zod.z.lazy(() => zod.z.object({
2553
+ enabled: zod.z.boolean().optional(),
2554
+ includeInAttack: zod.z.boolean().optional(),
2555
+ includeInGrading: zod.z.boolean().optional(),
2556
+ includeInternalSpans: zod.z.boolean().optional(),
2557
+ maxSpans: zod.z.int().positive().optional(),
2558
+ maxDepth: zod.z.int().positive().optional(),
2559
+ maxRetries: zod.z.int().nonnegative().optional(),
2560
+ retryDelayMs: zod.z.int().nonnegative().optional(),
2561
+ spanFilter: zod.z.array(zod.z.string()).optional(),
2562
+ sanitizeAttributes: zod.z.boolean().optional(),
2563
+ strategies: zod.z.record(zod.z.string(), zod.z.lazy(() => TracingConfigSchema)).optional()
2564
+ }));
2565
+ /**
2566
+ * Schema for redteam contexts - allows testing multiple security contexts/states
2567
+ */
2568
+ const RedteamContextSchema = zod.z.object({
2569
+ id: zod.z.string().describe("Unique identifier for the context"),
2570
+ purpose: zod.z.string().describe("Purpose/context for this context - used for generation and grading"),
2571
+ vars: zod.z.record(zod.z.string(), zod.z.string()).optional().describe("Variables passed to provider (e.g., context_file, user_role)")
2572
+ });
2573
+ const frameworkOptions = FRAMEWORK_COMPLIANCE_IDS;
2574
+ const pluginOptions = [...new Set([
2575
+ ...COLLECTIONS,
2576
+ ...ALL_PLUGINS,
2577
+ ...ALIASED_PLUGINS
2578
+ ])].sort();
2579
+ /**
2580
+ * Schema for individual redteam plugins
2581
+ */
2582
+ const RedteamPluginObjectSchema = zod.z.object({
2583
+ id: zod.z.union([zod.z.enum(pluginOptions).superRefine((val, ctx) => {
2584
+ if (!pluginOptions.includes(val)) ctx.addIssue({
2585
+ code: "custom",
2586
+ message: `Invalid plugin name "${val}". Must be one of: ${pluginOptions.join(", ")} (or a path starting with file://)`
2587
+ });
2588
+ }), zod.z.string().superRefine((val, ctx) => {
2589
+ if (!val.startsWith("file://")) ctx.addIssue({
2590
+ code: "custom",
2591
+ message: `Invalid plugin id "${val}". Custom plugins must start with file:// or use a built-in plugin. See https://www.promptfoo.dev/docs/red-team/plugins for available plugins.`
2592
+ });
2593
+ })]).describe("Name of the plugin"),
2594
+ numTests: zod.z.int().positive().prefault(5).describe("Number of tests to generate for this plugin"),
2595
+ config: zod.z.record(zod.z.string(), zod.z.unknown()).optional().describe("Plugin-specific configuration"),
2596
+ severity: SeveritySchema.optional().describe("Severity level for this plugin")
2597
+ });
2598
+ /**
2599
+ * Schema for individual redteam plugins or their shorthand.
2600
+ */
2601
+ const RedteamPluginSchema = zod.z.union([zod.z.union([zod.z.enum(pluginOptions).superRefine((val, ctx) => {
2602
+ if (!pluginOptions.includes(val)) ctx.addIssue({
2603
+ code: "custom",
2604
+ message: `Invalid plugin name "${val}". Must be one of: ${pluginOptions.join(", ")} (or a path starting with file://)`
2605
+ });
2606
+ }), zod.z.string().superRefine((val, ctx) => {
2607
+ if (!val.startsWith("file://")) ctx.addIssue({
2608
+ code: "custom",
2609
+ message: `Invalid plugin id "${val}". Custom plugins must start with file:// or use a built-in plugin. See https://www.promptfoo.dev/docs/red-team/plugins for available plugins.`
2610
+ });
2611
+ })]).describe("Name of the plugin or path to custom plugin"), RedteamPluginObjectSchema]);
2612
+ const strategyIdSchema = zod.z.union([
2613
+ zod.z.enum(ALL_STRATEGIES).superRefine((val, ctx) => {
2614
+ if (val === "multilingual") return;
2615
+ if (!ALL_STRATEGIES.includes(val)) ctx.addIssue({
2616
+ code: "custom",
2617
+ message: `Invalid strategy name "${val}". Must be one of: ${[...ALL_STRATEGIES].join(", ")} (or a path starting with file://)`
2618
+ });
2619
+ }),
2620
+ zod.z.string().refine((value) => {
2621
+ if (value === "multilingual") return true;
2622
+ return value.startsWith("file://") && require_fileExtensions.isJavascriptFile(value);
2623
+ }, { message: `Custom strategies must start with file:// and end with .js or .ts, or use one of the built-in strategies: ${[...ALL_STRATEGIES].join(", ")}` }),
2624
+ zod.z.string().refine((value) => {
2625
+ return isCustomStrategy(value);
2626
+ }, { message: `Strategy must be one of the built-in strategies: ${[...ALL_STRATEGIES].join(", ")} (or a path starting with file://)` })
2627
+ ]);
2628
+ /**
2629
+ * Schema for individual redteam strategies
2630
+ */
2631
+ const RedteamStrategySchema = zod.z.union([strategyIdSchema, zod.z.object({
2632
+ id: strategyIdSchema,
2633
+ config: zod.z.record(zod.z.string(), zod.z.unknown()).optional().describe("Strategy-specific configuration")
2634
+ })]);
2635
+ zod.z.object({
2636
+ addPlugins: zod.z.array(zod.z.enum(ADDITIONAL_PLUGINS)).optional().describe("Additional plugins to include"),
2637
+ addStrategies: zod.z.array(zod.z.enum(ADDITIONAL_STRATEGIES)).optional().describe("Additional strategies to include"),
2638
+ cache: zod.z.boolean().describe("Whether to use caching"),
2639
+ config: zod.z.string().optional().describe("Path to the configuration file"),
2640
+ target: zod.z.string().optional().describe("Cloud provider target ID to run the scan on"),
2641
+ defaultConfig: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Default configuration object"),
2642
+ defaultConfigPath: zod.z.string().optional().describe("Path to the default configuration file"),
2643
+ description: zod.z.string().optional().describe("Custom description/name for the generated tests"),
2644
+ delay: zod.z.int().nonnegative().optional().describe("Delay in milliseconds between plugin API calls"),
2645
+ envFile: zod.z.string().optional().describe("Path to the environment file"),
2646
+ force: zod.z.boolean().describe("Whether to force generation").prefault(false),
2647
+ injectVar: zod.z.string().optional().describe("Variable to inject"),
2648
+ language: zod.z.union([zod.z.string(), zod.z.array(zod.z.string())]).optional().describe("Language(s) of tests to generate"),
2649
+ frameworks: zod.z.array(zod.z.enum(frameworkOptions)).min(1).optional().describe("Subset of compliance frameworks to include when generating, reporting, and filtering results"),
2650
+ maxConcurrency: zod.z.int().positive().optional().describe("Maximum number of concurrent API calls"),
2651
+ numTests: zod.z.int().positive().optional().describe("Number of tests to generate"),
2652
+ output: zod.z.string().optional().describe("Output file path"),
2653
+ plugins: zod.z.array(RedteamPluginObjectSchema).optional().describe("Plugins to use"),
2654
+ provider: zod.z.string().optional().describe("Provider to use"),
2655
+ purpose: zod.z.string().optional().describe("Purpose of the redteam generation"),
2656
+ strategies: zod.z.array(RedteamStrategySchema).optional().describe("Strategies to use"),
2657
+ write: zod.z.boolean().describe("Whether to write the output"),
2658
+ burpEscapeJson: zod.z.boolean().describe("Whether to escape quotes in Burp payloads").optional(),
2659
+ progressBar: zod.z.boolean().describe("Whether to show a progress bar").optional(),
2660
+ configFromCloud: zod.z.any().optional().describe("A configuration object loaded from cloud"),
2661
+ strict: zod.z.boolean().optional().default(false).describe("Fail the scan if any plugins fail to generate test cases")
2662
+ });
2663
+ /**
2664
+ * Schema for `redteam` section of promptfooconfig.yaml
2665
+ */
2666
+ const RedteamConfigSchema = zod.z.object({
2667
+ injectVar: zod.z.string().optional().describe("Variable to inject. Can be a string or array of strings. If string, it's transformed to an array. Inferred from the prompts by default."),
2668
+ purpose: zod.z.string().optional().describe("Purpose override string - describes the prompt templates"),
2669
+ testGenerationInstructions: zod.z.string().optional().describe("Additional instructions for test generation applied to each plugin"),
2670
+ provider: ProviderSchema.optional().describe("Provider used for generating adversarial inputs"),
2671
+ numTests: zod.z.int().positive().optional().describe("Number of tests to generate"),
2672
+ language: zod.z.union([zod.z.string(), zod.z.array(zod.z.string())]).optional().describe("Language(s) of tests to generate for this plugin"),
2673
+ frameworks: zod.z.array(zod.z.enum(frameworkOptions)).min(1).optional().describe("Compliance frameworks to include across reports and commands"),
2674
+ entities: zod.z.array(zod.z.string()).optional().describe("Names of people, brands, or organizations related to your LLM application"),
2675
+ contexts: zod.z.array(RedteamContextSchema).optional().describe("Security contexts for testing multiple states - each context has its own purpose"),
2676
+ plugins: zod.z.array(RedteamPluginSchema).describe("Plugins to use for redteam generation").prefault(["default"]),
2677
+ strategies: zod.z.array(RedteamStrategySchema).describe(dedent.default`Strategies to use for redteam generation.
2678
+
2679
+ Defaults to ${DEFAULT_STRATEGIES.join(", ")}
2680
+ Supports ${ALL_STRATEGIES.join(", ")}
2681
+ `).optional().prefault(["default"]),
2682
+ maxConcurrency: zod.z.int().positive().optional().describe("Maximum number of concurrent API calls"),
2683
+ delay: zod.z.int().nonnegative().optional().describe("Delay in milliseconds between plugin API calls"),
2684
+ excludeTargetOutputFromAgenticAttackGeneration: zod.z.boolean().optional().describe("Whether to exclude target output from the agentific attack generation process"),
2685
+ tracing: TracingConfigSchema.optional().describe("Tracing defaults applied to all strategies unless overridden"),
2686
+ graderExamples: zod.z.array(zod.z.object({
2687
+ output: zod.z.string(),
2688
+ pass: zod.z.boolean(),
2689
+ score: zod.z.number(),
2690
+ reason: zod.z.string()
2691
+ })).optional().describe("Global grading examples that apply to all plugins")
2692
+ }).transform((data) => {
2693
+ const pluginMap = /* @__PURE__ */ new Map();
2694
+ const strategySet = /* @__PURE__ */ new Set();
2695
+ const frameworks = data.frameworks && data.frameworks.length > 0 ? Array.from(new Set(data.frameworks)) : void 0;
2696
+ const multilingualStrategy = data.strategies?.find((s) => (typeof s === "string" ? s : s.id) === "multilingual");
2697
+ if (multilingualStrategy && typeof multilingualStrategy !== "string") {
2698
+ const strategyLanguages = multilingualStrategy.config?.languages;
2699
+ if (Array.isArray(strategyLanguages) && strategyLanguages.length > 0) {
2700
+ console.debug("[DEPRECATED] The \"multilingual\" strategy is deprecated. Use the top-level \"language\" config instead. See: https://www.promptfoo.dev/docs/red-team/configuration/#language");
2701
+ if (data.language) {
2702
+ const existingLanguages = Array.isArray(data.language) ? data.language : [data.language];
2703
+ data.language = [...new Set([
2704
+ ...existingLanguages,
2705
+ "en",
2706
+ ...strategyLanguages
2707
+ ])];
2708
+ } else data.language = ["en", ...strategyLanguages];
2709
+ data.strategies = data.strategies?.filter((s) => {
2710
+ return (typeof s === "string" ? s : s.id) !== "multilingual";
2711
+ });
2712
+ }
2713
+ }
2714
+ const addPlugin = (id, config, numTests, severity) => {
2715
+ const key = `${id}:${JSON.stringify(config)}:${severity || ""}`;
2716
+ const pluginObject = { id };
2717
+ if (numTests !== void 0 || data.numTests !== void 0) pluginObject.numTests = numTests ?? data.numTests;
2718
+ if (config !== void 0) pluginObject.config = config;
2719
+ if (severity !== void 0) pluginObject.severity = severity;
2720
+ pluginMap.set(key, pluginObject);
2721
+ };
2722
+ const expandCollection = (collection, config, numTests, severity) => {
2723
+ (Array.isArray(collection) ? collection : Array.from(collection)).forEach((item) => {
2724
+ const existingPlugin = pluginMap.get(`${item}:${JSON.stringify(config)}:${severity || ""}`);
2725
+ if (!existingPlugin || existingPlugin.numTests === void 0) addPlugin(item, config, numTests, severity);
2726
+ });
2727
+ };
2728
+ const handleCollectionExpansion = (id, config, numTests, severity) => {
2729
+ if (id === "foundation") expandCollection([...FOUNDATION_PLUGINS], config, numTests, severity);
2730
+ else if (id === "harmful") expandCollection(Object.keys(HARM_PLUGINS), config, numTests, severity);
2731
+ else if (id === "pii") expandCollection([...PII_PLUGINS], config, numTests, severity);
2732
+ else if (id === "medical") expandCollection([...MEDICAL_PLUGINS], config, numTests, severity);
2733
+ else if (id === "pharmacy") expandCollection([...PHARMACY_PLUGINS], config, numTests, severity);
2734
+ else if (id === "insurance") expandCollection([...INSURANCE_PLUGINS], config, numTests, severity);
2735
+ else if (id === "financial") expandCollection([...FINANCIAL_PLUGINS], config, numTests, severity);
2736
+ else if (id === "default") expandCollection([...DEFAULT_PLUGINS], config, numTests, severity);
2737
+ else if (id === "guardrails-eval") expandCollection([...GUARDRAILS_EVALUATION_PLUGINS], config, numTests, severity);
2738
+ };
2739
+ const handlePlugin = (plugin) => {
2740
+ const pluginObj = typeof plugin === "string" ? {
2741
+ id: plugin,
2742
+ numTests: data.numTests,
2743
+ config: void 0,
2744
+ severity: void 0
2745
+ } : {
2746
+ ...plugin,
2747
+ numTests: plugin.numTests ?? data.numTests
2748
+ };
2749
+ if (ALIASED_PLUGIN_MAPPINGS[pluginObj.id]) Object.values(ALIASED_PLUGIN_MAPPINGS[pluginObj.id]).forEach(({ plugins, strategies }) => {
2750
+ plugins.forEach((id) => {
2751
+ if (COLLECTIONS.includes(id)) handleCollectionExpansion(id, pluginObj.config, pluginObj.numTests, pluginObj.severity);
2752
+ else addPlugin(id, pluginObj.config, pluginObj.numTests, pluginObj.severity);
2753
+ });
2754
+ strategies.forEach((strategy) => strategySet.add(strategy));
2755
+ });
2756
+ else if (COLLECTIONS.includes(pluginObj.id)) handleCollectionExpansion(pluginObj.id, pluginObj.config, pluginObj.numTests, pluginObj.severity);
2757
+ else {
2758
+ const mapping = Object.entries(ALIASED_PLUGIN_MAPPINGS).find(([, value]) => Object.keys(value).includes(pluginObj.id));
2759
+ if (mapping) {
2760
+ const [, aliasedMapping] = mapping;
2761
+ aliasedMapping[pluginObj.id].plugins.forEach((id) => {
2762
+ if (COLLECTIONS.includes(id)) handleCollectionExpansion(id, pluginObj.config, pluginObj.numTests, pluginObj.severity);
2763
+ else addPlugin(id, pluginObj.config, pluginObj.numTests, pluginObj.severity);
2764
+ });
2765
+ aliasedMapping[pluginObj.id].strategies.forEach((strategy) => strategySet.add(strategy));
2766
+ } else addPlugin(pluginObj.id, pluginObj.config, pluginObj.numTests, pluginObj.severity);
2767
+ }
2768
+ };
2769
+ data.plugins.forEach(handlePlugin);
2770
+ const uniquePlugins = Array.from(pluginMap.values()).filter((plugin) => !COLLECTIONS.includes(plugin.id)).sort((a, b) => {
2771
+ if (a.id !== b.id) return a.id.localeCompare(b.id);
2772
+ return JSON.stringify(a.config || {}).localeCompare(JSON.stringify(b.config || {}));
2773
+ });
2774
+ const getStrategyKey = (strategy) => {
2775
+ if (typeof strategy === "string") return strategy;
2776
+ if (strategy.id === "layer" && strategy.config) {
2777
+ if (strategy.config.label) return `layer/${strategy.config.label}`;
2778
+ if (strategy.config.steps) return `layer:${JSON.stringify(strategy.config.steps)}`;
2779
+ }
2780
+ if (strategy.config && Object.keys(strategy.config).length > 0) return `${strategy.id}:${JSON.stringify(strategy.config)}`;
2781
+ return strategy.id;
2782
+ };
2783
+ const strategies = Array.from(new Map([...data.strategies || [], ...Array.from(strategySet)].flatMap((strategy) => {
2784
+ if (typeof strategy === "string") {
2785
+ if (strategy === "basic") return [];
2786
+ return strategy === "default" ? DEFAULT_STRATEGIES.map((id) => [id, { id }]) : [[strategy, { id: strategy }]];
2787
+ }
2788
+ return [[getStrategyKey(strategy), strategy]];
2789
+ })).values()).sort((a, b) => {
2790
+ const aId = typeof a === "string" ? a : a.id;
2791
+ const bId = typeof b === "string" ? b : b.id;
2792
+ return aId.localeCompare(bId);
2793
+ });
2794
+ return {
2795
+ numTests: data.numTests,
2796
+ plugins: uniquePlugins,
2797
+ strategies,
2798
+ ...frameworks ? { frameworks } : {},
2799
+ ...data.delay ? { delay: data.delay } : {},
2800
+ ...data.entities ? { entities: data.entities } : {},
2801
+ ...data.injectVar ? { injectVar: data.injectVar } : {},
2802
+ ...data.language ? { language: data.language } : {},
2803
+ ...data.provider ? { provider: data.provider } : {},
2804
+ ...data.purpose ? { purpose: data.purpose } : {},
2805
+ ...data.contexts ? { contexts: data.contexts } : {},
2806
+ ...data.excludeTargetOutputFromAgenticAttackGeneration ? { excludeTargetOutputFromAgenticAttackGeneration: data.excludeTargetOutputFromAgenticAttackGeneration } : {},
2807
+ ...data.tracing ? { tracing: data.tracing } : {},
2808
+ ...data.graderExamples ? { graderExamples: data.graderExamples } : {}
2809
+ };
2810
+ });
2811
+ function assert() {}
2812
+ assert();
2813
+ //#endregion
2814
+ //#region src/validators/shared.ts
2815
+ const NunjucksFilterMapSchema = zod.z.record(zod.z.string(), zod.z.custom((v) => typeof v === "function"));
2816
+ //#endregion
2817
+ //#region src/types/providers.ts
2818
+ function isApiProvider(provider) {
2819
+ return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "function";
2820
+ }
2821
+ function isProviderOptions(provider) {
2822
+ return typeof provider === "object" && provider != null && "id" in provider && typeof provider.id === "string";
2823
+ }
2824
+ //#endregion
2825
+ //#region src/types/index.ts
2826
+ const CommandLineOptionsSchema = zod.z.object({
2827
+ description: zod.z.string().optional(),
2828
+ prompts: zod.z.array(zod.z.string()).optional(),
2829
+ providers: zod.z.array(zod.z.string()),
2830
+ output: zod.z.array(zod.z.string()),
2831
+ maxConcurrency: zod.z.coerce.number().int().positive().optional(),
2832
+ repeat: zod.z.coerce.number().int().positive().optional(),
2833
+ delay: zod.z.coerce.number().int().nonnegative().prefault(0),
2834
+ vars: zod.z.string().optional(),
2835
+ tests: zod.z.string().optional(),
2836
+ config: zod.z.array(zod.z.string()).optional(),
2837
+ assertions: zod.z.string().optional(),
2838
+ modelOutputs: zod.z.string().optional(),
2839
+ verbose: zod.z.boolean().optional(),
2840
+ grader: zod.z.string().optional(),
2841
+ tableCellMaxLength: zod.z.coerce.number().int().positive().optional(),
2842
+ write: zod.z.boolean().optional(),
2843
+ cache: zod.z.boolean().optional(),
2844
+ table: zod.z.boolean().optional(),
2845
+ share: zod.z.boolean().optional(),
2846
+ noShare: zod.z.boolean().optional(),
2847
+ progressBar: zod.z.boolean().optional(),
2848
+ watch: zod.z.boolean().optional(),
2849
+ filterErrorsOnly: zod.z.string().optional(),
2850
+ filterFailing: zod.z.string().optional(),
2851
+ filterFailingOnly: zod.z.string().optional(),
2852
+ filterFirstN: zod.z.coerce.number().int().positive().optional(),
2853
+ filterMetadata: zod.z.union([zod.z.string(), zod.z.array(zod.z.string())]).optional(),
2854
+ filterPattern: zod.z.string().optional(),
2855
+ filterPrompts: zod.z.string().optional(),
2856
+ filterProviders: zod.z.string().optional(),
2857
+ filterSample: zod.z.coerce.number().int().positive().optional(),
2858
+ filterTargets: zod.z.string().optional(),
2859
+ var: zod.z.record(zod.z.string(), zod.z.string()).optional(),
2860
+ generateSuggestions: zod.z.boolean().optional(),
2861
+ promptPrefix: zod.z.string().optional(),
2862
+ promptSuffix: zod.z.string().optional(),
2863
+ retryErrors: zod.z.boolean().optional(),
2864
+ envPath: zod.z.union([zod.z.string(), zod.z.array(zod.z.string())]).optional(),
2865
+ extension: zod.z.array(zod.z.string()).optional()
2866
+ });
2867
+ const GradingConfigSchema = zod.z.object({
2868
+ rubricPrompt: zod.z.union([
2869
+ zod.z.string(),
2870
+ zod.z.array(zod.z.string()),
2871
+ zod.z.array(zod.z.object({
2872
+ role: zod.z.string(),
2873
+ content: zod.z.string()
2874
+ }))
2875
+ ]).optional(),
2876
+ provider: zod.z.union([
2877
+ zod.z.string(),
2878
+ zod.z.any(),
2879
+ zod.z.record(zod.z.string(), zod.z.union([zod.z.string(), zod.z.any()])).optional()
2880
+ ]).optional(),
2881
+ factuality: zod.z.object({
2882
+ subset: zod.z.number().optional(),
2883
+ superset: zod.z.number().optional(),
2884
+ agree: zod.z.number().optional(),
2885
+ disagree: zod.z.number().optional(),
2886
+ differButFactual: zod.z.number().optional()
2887
+ }).optional()
2888
+ });
2889
+ const OutputConfigSchema = zod.z.object({
2890
+ postprocess: zod.z.string().optional(),
2891
+ transform: zod.z.string().optional(),
2892
+ transformVars: zod.z.string().optional(),
2893
+ storeOutputAs: zod.z.string().optional()
2894
+ });
2895
+ const EvaluateOptionsSchema = zod.z.object({
2896
+ cache: zod.z.boolean().optional(),
2897
+ delay: zod.z.number().optional(),
2898
+ eventSource: zod.z.string().optional(),
2899
+ generateSuggestions: zod.z.boolean().optional(),
2900
+ interactiveProviders: zod.z.boolean().optional(),
2901
+ maxConcurrency: zod.z.number().optional(),
2902
+ progressCallback: zod.z.custom((v) => typeof v === "function").optional(),
2903
+ repeat: zod.z.number().optional(),
2904
+ showProgressBar: zod.z.boolean().optional(),
2905
+ timeoutMs: zod.z.number().optional(),
2906
+ maxEvalTimeMs: zod.z.number().optional(),
2907
+ isRedteam: zod.z.boolean().optional(),
2908
+ silent: zod.z.boolean().optional()
2909
+ });
2910
+ const PromptMetricsSchema = zod.z.object({
2911
+ score: zod.z.number(),
2912
+ testPassCount: zod.z.number(),
2913
+ testFailCount: zod.z.number(),
2914
+ testErrorCount: zod.z.number(),
2915
+ assertPassCount: zod.z.number(),
2916
+ assertFailCount: zod.z.number(),
2917
+ totalLatencyMs: zod.z.number(),
2918
+ tokenUsage: BaseTokenUsageSchema,
2919
+ namedScores: zod.z.record(zod.z.string(), zod.z.number()),
2920
+ namedScoresCount: zod.z.record(zod.z.string(), zod.z.number()),
2921
+ namedScoreWeights: zod.z.record(zod.z.string(), zod.z.number()).optional(),
2922
+ redteam: zod.z.object({
2923
+ pluginPassCount: zod.z.record(zod.z.string(), zod.z.number()),
2924
+ pluginFailCount: zod.z.record(zod.z.string(), zod.z.number()),
2925
+ strategyPassCount: zod.z.record(zod.z.string(), zod.z.number()),
2926
+ strategyFailCount: zod.z.record(zod.z.string(), zod.z.number())
2927
+ }).optional(),
2928
+ cost: zod.z.number()
2929
+ });
2930
+ const CompletedPromptSchema = PromptSchema.extend({
2931
+ provider: zod.z.string(),
2932
+ metrics: PromptMetricsSchema.optional()
2933
+ });
2934
+ const ResultFailureReason = {
2935
+ NONE: 0,
2936
+ ASSERT: 1,
2937
+ ERROR: 2
2938
+ };
2939
+ const validResultFailureReasons = new Set(Object.values(ResultFailureReason));
2940
+ function isResultFailureReason(value) {
2941
+ return validResultFailureReasons.has(value);
2942
+ }
2943
+ function isGradingResult(result) {
2944
+ return typeof result === "object" && result !== null && typeof result.pass === "boolean" && typeof result.score === "number" && typeof result.reason === "string" && (typeof result.namedScores === "undefined" || typeof result.namedScores === "object") && (typeof result.namedScoreWeights === "undefined" || typeof result.namedScoreWeights === "object") && (typeof result.tokensUsed === "undefined" || typeof result.tokensUsed === "object") && (typeof result.componentResults === "undefined" || Array.isArray(result.componentResults)) && (typeof result.assertion === "undefined" || result.assertion === null || typeof result.assertion === "object") && (typeof result.comment === "undefined" || typeof result.comment === "string");
2945
+ }
2946
+ const BaseAssertionTypesSchema = zod.z.enum([
2947
+ "answer-relevance",
2948
+ "bleu",
2949
+ "classifier",
2950
+ "contains",
2951
+ "contains-all",
2952
+ "contains-any",
2953
+ "contains-html",
2954
+ "contains-json",
2955
+ "contains-sql",
2956
+ "contains-xml",
2957
+ "context-faithfulness",
2958
+ "context-recall",
2959
+ "context-relevance",
2960
+ "conversation-relevance",
2961
+ "cost",
2962
+ "equals",
2963
+ "factuality",
2964
+ "finish-reason",
2965
+ "g-eval",
2966
+ "gleu",
2967
+ "guardrails",
2968
+ "icontains",
2969
+ "icontains-all",
2970
+ "icontains-any",
2971
+ "is-html",
2972
+ "is-json",
2973
+ "is-refusal",
2974
+ "is-sql",
2975
+ "is-valid-function-call",
2976
+ "is-valid-openai-function-call",
2977
+ "is-valid-openai-tools-call",
2978
+ "is-xml",
2979
+ "javascript",
2980
+ "latency",
2981
+ "levenshtein",
2982
+ "llm-rubric",
2983
+ "pi",
2984
+ "meteor",
2985
+ "model-graded-closedqa",
2986
+ "model-graded-factuality",
2987
+ "moderation",
2988
+ "perplexity",
2989
+ "perplexity-score",
2990
+ "python",
2991
+ "regex",
2992
+ "rouge-n",
2993
+ "ruby",
2994
+ "similar",
2995
+ "similar:cosine",
2996
+ "similar:dot",
2997
+ "similar:euclidean",
2998
+ "starts-with",
2999
+ "tool-call-f1",
3000
+ "skill-used",
3001
+ "trajectory:goal-success",
3002
+ "trajectory:tool-args-match",
3003
+ "trajectory:step-count",
3004
+ "trajectory:tool-sequence",
3005
+ "trajectory:tool-used",
3006
+ "trace-error-spans",
3007
+ "trace-span-count",
3008
+ "trace-span-duration",
3009
+ "search-rubric",
3010
+ "webhook",
3011
+ "word-count"
3012
+ ]);
3013
+ const SpecialAssertionTypesSchema = zod.z.enum([
3014
+ "select-best",
3015
+ "human",
3016
+ "max-score"
3017
+ ]);
3018
+ const NotPrefixedAssertionTypesSchema = BaseAssertionTypesSchema.transform((baseType) => `not-${baseType}`);
3019
+ const AssertionTypeSchema = zod.z.union([
3020
+ BaseAssertionTypesSchema,
3021
+ NotPrefixedAssertionTypesSchema,
3022
+ SpecialAssertionTypesSchema,
3023
+ zod.z.custom()
3024
+ ]);
3025
+ const AssertionSetSchema = zod.z.object({
3026
+ type: zod.z.literal("assert-set"),
3027
+ assert: zod.z.array(zod.z.lazy(() => AssertionSchema)),
3028
+ weight: zod.z.number().optional(),
3029
+ metric: zod.z.string().optional(),
3030
+ threshold: zod.z.number().optional(),
3031
+ config: zod.z.record(zod.z.string(), zod.z.any()).optional()
3032
+ });
3033
+ const AssertionSchema = zod.z.object({
3034
+ type: AssertionTypeSchema,
3035
+ value: zod.z.custom().optional(),
3036
+ config: zod.z.record(zod.z.string(), zod.z.any()).optional(),
3037
+ threshold: zod.z.number().optional(),
3038
+ weight: zod.z.number().optional(),
3039
+ provider: zod.z.custom().optional(),
3040
+ rubricPrompt: zod.z.custom().optional(),
3041
+ metric: zod.z.string().optional(),
3042
+ transform: zod.z.string().optional(),
3043
+ contextTransform: zod.z.string().optional()
3044
+ });
3045
+ /**
3046
+ * Schema for validating individual assertions (regular or assert-set).
3047
+ * Used for runtime validation of user-provided config.
3048
+ */
3049
+ const AssertionOrSetSchema = zod.z.union([AssertionSetSchema, AssertionSchema]);
3050
+ const TestCasesWithMetadataPromptSchema = zod.z.object({
3051
+ prompt: CompletedPromptSchema,
3052
+ id: zod.z.string(),
3053
+ evalId: zod.z.string()
3054
+ });
3055
+ const ProviderPromptMapSchema = zod.z.record(zod.z.string(), zod.z.union([zod.z.string().transform((value) => [value]), zod.z.array(zod.z.string())]));
3056
+ const MetadataSchema = zod.z.record(zod.z.string(), zod.z.any());
3057
+ function isValidVarValue(value) {
3058
+ if (value === null || value === void 0) return false;
3059
+ const type = typeof value;
3060
+ if (type === "symbol" || type === "function") return false;
3061
+ return type === "string" || type === "number" || type === "boolean" || type === "object";
3062
+ }
3063
+ const VarsSchema = zod.z.custom((data) => {
3064
+ if (typeof data !== "object" || data === null || Array.isArray(data)) return false;
3065
+ if (Object.getPrototypeOf(data) !== Object.prototype && Object.getPrototypeOf(data) !== null) return false;
3066
+ return Object.values(data).every(isValidVarValue);
3067
+ });
3068
+ const TestCaseSchema = zod.z.object({
3069
+ description: zod.z.string().optional(),
3070
+ vars: VarsSchema.optional(),
3071
+ provider: zod.z.union([
3072
+ zod.z.string(),
3073
+ ProviderOptionsSchema,
3074
+ ApiProviderSchema
3075
+ ]).optional(),
3076
+ providers: zod.z.array(zod.z.string()).optional(),
3077
+ prompts: zod.z.array(zod.z.string()).optional(),
3078
+ providerOutput: zod.z.union([zod.z.string(), zod.z.record(zod.z.string(), zod.z.unknown())]).optional(),
3079
+ assert: zod.z.array(zod.z.union([AssertionSetSchema, AssertionSchema])).optional(),
3080
+ assertScoringFunction: zod.z.union([zod.z.string().regex(new RegExp(`^file://.*\\.(${require_fileExtensions.JAVASCRIPT_EXTENSIONS?.join("|")}|py)(?::[\\w.]+)?$`)), zod.z.custom()]).optional(),
3081
+ options: zod.z.object({
3082
+ ...PromptConfigSchema.shape,
3083
+ ...OutputConfigSchema.shape,
3084
+ ...GradingConfigSchema.shape,
3085
+ disableVarExpansion: zod.z.boolean().optional(),
3086
+ disableConversationVar: zod.z.boolean().optional(),
3087
+ runSerially: zod.z.boolean().optional()
3088
+ }).catchall(zod.z.any()).optional(),
3089
+ threshold: zod.z.number().optional(),
3090
+ metadata: zod.z.object({
3091
+ pluginConfig: zod.z.custom().optional(),
3092
+ strategyConfig: zod.z.custom().optional()
3093
+ }).catchall(zod.z.any()).optional()
3094
+ });
3095
+ const TestCaseWithVarsFileSchema = TestCaseSchema.extend({ vars: zod.z.union([
3096
+ VarsSchema,
3097
+ zod.z.string(),
3098
+ zod.z.array(zod.z.string())
3099
+ ]).optional() });
3100
+ const TestCasesWithMetadataSchema = zod.z.object({
3101
+ id: zod.z.string(),
3102
+ testCases: zod.z.union([zod.z.string(), zod.z.array(zod.z.union([zod.z.string(), TestCaseSchema]))]),
3103
+ recentEvalDate: zod.z.date(),
3104
+ recentEvalId: zod.z.string(),
3105
+ count: zod.z.number(),
3106
+ prompts: zod.z.array(TestCasesWithMetadataPromptSchema)
3107
+ });
3108
+ const ScenarioSchema = zod.z.object({
3109
+ description: zod.z.string().optional(),
3110
+ config: zod.z.array(TestCaseSchema.partial()),
3111
+ tests: zod.z.array(TestCaseSchema)
3112
+ });
3113
+ const AtomicTestCaseSchema = TestCaseSchema.extend({ vars: VarsSchema.optional() }).strict();
3114
+ /**
3115
+ * Configuration schema for test generators that accept parameters
3116
+ *
3117
+ * @example
3118
+ * ```yaml
3119
+ * tests:
3120
+ * - path: file://test_cases.py:generate_tests
3121
+ * config:
3122
+ * dataset: truthfulqa
3123
+ * split: validation
3124
+ * max_rows: 100
3125
+ * ```
3126
+ */
3127
+ const TestGeneratorConfigSchema = zod.z.object({
3128
+ path: zod.z.string(),
3129
+ config: zod.z.record(zod.z.string(), zod.z.union([
3130
+ zod.z.string(),
3131
+ zod.z.number(),
3132
+ zod.z.boolean(),
3133
+ zod.z.array(zod.z.union([
3134
+ zod.z.string(),
3135
+ zod.z.number(),
3136
+ zod.z.boolean()
3137
+ ])),
3138
+ zod.z.record(zod.z.string(), zod.z.any()),
3139
+ zod.z.any()
3140
+ ])).optional()
3141
+ });
3142
+ const DerivedMetricSchema = zod.z.object({
3143
+ name: zod.z.string(),
3144
+ value: zod.z.union([zod.z.string(), zod.z.function({
3145
+ input: [zod.z.record(zod.z.string(), zod.z.number()), zod.z.custom()],
3146
+ output: zod.z.number()
3147
+ })])
3148
+ });
3149
+ const TestSuiteSchema = zod.z.object({
3150
+ tags: zod.z.record(zod.z.string(), zod.z.string()).optional(),
3151
+ description: zod.z.string().optional(),
3152
+ providers: zod.z.array(ApiProviderSchema),
3153
+ prompts: zod.z.array(PromptSchema),
3154
+ providerPromptMap: ProviderPromptMapSchema.optional(),
3155
+ tests: zod.z.array(TestCaseSchema).optional(),
3156
+ scenarios: zod.z.array(ScenarioSchema).optional(),
3157
+ defaultTest: zod.z.union([zod.z.string().refine((val) => val.startsWith("file://"), { error: "defaultTest string must start with file://" }), TestCaseSchema.omit({ description: true })]).optional(),
3158
+ nunjucksFilters: NunjucksFilterMapSchema.optional(),
3159
+ env: ProviderEnvOverridesSchema.optional(),
3160
+ derivedMetrics: zod.z.array(DerivedMetricSchema).optional(),
3161
+ extensions: zod.z.array(zod.z.string().refine((value) => value.startsWith("file://"), { error: "Extension must start with file://" }).refine((value) => {
3162
+ const parts = value.split(":");
3163
+ return parts.length === 3 && parts.every((part) => part.trim() !== "");
3164
+ }, { error: "Extension must be of the form file://path/to/file.py:function_name" }).refine((value) => {
3165
+ const parts = value.split(":");
3166
+ return (parts[1].endsWith(".py") || require_fileExtensions.isJavascriptFile(parts[1])) && (parts.length === 3 || parts.length === 2);
3167
+ }, { error: "Extension must be a python (.py) or javascript (.js, .ts, .mjs, .cjs, etc.) file followed by a colon and function name" })).nullable().optional(),
3168
+ redteam: zod.z.custom().optional(),
3169
+ tracing: zod.z.object({
3170
+ enabled: zod.z.boolean(),
3171
+ otlp: zod.z.object({
3172
+ http: zod.z.object({
3173
+ enabled: zod.z.boolean(),
3174
+ port: zod.z.number(),
3175
+ host: zod.z.string().optional(),
3176
+ acceptFormats: zod.z.array(zod.z.enum(["protobuf", "json"])).optional()
3177
+ }).optional(),
3178
+ grpc: zod.z.object({
3179
+ enabled: zod.z.boolean(),
3180
+ port: zod.z.number()
3181
+ }).optional()
3182
+ }).optional(),
3183
+ storage: zod.z.object({
3184
+ type: zod.z.string(),
3185
+ retentionDays: zod.z.number()
3186
+ }).optional(),
3187
+ forwarding: zod.z.object({
3188
+ enabled: zod.z.boolean(),
3189
+ endpoint: zod.z.string(),
3190
+ headers: zod.z.record(zod.z.string(), zod.z.string()).optional()
3191
+ }).optional()
3192
+ }).optional()
3193
+ });
3194
+ const TestSuiteConfigSchema = zod.z.object({
3195
+ tags: zod.z.record(zod.z.string(), zod.z.string()).optional(),
3196
+ description: zod.z.string().optional(),
3197
+ providers: ProvidersSchema,
3198
+ prompts: zod.z.union([
3199
+ zod.z.string(),
3200
+ zod.z.array(zod.z.union([
3201
+ zod.z.string(),
3202
+ zod.z.object({
3203
+ id: zod.z.string(),
3204
+ label: zod.z.string().optional(),
3205
+ raw: zod.z.string().optional()
3206
+ }),
3207
+ PromptSchema
3208
+ ])),
3209
+ zod.z.record(zod.z.string(), zod.z.string())
3210
+ ]),
3211
+ tests: zod.z.union([
3212
+ zod.z.string(),
3213
+ zod.z.array(zod.z.union([
3214
+ zod.z.string(),
3215
+ TestCaseSchema,
3216
+ TestGeneratorConfigSchema
3217
+ ])),
3218
+ TestGeneratorConfigSchema
3219
+ ]).optional(),
3220
+ scenarios: zod.z.array(zod.z.union([zod.z.string(), ScenarioSchema])).optional(),
3221
+ defaultTest: zod.z.union([zod.z.string().refine((val) => val.startsWith("file://"), { error: "defaultTest string must start with file://" }), TestCaseSchema.omit({ description: true })]).optional(),
3222
+ outputPath: zod.z.union([zod.z.string(), zod.z.array(zod.z.string())]).optional(),
3223
+ sharing: zod.z.union([zod.z.boolean(), zod.z.object({
3224
+ apiBaseUrl: zod.z.string().optional(),
3225
+ appBaseUrl: zod.z.string().optional()
3226
+ })]).optional(),
3227
+ nunjucksFilters: zod.z.record(zod.z.string(), zod.z.string()).optional(),
3228
+ env: zod.z.union([ProviderEnvOverridesSchema, zod.z.record(zod.z.string(), zod.z.union([
3229
+ zod.z.string(),
3230
+ zod.z.number().transform((n) => String(n)),
3231
+ zod.z.boolean().transform((b) => String(b))
3232
+ ]))]).optional(),
3233
+ derivedMetrics: zod.z.array(DerivedMetricSchema).optional(),
3234
+ extensions: zod.z.array(zod.z.string()).nullable().optional(),
3235
+ metadata: MetadataSchema.optional(),
3236
+ redteam: RedteamConfigSchema.optional(),
3237
+ writeLatestResults: zod.z.boolean().optional(),
3238
+ tracing: zod.z.object({
3239
+ enabled: zod.z.boolean().prefault(false),
3240
+ otlp: zod.z.object({
3241
+ http: zod.z.object({
3242
+ enabled: zod.z.boolean().prefault(true),
3243
+ port: zod.z.number().prefault(4318),
3244
+ host: zod.z.string().prefault("0.0.0.0"),
3245
+ acceptFormats: zod.z.array(zod.z.enum(["protobuf", "json"])).prefault(["json", "protobuf"])
3246
+ }).optional(),
3247
+ grpc: zod.z.object({
3248
+ enabled: zod.z.boolean().prefault(false),
3249
+ port: zod.z.number().prefault(4317)
3250
+ }).optional()
3251
+ }).optional(),
3252
+ storage: zod.z.object({
3253
+ type: zod.z.enum(["sqlite"]).prefault("sqlite"),
3254
+ retentionDays: zod.z.number().prefault(30)
3255
+ }).optional(),
3256
+ forwarding: zod.z.object({
3257
+ enabled: zod.z.boolean().prefault(false),
3258
+ endpoint: zod.z.string(),
3259
+ headers: zod.z.record(zod.z.string(), zod.z.string()).optional()
3260
+ }).optional()
3261
+ }).optional()
3262
+ });
3263
+ const UnifiedConfigSchema = TestSuiteConfigSchema.extend({
3264
+ evaluateOptions: EvaluateOptionsSchema.optional(),
3265
+ commandLineOptions: CommandLineOptionsSchema.partial().optional(),
3266
+ providers: ProvidersSchema.optional(),
3267
+ targets: ProvidersSchema.optional()
3268
+ }).refine((data) => {
3269
+ const hasTargets = data.targets !== void 0;
3270
+ const hasProviders = data.providers !== void 0;
3271
+ return hasTargets && !hasProviders || !hasTargets && hasProviders;
3272
+ }, { message: "Exactly one of 'targets' or 'providers' must be provided, but not both" }).transform((data) => {
3273
+ if (data.targets && !data.providers) {
3274
+ data.providers = data.targets;
3275
+ delete data.targets;
3276
+ }
3277
+ if (data.extensions === null || data.extensions === void 0 || Array.isArray(data.extensions) && data.extensions.length === 0) delete data.extensions;
3278
+ return data;
3279
+ });
3280
+ const OutputFileExtension = zod.z.enum([
3281
+ "csv",
3282
+ "html",
3283
+ "json",
3284
+ "jsonl",
3285
+ "txt",
3286
+ "xml",
3287
+ "yaml",
3288
+ "yml"
3289
+ ]);
3290
+ const EvalResultsFilterMode = zod.z.enum([
3291
+ "all",
3292
+ "failures",
3293
+ "different",
3294
+ "highlights",
3295
+ "errors",
3296
+ "passes",
3297
+ "user-rated"
3298
+ ]);
3299
+ //#endregion
3300
+ Object.defineProperty(exports, "AGENTIC_STRATEGIES", {
3301
+ enumerable: true,
3302
+ get: function() {
3303
+ return AGENTIC_STRATEGIES;
3304
+ }
3305
+ });
3306
+ Object.defineProperty(exports, "ALIASED_PLUGIN_MAPPINGS", {
3307
+ enumerable: true,
3308
+ get: function() {
3309
+ return ALIASED_PLUGIN_MAPPINGS;
3310
+ }
3311
+ });
3312
+ Object.defineProperty(exports, "AssertionOrSetSchema", {
3313
+ enumerable: true,
3314
+ get: function() {
3315
+ return AssertionOrSetSchema;
3316
+ }
3317
+ });
3318
+ Object.defineProperty(exports, "AssertionSchema", {
3319
+ enumerable: true,
3320
+ get: function() {
3321
+ return AssertionSchema;
3322
+ }
3323
+ });
3324
+ Object.defineProperty(exports, "AssertionSetSchema", {
3325
+ enumerable: true,
3326
+ get: function() {
3327
+ return AssertionSetSchema;
3328
+ }
3329
+ });
3330
+ Object.defineProperty(exports, "AssertionTypeSchema", {
3331
+ enumerable: true,
3332
+ get: function() {
3333
+ return AssertionTypeSchema;
3334
+ }
3335
+ });
3336
+ Object.defineProperty(exports, "AtomicTestCaseSchema", {
3337
+ enumerable: true,
3338
+ get: function() {
3339
+ return AtomicTestCaseSchema;
3340
+ }
3341
+ });
3342
+ Object.defineProperty(exports, "BIAS_PLUGINS", {
3343
+ enumerable: true,
3344
+ get: function() {
3345
+ return BIAS_PLUGINS;
3346
+ }
3347
+ });
3348
+ Object.defineProperty(exports, "BaseAssertionTypesSchema", {
3349
+ enumerable: true,
3350
+ get: function() {
3351
+ return BaseAssertionTypesSchema;
3352
+ }
3353
+ });
3354
+ Object.defineProperty(exports, "BaseTokenUsageSchema", {
3355
+ enumerable: true,
3356
+ get: function() {
3357
+ return BaseTokenUsageSchema;
3358
+ }
3359
+ });
3360
+ Object.defineProperty(exports, "CommandLineOptionsSchema", {
3361
+ enumerable: true,
3362
+ get: function() {
3363
+ return CommandLineOptionsSchema;
3364
+ }
3365
+ });
3366
+ Object.defineProperty(exports, "CompletedPromptSchema", {
3367
+ enumerable: true,
3368
+ get: function() {
3369
+ return CompletedPromptSchema;
3370
+ }
3371
+ });
3372
+ Object.defineProperty(exports, "CompletionTokenDetailsSchema", {
3373
+ enumerable: true,
3374
+ get: function() {
3375
+ return CompletionTokenDetailsSchema;
3376
+ }
3377
+ });
3378
+ Object.defineProperty(exports, "ConversationMessageSchema", {
3379
+ enumerable: true,
3380
+ get: function() {
3381
+ return ConversationMessageSchema;
3382
+ }
3383
+ });
3384
+ Object.defineProperty(exports, "DATASET_EXEMPT_PLUGINS", {
3385
+ enumerable: true,
3386
+ get: function() {
3387
+ return DATASET_EXEMPT_PLUGINS;
3388
+ }
3389
+ });
3390
+ Object.defineProperty(exports, "DATASET_PLUGINS", {
3391
+ enumerable: true,
3392
+ get: function() {
3393
+ return DATASET_PLUGINS;
3394
+ }
3395
+ });
3396
+ Object.defineProperty(exports, "DEFAULT_PLUGINS", {
3397
+ enumerable: true,
3398
+ get: function() {
3399
+ return DEFAULT_PLUGINS;
3400
+ }
3401
+ });
3402
+ Object.defineProperty(exports, "DEFAULT_STRATEGIES", {
3403
+ enumerable: true,
3404
+ get: function() {
3405
+ return DEFAULT_STRATEGIES;
3406
+ }
3407
+ });
3408
+ Object.defineProperty(exports, "DerivedMetricSchema", {
3409
+ enumerable: true,
3410
+ get: function() {
3411
+ return DerivedMetricSchema;
3412
+ }
3413
+ });
3414
+ Object.defineProperty(exports, "EvalResultsFilterMode", {
3415
+ enumerable: true,
3416
+ get: function() {
3417
+ return EvalResultsFilterMode;
3418
+ }
3419
+ });
3420
+ Object.defineProperty(exports, "EvaluateOptionsSchema", {
3421
+ enumerable: true,
3422
+ get: function() {
3423
+ return EvaluateOptionsSchema;
3424
+ }
3425
+ });
3426
+ Object.defineProperty(exports, "FINANCIAL_PLUGINS", {
3427
+ enumerable: true,
3428
+ get: function() {
3429
+ return FINANCIAL_PLUGINS;
3430
+ }
3431
+ });
3432
+ Object.defineProperty(exports, "FOUNDATION_PLUGINS", {
3433
+ enumerable: true,
3434
+ get: function() {
3435
+ return FOUNDATION_PLUGINS;
3436
+ }
3437
+ });
3438
+ Object.defineProperty(exports, "GradingConfigSchema", {
3439
+ enumerable: true,
3440
+ get: function() {
3441
+ return GradingConfigSchema;
3442
+ }
3443
+ });
3444
+ Object.defineProperty(exports, "HARM_PLUGINS", {
3445
+ enumerable: true,
3446
+ get: function() {
3447
+ return HARM_PLUGINS;
3448
+ }
3449
+ });
3450
+ Object.defineProperty(exports, "INSURANCE_PLUGINS", {
3451
+ enumerable: true,
3452
+ get: function() {
3453
+ return INSURANCE_PLUGINS;
3454
+ }
3455
+ });
3456
+ Object.defineProperty(exports, "InputsSchema", {
3457
+ enumerable: true,
3458
+ get: function() {
3459
+ return InputsSchema;
3460
+ }
3461
+ });
3462
+ Object.defineProperty(exports, "LLAMA_GUARD_ENABLED_CATEGORIES", {
3463
+ enumerable: true,
3464
+ get: function() {
3465
+ return LLAMA_GUARD_ENABLED_CATEGORIES;
3466
+ }
3467
+ });
3468
+ Object.defineProperty(exports, "LLAMA_GUARD_REPLICATE_PROVIDER", {
3469
+ enumerable: true,
3470
+ get: function() {
3471
+ return LLAMA_GUARD_REPLICATE_PROVIDER;
3472
+ }
3473
+ });
3474
+ Object.defineProperty(exports, "MEDICAL_PLUGINS", {
3475
+ enumerable: true,
3476
+ get: function() {
3477
+ return MEDICAL_PLUGINS;
3478
+ }
3479
+ });
3480
+ Object.defineProperty(exports, "MULTI_INPUT_EXCLUDED_PLUGINS", {
3481
+ enumerable: true,
3482
+ get: function() {
3483
+ return MULTI_INPUT_EXCLUDED_PLUGINS;
3484
+ }
3485
+ });
3486
+ Object.defineProperty(exports, "MULTI_INPUT_VAR", {
3487
+ enumerable: true,
3488
+ get: function() {
3489
+ return MULTI_INPUT_VAR;
3490
+ }
3491
+ });
3492
+ Object.defineProperty(exports, "MULTI_TURN_STRATEGIES", {
3493
+ enumerable: true,
3494
+ get: function() {
3495
+ return MULTI_TURN_STRATEGIES;
3496
+ }
3497
+ });
3498
+ Object.defineProperty(exports, "NotPrefixedAssertionTypesSchema", {
3499
+ enumerable: true,
3500
+ get: function() {
3501
+ return NotPrefixedAssertionTypesSchema;
3502
+ }
3503
+ });
3504
+ Object.defineProperty(exports, "OutputConfigSchema", {
3505
+ enumerable: true,
3506
+ get: function() {
3507
+ return OutputConfigSchema;
3508
+ }
3509
+ });
3510
+ Object.defineProperty(exports, "OutputFileExtension", {
3511
+ enumerable: true,
3512
+ get: function() {
3513
+ return OutputFileExtension;
3514
+ }
3515
+ });
3516
+ Object.defineProperty(exports, "PHARMACY_PLUGINS", {
3517
+ enumerable: true,
3518
+ get: function() {
3519
+ return PHARMACY_PLUGINS;
3520
+ }
3521
+ });
3522
+ Object.defineProperty(exports, "PII_PLUGINS", {
3523
+ enumerable: true,
3524
+ get: function() {
3525
+ return PII_PLUGINS;
3526
+ }
3527
+ });
3528
+ Object.defineProperty(exports, "PLUGIN_CATEGORIES", {
3529
+ enumerable: true,
3530
+ get: function() {
3531
+ return PLUGIN_CATEGORIES;
3532
+ }
3533
+ });
3534
+ Object.defineProperty(exports, "PartialGenerationError", {
3535
+ enumerable: true,
3536
+ get: function() {
3537
+ return PartialGenerationError;
3538
+ }
3539
+ });
3540
+ Object.defineProperty(exports, "PluginConfigSchema", {
3541
+ enumerable: true,
3542
+ get: function() {
3543
+ return PluginConfigSchema;
3544
+ }
3545
+ });
3546
+ Object.defineProperty(exports, "PolicyObjectSchema", {
3547
+ enumerable: true,
3548
+ get: function() {
3549
+ return PolicyObjectSchema;
3550
+ }
3551
+ });
3552
+ Object.defineProperty(exports, "PromptSchema", {
3553
+ enumerable: true,
3554
+ get: function() {
3555
+ return PromptSchema;
3556
+ }
3557
+ });
3558
+ Object.defineProperty(exports, "ProviderOptionsSchema", {
3559
+ enumerable: true,
3560
+ get: function() {
3561
+ return ProviderOptionsSchema;
3562
+ }
3563
+ });
3564
+ Object.defineProperty(exports, "ProvidersSchema", {
3565
+ enumerable: true,
3566
+ get: function() {
3567
+ return ProvidersSchema;
3568
+ }
3569
+ });
3570
+ Object.defineProperty(exports, "REDTEAM_PROVIDER_HARM_PLUGINS", {
3571
+ enumerable: true,
3572
+ get: function() {
3573
+ return REDTEAM_PROVIDER_HARM_PLUGINS;
3574
+ }
3575
+ });
3576
+ Object.defineProperty(exports, "REMOTE_ONLY_PLUGIN_IDS", {
3577
+ enumerable: true,
3578
+ get: function() {
3579
+ return REMOTE_ONLY_PLUGIN_IDS;
3580
+ }
3581
+ });
3582
+ Object.defineProperty(exports, "RedteamConfigSchema", {
3583
+ enumerable: true,
3584
+ get: function() {
3585
+ return RedteamConfigSchema;
3586
+ }
3587
+ });
3588
+ Object.defineProperty(exports, "ResultFailureReason", {
3589
+ enumerable: true,
3590
+ get: function() {
3591
+ return ResultFailureReason;
3592
+ }
3593
+ });
3594
+ Object.defineProperty(exports, "STRATEGY_COLLECTIONS", {
3595
+ enumerable: true,
3596
+ get: function() {
3597
+ return STRATEGY_COLLECTIONS;
3598
+ }
3599
+ });
3600
+ Object.defineProperty(exports, "STRATEGY_COLLECTION_MAPPINGS", {
3601
+ enumerable: true,
3602
+ get: function() {
3603
+ return STRATEGY_COLLECTION_MAPPINGS;
3604
+ }
3605
+ });
3606
+ Object.defineProperty(exports, "STRATEGY_EXEMPT_PLUGINS", {
3607
+ enumerable: true,
3608
+ get: function() {
3609
+ return STRATEGY_EXEMPT_PLUGINS;
3610
+ }
3611
+ });
3612
+ Object.defineProperty(exports, "ScenarioSchema", {
3613
+ enumerable: true,
3614
+ get: function() {
3615
+ return ScenarioSchema;
3616
+ }
3617
+ });
3618
+ Object.defineProperty(exports, "Severity", {
3619
+ enumerable: true,
3620
+ get: function() {
3621
+ return Severity;
3622
+ }
3623
+ });
3624
+ Object.defineProperty(exports, "SpecialAssertionTypesSchema", {
3625
+ enumerable: true,
3626
+ get: function() {
3627
+ return SpecialAssertionTypesSchema;
3628
+ }
3629
+ });
3630
+ Object.defineProperty(exports, "StrategyConfigSchema", {
3631
+ enumerable: true,
3632
+ get: function() {
3633
+ return StrategyConfigSchema;
3634
+ }
3635
+ });
3636
+ Object.defineProperty(exports, "TELECOM_PLUGINS", {
3637
+ enumerable: true,
3638
+ get: function() {
3639
+ return TELECOM_PLUGINS;
3640
+ }
3641
+ });
3642
+ Object.defineProperty(exports, "TestCaseSchema", {
3643
+ enumerable: true,
3644
+ get: function() {
3645
+ return TestCaseSchema;
3646
+ }
3647
+ });
3648
+ Object.defineProperty(exports, "TestCaseWithVarsFileSchema", {
3649
+ enumerable: true,
3650
+ get: function() {
3651
+ return TestCaseWithVarsFileSchema;
3652
+ }
3653
+ });
3654
+ Object.defineProperty(exports, "TestCasesWithMetadataPromptSchema", {
3655
+ enumerable: true,
3656
+ get: function() {
3657
+ return TestCasesWithMetadataPromptSchema;
3658
+ }
3659
+ });
3660
+ Object.defineProperty(exports, "TestCasesWithMetadataSchema", {
3661
+ enumerable: true,
3662
+ get: function() {
3663
+ return TestCasesWithMetadataSchema;
3664
+ }
3665
+ });
3666
+ Object.defineProperty(exports, "TestGeneratorConfigSchema", {
3667
+ enumerable: true,
3668
+ get: function() {
3669
+ return TestGeneratorConfigSchema;
3670
+ }
3671
+ });
3672
+ Object.defineProperty(exports, "TestSuiteConfigSchema", {
3673
+ enumerable: true,
3674
+ get: function() {
3675
+ return TestSuiteConfigSchema;
3676
+ }
3677
+ });
3678
+ Object.defineProperty(exports, "TestSuiteSchema", {
3679
+ enumerable: true,
3680
+ get: function() {
3681
+ return TestSuiteSchema;
3682
+ }
3683
+ });
3684
+ Object.defineProperty(exports, "UNALIGNED_PROVIDER_HARM_PLUGINS", {
3685
+ enumerable: true,
3686
+ get: function() {
3687
+ return UNALIGNED_PROVIDER_HARM_PLUGINS;
3688
+ }
3689
+ });
3690
+ Object.defineProperty(exports, "UnifiedConfigSchema", {
3691
+ enumerable: true,
3692
+ get: function() {
3693
+ return UnifiedConfigSchema;
3694
+ }
3695
+ });
3696
+ Object.defineProperty(exports, "VarsSchema", {
3697
+ enumerable: true,
3698
+ get: function() {
3699
+ return VarsSchema;
3700
+ }
3701
+ });
3702
+ Object.defineProperty(exports, "categoryAliases", {
3703
+ enumerable: true,
3704
+ get: function() {
3705
+ return categoryAliases;
3706
+ }
3707
+ });
3708
+ Object.defineProperty(exports, "getDefaultNFanout", {
3709
+ enumerable: true,
3710
+ get: function() {
3711
+ return getDefaultNFanout;
3712
+ }
3713
+ });
3714
+ Object.defineProperty(exports, "isApiProvider", {
3715
+ enumerable: true,
3716
+ get: function() {
3717
+ return isApiProvider;
3718
+ }
3719
+ });
3720
+ Object.defineProperty(exports, "isCustomStrategy", {
3721
+ enumerable: true,
3722
+ get: function() {
3723
+ return isCustomStrategy;
3724
+ }
3725
+ });
3726
+ Object.defineProperty(exports, "isFanoutStrategy", {
3727
+ enumerable: true,
3728
+ get: function() {
3729
+ return isFanoutStrategy;
3730
+ }
3731
+ });
3732
+ Object.defineProperty(exports, "isGradingResult", {
3733
+ enumerable: true,
3734
+ get: function() {
3735
+ return isGradingResult;
3736
+ }
3737
+ });
3738
+ Object.defineProperty(exports, "isProviderOptions", {
3739
+ enumerable: true,
3740
+ get: function() {
3741
+ return isProviderOptions;
3742
+ }
3743
+ });
3744
+ Object.defineProperty(exports, "isResultFailureReason", {
3745
+ enumerable: true,
3746
+ get: function() {
3747
+ return isResultFailureReason;
3748
+ }
3749
+ });
3750
+ Object.defineProperty(exports, "isValidReusablePolicyId", {
3751
+ enumerable: true,
3752
+ get: function() {
3753
+ return isValidReusablePolicyId;
3754
+ }
3755
+ });
3756
+ Object.defineProperty(exports, "pluginDescriptions", {
3757
+ enumerable: true,
3758
+ get: function() {
3759
+ return pluginDescriptions;
3760
+ }
3761
+ });
3762
+ Object.defineProperty(exports, "riskCategorySeverityMap", {
3763
+ enumerable: true,
3764
+ get: function() {
3765
+ return riskCategorySeverityMap;
3766
+ }
3767
+ });
3768
+
3769
+ //# sourceMappingURL=types-5aqHpBwE.cjs.map