@use-lattice/litmus 0.121.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +19 -0
- package/dist/src/accounts-Bt1oJb1Z.cjs +219 -0
- package/dist/src/accounts-DjOU8Rm3.js +178 -0
- package/dist/src/agentic-utils-D03IiXQc.js +153 -0
- package/dist/src/agentic-utils-Dh7xaMQM.cjs +180 -0
- package/dist/src/agents-C6BIMlZa.js +231 -0
- package/dist/src/agents-DvIpNX1L.cjs +666 -0
- package/dist/src/agents-ZP0RP9vV.cjs +231 -0
- package/dist/src/agents-maJXdjbR.js +665 -0
- package/dist/src/aimlapi-BTbQjG2E.cjs +30 -0
- package/dist/src/aimlapi-CwMxqfXP.js +30 -0
- package/dist/src/audio-BBUdvsde.cjs +97 -0
- package/dist/src/audio-D5DPZ7I-.js +97 -0
- package/dist/src/base-BEysXrkq.cjs +222 -0
- package/dist/src/base-C451JQfq.js +193 -0
- package/dist/src/blobs-BY8MDmpo.js +230 -0
- package/dist/src/blobs-BgcNn97m.cjs +256 -0
- package/dist/src/cache-BBE_lsTA.cjs +4 -0
- package/dist/src/cache-BkrqU5Ba.js +237 -0
- package/dist/src/cache-DsCxFlsZ.cjs +297 -0
- package/dist/src/chat-CPJWDP6a.cjs +289 -0
- package/dist/src/chat-CXX3xzkk.cjs +811 -0
- package/dist/src/chat-CcDgZFJ4.js +787 -0
- package/dist/src/chat-Dz5ZeGO2.js +289 -0
- package/dist/src/chatkit-Dw0mKkML.cjs +1158 -0
- package/dist/src/chatkit-swAIVuea.js +1157 -0
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/claude-agent-sdk-BXZJtOg6.js +379 -0
- package/dist/src/claude-agent-sdk-CkfyjDoG.cjs +383 -0
- package/dist/src/cloudflare-ai-BzpJcqUH.js +161 -0
- package/dist/src/cloudflare-ai-Cmy_R1y2.cjs +161 -0
- package/dist/src/cloudflare-gateway-B9tVQKok.cjs +272 -0
- package/dist/src/cloudflare-gateway-DrD3ew3H.js +272 -0
- package/dist/src/codex-sdk-Dezj9Nwm.js +1056 -0
- package/dist/src/codex-sdk-Dl9D4k5B.cjs +1060 -0
- package/dist/src/cometapi-C-9YvCHC.js +54 -0
- package/dist/src/cometapi-DHgDKoO2.cjs +54 -0
- package/dist/src/completion-B8Ctyxpr.js +120 -0
- package/dist/src/completion-Cxrt08sj.cjs +131 -0
- package/dist/src/createHash-BwgE13yv.cjs +27 -0
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/docker-BiqcTwLv.js +80 -0
- package/dist/src/docker-C7tEJnP-.cjs +80 -0
- package/dist/src/esm-C62Zofr1.cjs +409 -0
- package/dist/src/esm-DMVc93eh.js +379 -0
- package/dist/src/evalResult-C3NJPQOo.cjs +301 -0
- package/dist/src/evalResult-C7JJAPBb.js +295 -0
- package/dist/src/evalResult-DoVTZZWI.cjs +2 -0
- package/dist/src/extractor-DnMD3fwt.cjs +391 -0
- package/dist/src/extractor-DtlL28vL.js +374 -0
- package/dist/src/fetch-BTxakTSg.cjs +1133 -0
- package/dist/src/fetch-DQckpUFz.js +928 -0
- package/dist/src/fileExtensions-DnqA1y9x.js +85 -0
- package/dist/src/fileExtensions-bYh77CN8.cjs +114 -0
- package/dist/src/genaiTracer-CyZrmaK0.cjs +268 -0
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/graders-BNscxFrU.js +13644 -0
- package/dist/src/graders-D2oE9Msq.js +2 -0
- package/dist/src/graders-c0Ez_w-9.cjs +2 -0
- package/dist/src/graders-d0F2M3e9.cjs +14056 -0
- package/dist/src/image-0ZhE0VlR.cjs +280 -0
- package/dist/src/image-CWE1pdNv.js +257 -0
- package/dist/src/image-D9ZK6hwL.js +163 -0
- package/dist/src/image-DKZgZITg.cjs +163 -0
- package/dist/src/index.cjs +11366 -0
- package/dist/src/index.d.cts +19640 -0
- package/dist/src/index.d.ts +19641 -0
- package/dist/src/index.js +11306 -0
- package/dist/src/invariant-Ddh24eXh.js +25 -0
- package/dist/src/invariant-kfQ8Bu82.cjs +30 -0
- package/dist/src/knowledgeBase-BgPyGFUd.cjs +122 -0
- package/dist/src/knowledgeBase-DyHilYaP.js +122 -0
- package/dist/src/litellm-CyMeneHS.js +135 -0
- package/dist/src/litellm-DWDF73yF.cjs +135 -0
- package/dist/src/logger-C40ZGil9.js +717 -0
- package/dist/src/logger-DyfK9PBt.cjs +917 -0
- package/dist/src/luma-ray-BAU9X_ep.cjs +315 -0
- package/dist/src/luma-ray-nwVseBbv.js +313 -0
- package/dist/src/messages-B5ADWTTv.js +245 -0
- package/dist/src/messages-BCnZfqrS.cjs +257 -0
- package/dist/src/meteor-DLZZ3osF.cjs +134 -0
- package/dist/src/meteor-DUiCJRC-.js +134 -0
- package/dist/src/modelslab-00cveB8L.cjs +163 -0
- package/dist/src/modelslab-D9sCU_L7.js +163 -0
- package/dist/src/nova-reel-CTapvqYH.js +276 -0
- package/dist/src/nova-reel-DlWuuroF.cjs +278 -0
- package/dist/src/nova-sonic-5UPWfeMv.cjs +363 -0
- package/dist/src/nova-sonic-BhSwQNym.js +363 -0
- package/dist/src/openai-BWrJK9d8.cjs +52 -0
- package/dist/src/openai-DumO8WQn.js +47 -0
- package/dist/src/openclaw-B8brrjC_.cjs +577 -0
- package/dist/src/openclaw-Bkayww9q.js +571 -0
- package/dist/src/opencode-sdk-7xjoDNiM.cjs +562 -0
- package/dist/src/opencode-sdk-SGwAPxht.js +558 -0
- package/dist/src/otlpReceiver-CoAHfAN9.cjs +15 -0
- package/dist/src/otlpReceiver-oO3EQwI9.js +14 -0
- package/dist/src/providerRegistry-4yjhaEM8.js +45 -0
- package/dist/src/providerRegistry-DhV4rJIc.cjs +50 -0
- package/dist/src/providers-B5RJVG-7.cjs +33609 -0
- package/dist/src/providers-BdmZCLzV.js +33262 -0
- package/dist/src/providers-CxtRxn8e.js +2 -0
- package/dist/src/providers-DnQLNbx1.cjs +3 -0
- package/dist/src/pythonUtils-BD0druiM.cjs +275 -0
- package/dist/src/pythonUtils-IBhn5YGR.js +249 -0
- package/dist/src/quiverai-BDOwZBsM.cjs +213 -0
- package/dist/src/quiverai-D3JTF5lD.js +213 -0
- package/dist/src/responses-B2LCDCXZ.js +667 -0
- package/dist/src/responses-BvNm4Xv9.cjs +685 -0
- package/dist/src/rubyUtils-B0NwnfpY.cjs +245 -0
- package/dist/src/rubyUtils-BroxzZ7c.cjs +2 -0
- package/dist/src/rubyUtils-hqVw5UvJ.js +222 -0
- package/dist/src/sagemaker-Cno2V-Sx.js +689 -0
- package/dist/src/sagemaker-fV_KUgs5.cjs +691 -0
- package/dist/src/server-BOuAXb06.cjs +238 -0
- package/dist/src/server-CtI-EWzm.cjs +2 -0
- package/dist/src/server-Cy3DZymt.js +189 -0
- package/dist/src/slack-CP8xBePa.js +135 -0
- package/dist/src/slack-DSQ1yXVb.cjs +135 -0
- package/dist/src/store-BwDDaBjb.cjs +246 -0
- package/dist/src/store-DcbLC593.cjs +2 -0
- package/dist/src/store-IGpqMIkv.js +240 -0
- package/dist/src/tables-3Q2cL7So.cjs +373 -0
- package/dist/src/tables-Bi2fjr4W.js +288 -0
- package/dist/src/telemetry-Bg2WqF79.js +161 -0
- package/dist/src/telemetry-D0x6u5kX.cjs +166 -0
- package/dist/src/telemetry-DXNimrI0.cjs +2 -0
- package/dist/src/text-B_UCRPp2.js +22 -0
- package/dist/src/text-CW1cyrwj.cjs +33 -0
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/tokenUsageUtils-bVa1ga6f.cjs +173 -0
- package/dist/src/transcription-Cl_W16Pr.js +122 -0
- package/dist/src/transcription-yt1EecY8.cjs +124 -0
- package/dist/src/transform-BCtGrl_W.cjs +228 -0
- package/dist/src/transform-Bv6gG2MJ.cjs +1688 -0
- package/dist/src/transform-CY1wbpRy.js +1507 -0
- package/dist/src/transform-DU8rUL9P.cjs +2 -0
- package/dist/src/transform-yWaShiKr.js +216 -0
- package/dist/src/transformersAvailability-BGkzavwb.js +35 -0
- package/dist/src/transformersAvailability-DKoRtQLy.cjs +35 -0
- package/dist/src/types-5aqHpBwE.cjs +3769 -0
- package/dist/src/types-Bn6D9c4U.js +3300 -0
- package/dist/src/util-BkKlTkI2.js +293 -0
- package/dist/src/util-CTh0bfOm.cjs +1119 -0
- package/dist/src/util-D17oBwo7.cjs +328 -0
- package/dist/src/util-DsS_-v4p.js +613 -0
- package/dist/src/util-DuntT1Ga.js +951 -0
- package/dist/src/util-aWjdCYMI.cjs +667 -0
- package/dist/src/utils-CisQwpjA.js +94 -0
- package/dist/src/utils-yWamDvmz.cjs +123 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/drizzle/0000_lush_hellion.sql +36 -0
- package/drizzle/0001_wide_calypso.sql +3 -0
- package/drizzle/0002_tidy_juggernaut.sql +1 -0
- package/drizzle/0003_lively_naoko.sql +8 -0
- package/drizzle/0004_minor_peter_quill.sql +19 -0
- package/drizzle/0005_silky_millenium_guard.sql +2 -0
- package/drizzle/0006_harsh_caretaker.sql +42 -0
- package/drizzle/0007_cloudy_wong.sql +1 -0
- package/drizzle/0008_broad_boomer.sql +2 -0
- package/drizzle/0009_strong_marten_broadcloak.sql +19 -0
- package/drizzle/0010_needy_bishop.sql +11 -0
- package/drizzle/0011_moaning_millenium_guard.sql +1 -0
- package/drizzle/0012_late_marten_broadcloak.sql +2 -0
- package/drizzle/0013_previous_dormammu.sql +9 -0
- package/drizzle/0014_lazy_captain_universe.sql +2 -0
- package/drizzle/0015_zippy_wallop.sql +29 -0
- package/drizzle/0016_jazzy_zemo.sql +2 -0
- package/drizzle/0017_reflective_praxagora.sql +4 -0
- package/drizzle/0018_fat_vanisher.sql +22 -0
- package/drizzle/0019_new_clint_barton.sql +8 -0
- package/drizzle/0020_skinny_maverick.sql +1 -0
- package/drizzle/0021_mysterious_madelyne_pryor.sql +13 -0
- package/drizzle/0022_sleepy_ultimo.sql +25 -0
- package/drizzle/0023_wooden_mandrill.sql +2 -0
- package/drizzle/AGENTS.md +68 -0
- package/drizzle/CLAUDE.md +1 -0
- package/drizzle/meta/0000_snapshot.json +221 -0
- package/drizzle/meta/0001_snapshot.json +214 -0
- package/drizzle/meta/0002_snapshot.json +221 -0
- package/drizzle/meta/0005_snapshot.json +369 -0
- package/drizzle/meta/0006_snapshot.json +638 -0
- package/drizzle/meta/0007_snapshot.json +640 -0
- package/drizzle/meta/0008_snapshot.json +649 -0
- package/drizzle/meta/0009_snapshot.json +554 -0
- package/drizzle/meta/0010_snapshot.json +619 -0
- package/drizzle/meta/0011_snapshot.json +627 -0
- package/drizzle/meta/0012_snapshot.json +639 -0
- package/drizzle/meta/0013_snapshot.json +717 -0
- package/drizzle/meta/0014_snapshot.json +717 -0
- package/drizzle/meta/0015_snapshot.json +897 -0
- package/drizzle/meta/0016_snapshot.json +1031 -0
- package/drizzle/meta/0018_snapshot.json +1210 -0
- package/drizzle/meta/0019_snapshot.json +1165 -0
- package/drizzle/meta/0020_snapshot.json +1232 -0
- package/drizzle/meta/0021_snapshot.json +1311 -0
- package/drizzle/meta/0022_snapshot.json +1481 -0
- package/drizzle/meta/0023_snapshot.json +1496 -0
- package/drizzle/meta/_journal.json +174 -0
- package/package.json +240 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
const require_logger = require("./logger-DyfK9PBt.cjs");
|
|
2
|
+
const require_esm = require("./esm-C62Zofr1.cjs");
|
|
3
|
+
const require_transform = require("./transform-Bv6gG2MJ.cjs");
|
|
4
|
+
const require_util = require("./util-D17oBwo7.cjs");
|
|
5
|
+
const require_agentic_utils = require("./agentic-utils-Dh7xaMQM.cjs");
|
|
6
|
+
let node_fs = require("node:fs");
|
|
7
|
+
node_fs = require_logger.__toESM(node_fs);
|
|
8
|
+
let node_path = require("node:path");
|
|
9
|
+
node_path = require_logger.__toESM(node_path);
|
|
10
|
+
let dedent = require("dedent");
|
|
11
|
+
dedent = require_logger.__toESM(dedent);
|
|
12
|
+
let node_os = require("node:os");
|
|
13
|
+
node_os = require_logger.__toESM(node_os);
|
|
14
|
+
//#region src/providers/claude-agent-sdk.ts
|
|
15
|
+
function deriveSkillCalls(toolCalls) {
|
|
16
|
+
return toolCalls.filter((toolCall) => toolCall.name === "Skill").flatMap((toolCall) => {
|
|
17
|
+
const skillName = toolCall.input && typeof toolCall.input === "object" && typeof toolCall.input.skill === "string" ? toolCall.input.skill.trim() : "";
|
|
18
|
+
if (!skillName) return [];
|
|
19
|
+
return [{
|
|
20
|
+
name: skillName,
|
|
21
|
+
input: toolCall.input,
|
|
22
|
+
is_error: toolCall.is_error,
|
|
23
|
+
source: "tool"
|
|
24
|
+
}];
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Claude Agent SDK Provider
|
|
29
|
+
*
|
|
30
|
+
* This provider requires the @anthropic-ai/claude-agent-sdk package to be installed separately:
|
|
31
|
+
* npm install @anthropic-ai/claude-agent-sdk
|
|
32
|
+
*
|
|
33
|
+
* Two default configurations:
|
|
34
|
+
* - No working_dir: Runs in temp directory with no tools - behaves like plain chat API
|
|
35
|
+
* - With working_dir: Runs in specified directory with read-only file tools (Read/Grep/Glob/LS)
|
|
36
|
+
*
|
|
37
|
+
* User can override tool permissions with 'custom_allowed_tools', 'append_allowed_tools', 'disallowed_tools', and 'permission_mode'.
|
|
38
|
+
*
|
|
39
|
+
* For side effects (file writes, system calls, etc.), user can override permissions and use a custom working directory. They're then responsible for setup/teardown and security considerations.
|
|
40
|
+
*
|
|
41
|
+
* MCP server connection details are passed through from config. strict_mcp_config is true by default to only allow explicitly configured MCP servers.
|
|
42
|
+
*/
|
|
43
|
+
const FS_READONLY_ALLOWED_TOOLS = [
|
|
44
|
+
"Read",
|
|
45
|
+
"Grep",
|
|
46
|
+
"Glob",
|
|
47
|
+
"LS"
|
|
48
|
+
].sort();
|
|
49
|
+
const CLAUDE_CODE_MODEL_ALIASES = [
|
|
50
|
+
"default",
|
|
51
|
+
"sonnet",
|
|
52
|
+
"opus",
|
|
53
|
+
"haiku",
|
|
54
|
+
"sonnet[1m]",
|
|
55
|
+
"opusplan"
|
|
56
|
+
];
|
|
57
|
+
/**
|
|
58
|
+
* Helper to load the Claude Agent SDK ESM module
|
|
59
|
+
* Uses resolvePackageEntryPoint to handle ESM-only packages with restrictive exports
|
|
60
|
+
*/
|
|
61
|
+
async function loadClaudeCodeSDK() {
|
|
62
|
+
const claudeCodePath = require_esm.resolvePackageEntryPoint("@anthropic-ai/claude-agent-sdk", require_logger.state.basePath && node_path.default.isAbsolute(require_logger.state.basePath) ? require_logger.state.basePath : process.cwd());
|
|
63
|
+
if (!claudeCodePath) throw new Error(dedent.default`The @anthropic-ai/claude-agent-sdk package is required but not installed.
|
|
64
|
+
|
|
65
|
+
To use the Claude Agent SDK provider, install it with:
|
|
66
|
+
npm install @anthropic-ai/claude-agent-sdk
|
|
67
|
+
|
|
68
|
+
For more information, see: https://www.promptfoo.dev/docs/providers/claude-agent-sdk/`);
|
|
69
|
+
try {
|
|
70
|
+
return require_esm.importModule(claudeCodePath);
|
|
71
|
+
} catch (err) {
|
|
72
|
+
require_logger.logger.error(`Failed to load Claude Agent SDK: ${err}`);
|
|
73
|
+
if (err.stack) require_logger.logger.error(err.stack);
|
|
74
|
+
throw new Error(dedent.default`Failed to load @anthropic-ai/claude-agent-sdk.
|
|
75
|
+
|
|
76
|
+
The package was found but could not be loaded. This may be due to:
|
|
77
|
+
- Incompatible Node.js version (requires Node.js 20.20+ or 22.22+)
|
|
78
|
+
- Corrupted installation
|
|
79
|
+
|
|
80
|
+
Try reinstalling:
|
|
81
|
+
npm install @anthropic-ai/claude-agent-sdk
|
|
82
|
+
|
|
83
|
+
For more information, see: https://www.promptfoo.dev/docs/providers/claude-agent-sdk/`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Creates a canUseTool callback for handling AskUserQuestion tool in automated evaluations.
|
|
88
|
+
* This provides automated responses to questions that would normally require user input.
|
|
89
|
+
*
|
|
90
|
+
* The callback wraps an optional user-provided canUseTool and handles AskUserQuestion specifically,
|
|
91
|
+
* deferring to the wrapped callback for all other tools.
|
|
92
|
+
*/
|
|
93
|
+
function createAskUserQuestionCanUseTool(behavior = "first_option", wrappedCanUseTool) {
|
|
94
|
+
return async (toolName, input, options) => {
|
|
95
|
+
if (toolName !== "AskUserQuestion") {
|
|
96
|
+
if (wrappedCanUseTool) return wrappedCanUseTool(toolName, input, options);
|
|
97
|
+
return {
|
|
98
|
+
behavior: "allow",
|
|
99
|
+
updatedInput: input
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
if (behavior === "deny") return {
|
|
103
|
+
behavior: "deny",
|
|
104
|
+
message: "AskUserQuestion is disabled in automated evaluation mode"
|
|
105
|
+
};
|
|
106
|
+
const toolInput = input;
|
|
107
|
+
const answers = {};
|
|
108
|
+
for (const question of toolInput.questions) {
|
|
109
|
+
if (!question.options || question.options.length === 0) continue;
|
|
110
|
+
let selectedLabels;
|
|
111
|
+
if (behavior === "random") {
|
|
112
|
+
const randomIndex = Math.floor(Math.random() * question.options.length);
|
|
113
|
+
selectedLabels = [question.options[randomIndex].label];
|
|
114
|
+
} else selectedLabels = [question.options[0].label];
|
|
115
|
+
answers[question.question] = selectedLabels.join(", ");
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
behavior: "allow",
|
|
119
|
+
updatedInput: {
|
|
120
|
+
questions: toolInput.questions,
|
|
121
|
+
answers
|
|
122
|
+
}
|
|
123
|
+
};
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
var ClaudeCodeSDKProvider = class ClaudeCodeSDKProvider {
|
|
127
|
+
static ANTHROPIC_MODELS = require_util.ANTHROPIC_MODELS;
|
|
128
|
+
static ANTHROPIC_MODELS_NAMES = require_util.ANTHROPIC_MODELS.map((model) => model.id);
|
|
129
|
+
config;
|
|
130
|
+
env;
|
|
131
|
+
apiKey;
|
|
132
|
+
providerId = "anthropic:claude-agent-sdk";
|
|
133
|
+
claudeCodeModule;
|
|
134
|
+
constructor(options = {}) {
|
|
135
|
+
const { config, env, id } = options;
|
|
136
|
+
this.config = config ?? {};
|
|
137
|
+
this.env = env;
|
|
138
|
+
this.apiKey = this.getApiKey();
|
|
139
|
+
this.providerId = id ?? this.providerId;
|
|
140
|
+
if (this.config.model && !ClaudeCodeSDKProvider.ANTHROPIC_MODELS_NAMES.includes(this.config.model) && !CLAUDE_CODE_MODEL_ALIASES.includes(this.config.model)) require_logger.logger.warn(`Using unknown model for Claude Agent SDK: ${this.config.model}`);
|
|
141
|
+
if (this.config.fallback_model && !ClaudeCodeSDKProvider.ANTHROPIC_MODELS_NAMES.includes(this.config.fallback_model) && !CLAUDE_CODE_MODEL_ALIASES.includes(this.config.fallback_model)) require_logger.logger.warn(`Using unknown model for Claude Agent SDK fallback: ${this.config.fallback_model}`);
|
|
142
|
+
}
|
|
143
|
+
id() {
|
|
144
|
+
return this.providerId;
|
|
145
|
+
}
|
|
146
|
+
async callApi(prompt, context, callOptions) {
|
|
147
|
+
const config = {
|
|
148
|
+
...this.config,
|
|
149
|
+
...context?.prompt?.config
|
|
150
|
+
};
|
|
151
|
+
const env = {};
|
|
152
|
+
for (const key of Object.keys(process.env).sort()) if (process.env[key] !== void 0) env[key] = process.env[key];
|
|
153
|
+
if (this.env) for (const key of Object.keys(this.env).sort()) {
|
|
154
|
+
const value = this.env[key];
|
|
155
|
+
if (value !== void 0) env[key] = value;
|
|
156
|
+
}
|
|
157
|
+
if (this.apiKey) env.ANTHROPIC_API_KEY = this.apiKey;
|
|
158
|
+
if (!this.apiKey && !(config.apiKeyRequired === false || env.CLAUDE_CODE_USE_BEDROCK || env.CLAUDE_CODE_USE_VERTEX)) throw new Error(dedent.default`Anthropic API key is not set. Set the ANTHROPIC_API_KEY environment variable or add "apiKey" to the provider config.
|
|
159
|
+
|
|
160
|
+
Use CLAUDE_CODE_USE_BEDROCK or CLAUDE_CODE_USE_VERTEX environment variables to use Bedrock or Vertex instead.`);
|
|
161
|
+
if (config.allow_all_tools && ("custom_allowed_tools" in config || "append_allowed_tools" in config)) throw new Error("Cannot specify both allow_all_tools and custom_allowed_tools or append_allowed_tools");
|
|
162
|
+
if ("custom_allowed_tools" in config && "append_allowed_tools" in config) throw new Error("Cannot specify both custom_allowed_tools and append_allowed_tools");
|
|
163
|
+
if (config.permission_mode === "bypassPermissions" && !config.allow_dangerously_skip_permissions) throw new Error("permission_mode 'bypassPermissions' requires allow_dangerously_skip_permissions: true as a safety measure");
|
|
164
|
+
const defaultAllowedTools = config.working_dir ? FS_READONLY_ALLOWED_TOOLS : [];
|
|
165
|
+
let allowedTools = config.allow_all_tools ? void 0 : defaultAllowedTools;
|
|
166
|
+
if ("custom_allowed_tools" in config) allowedTools = Array.from(new Set(config.custom_allowed_tools ?? [])).sort();
|
|
167
|
+
else if (config.append_allowed_tools) allowedTools = Array.from(new Set([...defaultAllowedTools, ...config.append_allowed_tools])).sort();
|
|
168
|
+
const disallowedTools = config.disallowed_tools ? Array.from(new Set(config.disallowed_tools)).sort() : void 0;
|
|
169
|
+
const basePath = require_logger.state.basePath ? node_path.default.resolve(require_logger.state.basePath) : process.cwd();
|
|
170
|
+
let isTempDir = false;
|
|
171
|
+
let workingDir;
|
|
172
|
+
if (config.working_dir) workingDir = require_esm.safeResolve(basePath, config.working_dir);
|
|
173
|
+
else isTempDir = true;
|
|
174
|
+
let canUseTool;
|
|
175
|
+
if (config.ask_user_question) canUseTool = createAskUserQuestionCanUseTool(config.ask_user_question.behavior);
|
|
176
|
+
const cacheKeyQueryOptions = {
|
|
177
|
+
maxTurns: config.max_turns,
|
|
178
|
+
model: config.model,
|
|
179
|
+
fallbackModel: config.fallback_model,
|
|
180
|
+
strictMcpConfig: config.strict_mcp_config ?? true,
|
|
181
|
+
permissionMode: config.permission_mode,
|
|
182
|
+
systemPrompt: config.custom_system_prompt ? config.custom_system_prompt : {
|
|
183
|
+
type: "preset",
|
|
184
|
+
preset: "claude_code",
|
|
185
|
+
append: config.append_system_prompt
|
|
186
|
+
},
|
|
187
|
+
maxThinkingTokens: config.max_thinking_tokens,
|
|
188
|
+
allowedTools,
|
|
189
|
+
disallowedTools,
|
|
190
|
+
plugins: config.plugins?.map((plugin) => ({
|
|
191
|
+
...plugin,
|
|
192
|
+
path: require_esm.safeResolve(basePath, plugin.path)
|
|
193
|
+
})),
|
|
194
|
+
maxBudgetUsd: config.max_budget_usd,
|
|
195
|
+
additionalDirectories: config.additional_directories?.map((dir) => require_esm.safeResolve(basePath, dir)),
|
|
196
|
+
resume: config.resume,
|
|
197
|
+
forkSession: config.fork_session,
|
|
198
|
+
resumeSessionAt: config.resume_session_at,
|
|
199
|
+
continue: config.continue,
|
|
200
|
+
agents: config.agents,
|
|
201
|
+
outputFormat: config.output_format,
|
|
202
|
+
hooks: config.hooks,
|
|
203
|
+
includePartialMessages: config.include_partial_messages,
|
|
204
|
+
betas: config.betas,
|
|
205
|
+
thinking: config.thinking,
|
|
206
|
+
effort: config.effort,
|
|
207
|
+
agent: config.agent,
|
|
208
|
+
sessionId: config.session_id,
|
|
209
|
+
debug: config.debug,
|
|
210
|
+
debugFile: config.debug_file ? require_esm.safeResolve(basePath, config.debug_file) : void 0,
|
|
211
|
+
sandbox: config.sandbox,
|
|
212
|
+
allowDangerouslySkipPermissions: config.allow_dangerously_skip_permissions,
|
|
213
|
+
permissionPromptToolName: config.permission_prompt_tool_name,
|
|
214
|
+
executable: config.executable,
|
|
215
|
+
executableArgs: config.executable_args,
|
|
216
|
+
extraArgs: config.extra_args,
|
|
217
|
+
pathToClaudeCodeExecutable: config.path_to_claude_code_executable ? require_esm.safeResolve(basePath, config.path_to_claude_code_executable) : void 0,
|
|
218
|
+
settingSources: config.setting_sources,
|
|
219
|
+
tools: config.tools,
|
|
220
|
+
enableFileCheckpointing: config.enable_file_checkpointing,
|
|
221
|
+
persistSession: config.persist_session,
|
|
222
|
+
env
|
|
223
|
+
};
|
|
224
|
+
const cacheResult = await require_agentic_utils.initializeAgenticCache({
|
|
225
|
+
cacheKeyPrefix: "anthropic:claude-agent-sdk",
|
|
226
|
+
workingDir,
|
|
227
|
+
bustCache: context?.bustCache,
|
|
228
|
+
mcp: config.mcp?.servers?.length ? config.mcp : void 0,
|
|
229
|
+
cacheMcp: config.cache_mcp
|
|
230
|
+
}, {
|
|
231
|
+
prompt,
|
|
232
|
+
cacheKeyQueryOptions
|
|
233
|
+
});
|
|
234
|
+
const cachedResponse = await require_agentic_utils.getCachedResponse(cacheResult, "Claude Agent SDK");
|
|
235
|
+
if (cachedResponse) return cachedResponse;
|
|
236
|
+
const mcpServers = config.mcp ? await require_transform.transformMCPConfigToClaudeCode(config.mcp) : {};
|
|
237
|
+
if (workingDir) {
|
|
238
|
+
let stats;
|
|
239
|
+
try {
|
|
240
|
+
stats = node_fs.default.statSync(workingDir);
|
|
241
|
+
} catch (err) {
|
|
242
|
+
throw new Error(`Working dir ${config.working_dir} does not exist or isn't accessible: ${err.message}`);
|
|
243
|
+
}
|
|
244
|
+
if (!stats.isDirectory()) throw new Error(`Working dir ${config.working_dir} is not a directory`);
|
|
245
|
+
} else if (isTempDir) workingDir = node_fs.default.mkdtempSync(node_path.default.join(node_os.default.tmpdir(), "promptfoo-claude-agent-sdk-"));
|
|
246
|
+
if (callOptions?.abortSignal?.aborted) return { error: "Claude Agent SDK call aborted before it started" };
|
|
247
|
+
const abortController = new AbortController();
|
|
248
|
+
let abortHandler;
|
|
249
|
+
if (callOptions?.abortSignal) {
|
|
250
|
+
abortHandler = () => {
|
|
251
|
+
abortController.abort(callOptions.abortSignal.reason);
|
|
252
|
+
};
|
|
253
|
+
callOptions.abortSignal.addEventListener("abort", abortHandler);
|
|
254
|
+
}
|
|
255
|
+
const options = {
|
|
256
|
+
...cacheKeyQueryOptions,
|
|
257
|
+
abortController,
|
|
258
|
+
mcpServers,
|
|
259
|
+
cwd: workingDir,
|
|
260
|
+
stderr: config.stderr,
|
|
261
|
+
spawnClaudeCodeProcess: config.spawn_claude_code_process,
|
|
262
|
+
canUseTool
|
|
263
|
+
};
|
|
264
|
+
const queryParams = {
|
|
265
|
+
prompt,
|
|
266
|
+
options
|
|
267
|
+
};
|
|
268
|
+
require_logger.logger.debug(`Calling Claude Agent SDK: ${JSON.stringify({
|
|
269
|
+
prompt,
|
|
270
|
+
options: {
|
|
271
|
+
...options,
|
|
272
|
+
mcpServers: options.mcpServers ? Object.keys(options.mcpServers) : void 0,
|
|
273
|
+
env: Object.keys(env).length > 0 ? Object.keys(env) : void 0
|
|
274
|
+
}
|
|
275
|
+
})}`);
|
|
276
|
+
try {
|
|
277
|
+
if (!this.claudeCodeModule) this.claudeCodeModule = await loadClaudeCodeSDK();
|
|
278
|
+
const res = await this.claudeCodeModule.query(queryParams);
|
|
279
|
+
const toolCallsMap = /* @__PURE__ */ new Map();
|
|
280
|
+
for await (const msg of res) if (msg.type === "assistant") {
|
|
281
|
+
for (const block of msg.message.content) if (block.type === "tool_use") toolCallsMap.set(block.id, {
|
|
282
|
+
id: block.id,
|
|
283
|
+
name: block.name,
|
|
284
|
+
input: block.input,
|
|
285
|
+
output: void 0,
|
|
286
|
+
is_error: false,
|
|
287
|
+
parentToolUseId: msg.parent_tool_use_id
|
|
288
|
+
});
|
|
289
|
+
} else if (msg.type === "user") {
|
|
290
|
+
const content = msg.message?.content;
|
|
291
|
+
if (Array.isArray(content)) {
|
|
292
|
+
for (const block of content) if (block.type === "tool_result") {
|
|
293
|
+
const entry = toolCallsMap.get(block.tool_use_id);
|
|
294
|
+
if (entry) {
|
|
295
|
+
entry.output = block.content;
|
|
296
|
+
entry.is_error = block.is_error ?? false;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
} else if (msg.type === "result") {
|
|
301
|
+
const raw = JSON.stringify(msg);
|
|
302
|
+
const tokenUsage = {
|
|
303
|
+
prompt: msg.usage?.input_tokens,
|
|
304
|
+
completion: msg.usage?.output_tokens,
|
|
305
|
+
total: msg.usage?.input_tokens && msg.usage?.output_tokens ? msg.usage?.input_tokens + msg.usage?.output_tokens : void 0
|
|
306
|
+
};
|
|
307
|
+
const cost = msg.total_cost_usd ?? 0;
|
|
308
|
+
const sessionId = msg.session_id;
|
|
309
|
+
const toolCallsArray = Array.from(toolCallsMap.values());
|
|
310
|
+
const skillCalls = deriveSkillCalls(toolCallsArray);
|
|
311
|
+
if (msg.subtype === "success") {
|
|
312
|
+
require_logger.logger.debug(`Claude Agent SDK response: ${raw}`);
|
|
313
|
+
const response = {
|
|
314
|
+
output: msg.structured_output === void 0 ? msg.result : msg.structured_output,
|
|
315
|
+
tokenUsage,
|
|
316
|
+
cost,
|
|
317
|
+
raw,
|
|
318
|
+
sessionId,
|
|
319
|
+
metadata: {
|
|
320
|
+
skillCalls,
|
|
321
|
+
toolCalls: toolCallsArray,
|
|
322
|
+
numTurns: msg.num_turns,
|
|
323
|
+
durationMs: msg.duration_ms,
|
|
324
|
+
durationApiMs: msg.duration_api_ms,
|
|
325
|
+
modelUsage: msg.modelUsage,
|
|
326
|
+
permissionDenials: msg.permission_denials,
|
|
327
|
+
...msg.structured_output === void 0 ? {} : { structuredOutput: msg.structured_output }
|
|
328
|
+
}
|
|
329
|
+
};
|
|
330
|
+
await require_agentic_utils.cacheResponse(cacheResult, response, "Claude Agent SDK");
|
|
331
|
+
return response;
|
|
332
|
+
} else return {
|
|
333
|
+
error: `Claude Agent SDK call failed: ${msg.subtype}`,
|
|
334
|
+
tokenUsage,
|
|
335
|
+
cost,
|
|
336
|
+
raw,
|
|
337
|
+
sessionId,
|
|
338
|
+
metadata: {
|
|
339
|
+
skillCalls,
|
|
340
|
+
toolCalls: toolCallsArray,
|
|
341
|
+
numTurns: msg.num_turns,
|
|
342
|
+
durationMs: msg.duration_ms,
|
|
343
|
+
durationApiMs: msg.duration_api_ms,
|
|
344
|
+
modelUsage: msg.modelUsage,
|
|
345
|
+
permissionDenials: msg.permission_denials
|
|
346
|
+
}
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
return { error: "Claude Agent SDK call didn't return a result" };
|
|
350
|
+
} catch (error) {
|
|
351
|
+
if (error?.name === "AbortError" || callOptions?.abortSignal?.aborted) {
|
|
352
|
+
require_logger.logger.warn("Claude Agent SDK call aborted");
|
|
353
|
+
return { error: "Claude Agent SDK call aborted" };
|
|
354
|
+
}
|
|
355
|
+
require_logger.logger.error(`Error calling Claude Agent SDK: ${error}`);
|
|
356
|
+
return { error: `Error calling Claude Agent SDK: ${error}` };
|
|
357
|
+
} finally {
|
|
358
|
+
if (isTempDir && workingDir) node_fs.default.rmSync(workingDir, {
|
|
359
|
+
recursive: true,
|
|
360
|
+
force: true
|
|
361
|
+
});
|
|
362
|
+
if (callOptions?.abortSignal && abortHandler) callOptions.abortSignal.removeEventListener("abort", abortHandler);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
toString() {
|
|
366
|
+
return "[Anthropic Claude Agent SDK Provider]";
|
|
367
|
+
}
|
|
368
|
+
/**
|
|
369
|
+
* For normal Claude Agent SDK support, just use the Anthropic API key
|
|
370
|
+
* Users can also use Bedrock (with CLAUDE_CODE_USE_BEDROCK env var) or Vertex (with CLAUDE_CODE_USE_VERTEX env var)
|
|
371
|
+
*/
|
|
372
|
+
requiresApiKey() {
|
|
373
|
+
return !(this.env?.CLAUDE_CODE_USE_BEDROCK || this.env?.CLAUDE_CODE_USE_VERTEX || require_logger.getEnvString("CLAUDE_CODE_USE_BEDROCK") || require_logger.getEnvString("CLAUDE_CODE_USE_VERTEX"));
|
|
374
|
+
}
|
|
375
|
+
getApiKey() {
|
|
376
|
+
return this.config?.apiKey || this.env?.ANTHROPIC_API_KEY || require_logger.getEnvString("ANTHROPIC_API_KEY");
|
|
377
|
+
}
|
|
378
|
+
async cleanup() {}
|
|
379
|
+
};
|
|
380
|
+
//#endregion
|
|
381
|
+
exports.ClaudeCodeSDKProvider = ClaudeCodeSDKProvider;
|
|
382
|
+
|
|
383
|
+
//# sourceMappingURL=claude-agent-sdk-CkfyjDoG.cjs.map
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { x as getEnvString } from "./logger-C40ZGil9.js";
|
|
2
|
+
import { t as invariant } from "./invariant-Ddh24eXh.js";
|
|
3
|
+
import { t as OpenAiChatCompletionProvider } from "./chat-CcDgZFJ4.js";
|
|
4
|
+
import { n as OpenAiEmbeddingProvider, t as OpenAiCompletionProvider } from "./completion-B8Ctyxpr.js";
|
|
5
|
+
//#region src/providers/cloudflare-ai.ts
|
|
6
|
+
function getCloudflareApiConfig(config, env) {
|
|
7
|
+
const apiTokenCandidate = config?.apiKey || (config?.apiKeyEnvar ? getEnvString(config.apiKeyEnvar) || env?.[config.apiKeyEnvar] : void 0) || env?.CLOUDFLARE_API_KEY || getEnvString("CLOUDFLARE_API_KEY");
|
|
8
|
+
invariant(apiTokenCandidate, "Cloudflare API token required. Supply it via config apiKey or apiKeyEnvar, or the CLOUDFLARE_API_KEY environment variable");
|
|
9
|
+
const accountIdCandidate = config?.accountId || (config?.accountIdEnvar ? getEnvString(config.accountIdEnvar) || env?.[config.accountIdEnvar] : void 0) || env?.CLOUDFLARE_ACCOUNT_ID || getEnvString("CLOUDFLARE_ACCOUNT_ID");
|
|
10
|
+
invariant(accountIdCandidate, "Cloudflare account ID required. Supply it via config accountId or accountIdEnvar, or the CLOUDFLARE_ACCOUNT_ID environment variable");
|
|
11
|
+
return {
|
|
12
|
+
apiToken: apiTokenCandidate,
|
|
13
|
+
accountId: accountIdCandidate
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
function getApiBaseUrl(config, env) {
|
|
17
|
+
if (config?.apiBaseUrl) return config.apiBaseUrl;
|
|
18
|
+
const { accountId } = getCloudflareApiConfig(config, env);
|
|
19
|
+
return `https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/v1`;
|
|
20
|
+
}
|
|
21
|
+
function getPassthroughConfig(config) {
|
|
22
|
+
const { accountId: _accountId, accountIdEnvar: _accountIdEnvar, apiKey: _apiKey, apiKeyEnvar: _apiKeyEnvar, apiBaseUrl: _apiBaseUrl, ...passthrough } = config || {};
|
|
23
|
+
return passthrough;
|
|
24
|
+
}
|
|
25
|
+
var CloudflareAiChatCompletionProvider = class extends OpenAiChatCompletionProvider {
|
|
26
|
+
cloudflareConfig;
|
|
27
|
+
modelType = "chat";
|
|
28
|
+
constructor(modelName, providerOptions) {
|
|
29
|
+
const apiBaseUrl = getApiBaseUrl(providerOptions.config, providerOptions.env);
|
|
30
|
+
const passthrough = getPassthroughConfig(providerOptions.config);
|
|
31
|
+
const config = {
|
|
32
|
+
...providerOptions.config,
|
|
33
|
+
apiKeyEnvar: "CLOUDFLARE_API_KEY",
|
|
34
|
+
apiBaseUrl,
|
|
35
|
+
passthrough
|
|
36
|
+
};
|
|
37
|
+
super(modelName, {
|
|
38
|
+
...providerOptions,
|
|
39
|
+
config
|
|
40
|
+
});
|
|
41
|
+
this.cloudflareConfig = providerOptions.config || {};
|
|
42
|
+
}
|
|
43
|
+
id() {
|
|
44
|
+
return `cloudflare-ai:${this.modelType}:${this.modelName}`;
|
|
45
|
+
}
|
|
46
|
+
toString() {
|
|
47
|
+
return `[Cloudflare AI ${this.modelType} Provider ${this.modelName}]`;
|
|
48
|
+
}
|
|
49
|
+
getApiKey() {
|
|
50
|
+
const { apiToken } = getCloudflareApiConfig(this.cloudflareConfig, this.env);
|
|
51
|
+
return apiToken;
|
|
52
|
+
}
|
|
53
|
+
toJSON() {
|
|
54
|
+
return {
|
|
55
|
+
provider: "cloudflare-ai",
|
|
56
|
+
model: this.modelName,
|
|
57
|
+
modelType: this.modelType,
|
|
58
|
+
config: {
|
|
59
|
+
...this.config,
|
|
60
|
+
...this.getApiKey() && { apiKey: void 0 }
|
|
61
|
+
}
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
var CloudflareAiCompletionProvider = class extends OpenAiCompletionProvider {
|
|
66
|
+
cloudflareConfig;
|
|
67
|
+
modelType = "completion";
|
|
68
|
+
constructor(modelName, providerOptions) {
|
|
69
|
+
const apiBaseUrl = getApiBaseUrl(providerOptions.config, providerOptions.env);
|
|
70
|
+
const passthrough = getPassthroughConfig(providerOptions.config);
|
|
71
|
+
const config = {
|
|
72
|
+
...providerOptions.config,
|
|
73
|
+
apiKeyEnvar: "CLOUDFLARE_API_KEY",
|
|
74
|
+
apiBaseUrl,
|
|
75
|
+
passthrough
|
|
76
|
+
};
|
|
77
|
+
super(modelName, {
|
|
78
|
+
...providerOptions,
|
|
79
|
+
config
|
|
80
|
+
});
|
|
81
|
+
this.cloudflareConfig = providerOptions.config || {};
|
|
82
|
+
}
|
|
83
|
+
id() {
|
|
84
|
+
return `cloudflare-ai:${this.modelType}:${this.modelName}`;
|
|
85
|
+
}
|
|
86
|
+
toString() {
|
|
87
|
+
return `[Cloudflare AI ${this.modelType} Provider ${this.modelName}]`;
|
|
88
|
+
}
|
|
89
|
+
getApiKey() {
|
|
90
|
+
const { apiToken } = getCloudflareApiConfig(this.cloudflareConfig, this.env);
|
|
91
|
+
return apiToken;
|
|
92
|
+
}
|
|
93
|
+
toJSON() {
|
|
94
|
+
return {
|
|
95
|
+
provider: "cloudflare-ai",
|
|
96
|
+
model: this.modelName,
|
|
97
|
+
modelType: this.modelType,
|
|
98
|
+
config: {
|
|
99
|
+
...this.config,
|
|
100
|
+
...this.getApiKey() && { apiKey: void 0 }
|
|
101
|
+
}
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
var CloudflareAiEmbeddingProvider = class extends OpenAiEmbeddingProvider {
|
|
106
|
+
cloudflareConfig;
|
|
107
|
+
modelType = "embedding";
|
|
108
|
+
constructor(modelName, providerOptions) {
|
|
109
|
+
const apiBaseUrl = getApiBaseUrl(providerOptions.config, providerOptions.env);
|
|
110
|
+
const passthrough = getPassthroughConfig(providerOptions.config);
|
|
111
|
+
const config = {
|
|
112
|
+
...providerOptions.config,
|
|
113
|
+
apiKeyEnvar: "CLOUDFLARE_API_KEY",
|
|
114
|
+
apiBaseUrl,
|
|
115
|
+
passthrough
|
|
116
|
+
};
|
|
117
|
+
super(modelName, {
|
|
118
|
+
...providerOptions,
|
|
119
|
+
config
|
|
120
|
+
});
|
|
121
|
+
this.cloudflareConfig = providerOptions.config || {};
|
|
122
|
+
}
|
|
123
|
+
id() {
|
|
124
|
+
return `cloudflare-ai:${this.modelType}:${this.modelName}`;
|
|
125
|
+
}
|
|
126
|
+
toString() {
|
|
127
|
+
return `[Cloudflare AI ${this.modelType} Provider ${this.modelName}]`;
|
|
128
|
+
}
|
|
129
|
+
getApiKey() {
|
|
130
|
+
const { apiToken } = getCloudflareApiConfig(this.cloudflareConfig, this.env);
|
|
131
|
+
return apiToken;
|
|
132
|
+
}
|
|
133
|
+
toJSON() {
|
|
134
|
+
return {
|
|
135
|
+
provider: "cloudflare-ai",
|
|
136
|
+
model: this.modelName,
|
|
137
|
+
modelType: this.modelType,
|
|
138
|
+
config: {
|
|
139
|
+
...this.config,
|
|
140
|
+
...this.getApiKey() && { apiKey: void 0 }
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
function createCloudflareAiProvider(providerPath, options = {}) {
|
|
146
|
+
const splits = providerPath.split(":");
|
|
147
|
+
const modelType = splits[1];
|
|
148
|
+
const modelName = splits.slice(2).join(":");
|
|
149
|
+
invariant(modelName, "Model name is required");
|
|
150
|
+
switch (modelType) {
|
|
151
|
+
case "chat": return new CloudflareAiChatCompletionProvider(modelName, options);
|
|
152
|
+
case "completion": return new CloudflareAiCompletionProvider(modelName, options);
|
|
153
|
+
case "embedding":
|
|
154
|
+
case "embeddings": return new CloudflareAiEmbeddingProvider(modelName, options);
|
|
155
|
+
default: throw new Error(`Unknown Cloudflare AI model type: ${modelType}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
//#endregion
|
|
159
|
+
export { createCloudflareAiProvider };
|
|
160
|
+
|
|
161
|
+
//# sourceMappingURL=cloudflare-ai-BzpJcqUH.js.map
|