@browserbasehq/orca 3.2.0-preview.4 → 3.2.0-preview.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/lib/inference.d.ts +3 -1
- package/dist/cjs/lib/inference.js +2 -2
- package/dist/cjs/lib/inference.js.map +1 -1
- package/dist/cjs/lib/prompt.d.ts +1 -1
- package/dist/cjs/lib/prompt.js +11 -2
- package/dist/cjs/lib/prompt.js.map +1 -1
- package/dist/cjs/lib/v3/agent/AgentClient.d.ts +8 -0
- package/dist/cjs/lib/v3/agent/AgentClient.js +13 -0
- package/dist/cjs/lib/v3/agent/AgentClient.js.map +1 -1
- package/dist/cjs/lib/v3/agent/AnthropicCUAClient.js +1 -0
- package/dist/cjs/lib/v3/agent/AnthropicCUAClient.js.map +1 -1
- package/dist/cjs/lib/v3/agent/GoogleCUAClient.js +1 -0
- package/dist/cjs/lib/v3/agent/GoogleCUAClient.js.map +1 -1
- package/dist/cjs/lib/v3/agent/MicrosoftCUAClient.js +1 -0
- package/dist/cjs/lib/v3/agent/MicrosoftCUAClient.js.map +1 -1
- package/dist/cjs/lib/v3/agent/OpenAICUAClient.d.ts +4 -4
- package/dist/cjs/lib/v3/agent/OpenAICUAClient.js +62 -1
- package/dist/cjs/lib/v3/agent/OpenAICUAClient.js.map +1 -1
- package/dist/cjs/lib/v3/agent/prompts/agentSystemPrompt.d.ts +2 -2
- package/dist/cjs/lib/v3/agent/prompts/agentSystemPrompt.js +10 -11
- package/dist/cjs/lib/v3/agent/prompts/agentSystemPrompt.js.map +1 -1
- package/dist/cjs/lib/v3/agent/tools/fillform.d.ts +0 -1
- package/dist/cjs/lib/v3/agent/tools/fillform.js +7 -10
- package/dist/cjs/lib/v3/agent/tools/fillform.js.map +1 -1
- package/dist/cjs/lib/v3/agent/utils/captchaSolver.d.ts +76 -0
- package/dist/cjs/lib/v3/agent/utils/captchaSolver.js +175 -0
- package/dist/cjs/lib/v3/agent/utils/captchaSolver.js.map +1 -0
- package/dist/cjs/lib/v3/agent/utils/variables.d.ts +5 -0
- package/dist/cjs/lib/v3/agent/utils/variables.js +9 -0
- package/dist/cjs/lib/v3/agent/utils/variables.js.map +1 -1
- package/dist/cjs/lib/v3/api.js +2 -9
- package/dist/cjs/lib/v3/api.js.map +1 -1
- package/dist/cjs/lib/v3/flowlogger/EventStore.js +1 -1
- package/dist/cjs/lib/v3/flowlogger/EventStore.js.map +1 -1
- package/dist/cjs/lib/v3/handlers/observeHandler.js +2 -1
- package/dist/cjs/lib/v3/handlers/observeHandler.js.map +1 -1
- package/dist/cjs/lib/v3/handlers/v3AgentHandler.d.ts +2 -1
- package/dist/cjs/lib/v3/handlers/v3AgentHandler.js +110 -46
- package/dist/cjs/lib/v3/handlers/v3AgentHandler.js.map +1 -1
- package/dist/cjs/lib/v3/handlers/v3CuaAgentHandler.d.ts +5 -0
- package/dist/cjs/lib/v3/handlers/v3CuaAgentHandler.js +125 -1
- package/dist/cjs/lib/v3/handlers/v3CuaAgentHandler.js.map +1 -1
- package/dist/cjs/lib/v3/types/private/cache.d.ts +0 -1
- package/dist/cjs/lib/v3/types/private/cache.js.map +1 -1
- package/dist/cjs/lib/v3/types/private/handlers.d.ts +1 -0
- package/dist/cjs/lib/v3/types/private/handlers.js.map +1 -1
- package/dist/cjs/lib/v3/types/public/api.d.ts +24 -7
- package/dist/cjs/lib/v3/types/public/api.js +41 -14
- package/dist/cjs/lib/v3/types/public/api.js.map +1 -1
- package/dist/cjs/lib/v3/types/public/methods.d.ts +1 -0
- package/dist/cjs/lib/v3/types/public/methods.js.map +1 -1
- package/dist/cjs/lib/v3/types/public/variables.d.ts +7 -0
- package/dist/cjs/lib/v3/types/public/variables.js +22 -0
- package/dist/cjs/lib/v3/types/public/variables.js.map +1 -0
- package/dist/cjs/lib/v3/v3.d.ts +5 -0
- package/dist/cjs/lib/v3/v3.js +19 -4
- package/dist/cjs/lib/v3/v3.js.map +1 -1
- package/dist/cjs/lib/version.d.ts +1 -1
- package/dist/cjs/lib/version.js +1 -1
- package/dist/cjs/lib/version.js.map +1 -1
- package/dist/cjs/tests/integration/agent-captcha-autosolve.spec.d.ts +1 -0
- package/dist/cjs/tests/integration/agent-captcha-autosolve.spec.js +56 -0
- package/dist/cjs/tests/integration/agent-captcha-autosolve.spec.js.map +1 -0
- package/dist/cjs/tests/integration/agent-hybrid-mode.spec.js +6 -6
- package/dist/cjs/tests/integration/agent-hybrid-mode.spec.js.map +1 -1
- package/dist/cjs/tests/integration/timeouts.spec.js +1 -1
- package/dist/cjs/tests/integration/timeouts.spec.js.map +1 -1
- package/dist/cjs/tests/unit/agent-captcha-hooks.test.d.ts +1 -0
- package/dist/cjs/tests/unit/agent-captcha-hooks.test.js +285 -0
- package/dist/cjs/tests/unit/agent-captcha-hooks.test.js.map +1 -0
- package/dist/cjs/tests/unit/agent-execution-model.test.js +25 -3
- package/dist/cjs/tests/unit/agent-execution-model.test.js.map +1 -1
- package/dist/cjs/tests/unit/agent-system-prompt-variables.test.d.ts +1 -0
- package/dist/cjs/tests/unit/agent-system-prompt-variables.test.js +23 -0
- package/dist/cjs/tests/unit/agent-system-prompt-variables.test.js.map +1 -0
- package/dist/cjs/tests/unit/api-client-observe-variables.test.d.ts +1 -0
- package/dist/cjs/tests/unit/api-client-observe-variables.test.js +86 -0
- package/dist/cjs/tests/unit/api-client-observe-variables.test.js.map +1 -0
- package/dist/cjs/tests/unit/api-variables-schema.test.d.ts +1 -0
- package/dist/cjs/tests/unit/api-variables-schema.test.js +37 -0
- package/dist/cjs/tests/unit/api-variables-schema.test.js.map +1 -0
- package/dist/cjs/tests/unit/browserbase-session-accessors.test.js +20 -0
- package/dist/cjs/tests/unit/browserbase-session-accessors.test.js.map +1 -1
- package/dist/cjs/tests/unit/captcha-solver.test.d.ts +1 -0
- package/dist/cjs/tests/unit/captcha-solver.test.js +154 -0
- package/dist/cjs/tests/unit/captcha-solver.test.js.map +1 -0
- package/dist/cjs/tests/unit/flowlogger-eventstore.test.js +1 -1
- package/dist/cjs/tests/unit/flowlogger-eventstore.test.js.map +1 -1
- package/dist/cjs/tests/unit/openai-cua-client.test.d.ts +1 -0
- package/dist/cjs/tests/unit/openai-cua-client.test.js +71 -0
- package/dist/cjs/tests/unit/openai-cua-client.test.js.map +1 -0
- package/dist/cjs/tests/unit/prompt-observe-variables.test.d.ts +1 -0
- package/dist/cjs/tests/unit/prompt-observe-variables.test.js +19 -0
- package/dist/cjs/tests/unit/prompt-observe-variables.test.js.map +1 -0
- package/dist/cjs/tests/unit/public-api/public-types.test.js.map +1 -1
- package/dist/cjs/tests/unit/timeout-handlers.test.js +50 -0
- package/dist/cjs/tests/unit/timeout-handlers.test.js.map +1 -1
- package/dist/esm/lib/inference.d.ts +3 -1
- package/dist/esm/lib/inference.js +2 -2
- package/dist/esm/lib/inference.js.map +1 -1
- package/dist/esm/lib/prompt.d.ts +1 -1
- package/dist/esm/lib/prompt.js +11 -2
- package/dist/esm/lib/prompt.js.map +1 -1
- package/dist/esm/lib/v3/agent/AgentClient.d.ts +8 -0
- package/dist/esm/lib/v3/agent/AgentClient.js +13 -0
- package/dist/esm/lib/v3/agent/AgentClient.js.map +1 -1
- package/dist/esm/lib/v3/agent/AnthropicCUAClient.js +1 -0
- package/dist/esm/lib/v3/agent/AnthropicCUAClient.js.map +1 -1
- package/dist/esm/lib/v3/agent/GoogleCUAClient.js +1 -0
- package/dist/esm/lib/v3/agent/GoogleCUAClient.js.map +1 -1
- package/dist/esm/lib/v3/agent/MicrosoftCUAClient.js +1 -0
- package/dist/esm/lib/v3/agent/MicrosoftCUAClient.js.map +1 -1
- package/dist/esm/lib/v3/agent/OpenAICUAClient.d.ts +4 -4
- package/dist/esm/lib/v3/agent/OpenAICUAClient.js +62 -1
- package/dist/esm/lib/v3/agent/OpenAICUAClient.js.map +1 -1
- package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.d.ts +2 -2
- package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.js +10 -11
- package/dist/esm/lib/v3/agent/prompts/agentSystemPrompt.js.map +1 -1
- package/dist/esm/lib/v3/agent/tools/fillform.d.ts +0 -1
- package/dist/esm/lib/v3/agent/tools/fillform.js +7 -10
- package/dist/esm/lib/v3/agent/tools/fillform.js.map +1 -1
- package/dist/esm/lib/v3/agent/utils/captchaSolver.d.ts +76 -0
- package/dist/esm/lib/v3/agent/utils/captchaSolver.js +171 -0
- package/dist/esm/lib/v3/agent/utils/captchaSolver.js.map +1 -0
- package/dist/esm/lib/v3/agent/utils/variables.d.ts +5 -0
- package/dist/esm/lib/v3/agent/utils/variables.js +8 -0
- package/dist/esm/lib/v3/agent/utils/variables.js.map +1 -1
- package/dist/esm/lib/v3/api.js +2 -9
- package/dist/esm/lib/v3/api.js.map +1 -1
- package/dist/esm/lib/v3/flowlogger/EventStore.js +1 -1
- package/dist/esm/lib/v3/flowlogger/EventStore.js.map +1 -1
- package/dist/esm/lib/v3/handlers/observeHandler.js +2 -1
- package/dist/esm/lib/v3/handlers/observeHandler.js.map +1 -1
- package/dist/esm/lib/v3/handlers/v3AgentHandler.d.ts +2 -1
- package/dist/esm/lib/v3/handlers/v3AgentHandler.js +110 -46
- package/dist/esm/lib/v3/handlers/v3AgentHandler.js.map +1 -1
- package/dist/esm/lib/v3/handlers/v3CuaAgentHandler.d.ts +5 -0
- package/dist/esm/lib/v3/handlers/v3CuaAgentHandler.js +125 -1
- package/dist/esm/lib/v3/handlers/v3CuaAgentHandler.js.map +1 -1
- package/dist/esm/lib/v3/types/private/cache.d.ts +0 -1
- package/dist/esm/lib/v3/types/private/cache.js.map +1 -1
- package/dist/esm/lib/v3/types/private/handlers.d.ts +1 -0
- package/dist/esm/lib/v3/types/private/handlers.js.map +1 -1
- package/dist/esm/lib/v3/types/public/api.d.ts +24 -7
- package/dist/esm/lib/v3/types/public/api.js +36 -12
- package/dist/esm/lib/v3/types/public/api.js.map +1 -1
- package/dist/esm/lib/v3/types/public/methods.d.ts +1 -0
- package/dist/esm/lib/v3/types/public/methods.js.map +1 -1
- package/dist/esm/lib/v3/types/public/variables.d.ts +7 -0
- package/dist/esm/lib/v3/types/public/variables.js +19 -0
- package/dist/esm/lib/v3/types/public/variables.js.map +1 -0
- package/dist/esm/lib/v3/v3.d.ts +5 -0
- package/dist/esm/lib/v3/v3.js +19 -4
- package/dist/esm/lib/v3/v3.js.map +1 -1
- package/dist/esm/lib/version.d.ts +1 -1
- package/dist/esm/lib/version.js +1 -1
- package/dist/esm/lib/version.js.map +1 -1
- package/dist/esm/tests/integration/agent-captcha-autosolve.spec.d.ts +1 -0
- package/dist/esm/tests/integration/agent-captcha-autosolve.spec.js +54 -0
- package/dist/esm/tests/integration/agent-captcha-autosolve.spec.js.map +1 -0
- package/dist/esm/tests/integration/agent-hybrid-mode.spec.js +6 -6
- package/dist/esm/tests/integration/agent-hybrid-mode.spec.js.map +1 -1
- package/dist/esm/tests/integration/timeouts.spec.js +1 -1
- package/dist/esm/tests/integration/timeouts.spec.js.map +1 -1
- package/dist/esm/tests/unit/agent-captcha-hooks.test.d.ts +1 -0
- package/dist/esm/tests/unit/agent-captcha-hooks.test.js +283 -0
- package/dist/esm/tests/unit/agent-captcha-hooks.test.js.map +1 -0
- package/dist/esm/tests/unit/agent-execution-model.test.js +25 -3
- package/dist/esm/tests/unit/agent-execution-model.test.js.map +1 -1
- package/dist/esm/tests/unit/agent-system-prompt-variables.test.d.ts +1 -0
- package/dist/esm/tests/unit/agent-system-prompt-variables.test.js +21 -0
- package/dist/esm/tests/unit/agent-system-prompt-variables.test.js.map +1 -0
- package/dist/esm/tests/unit/api-client-observe-variables.test.d.ts +1 -0
- package/dist/esm/tests/unit/api-client-observe-variables.test.js +84 -0
- package/dist/esm/tests/unit/api-client-observe-variables.test.js.map +1 -0
- package/dist/esm/tests/unit/api-variables-schema.test.d.ts +1 -0
- package/dist/esm/tests/unit/api-variables-schema.test.js +35 -0
- package/dist/esm/tests/unit/api-variables-schema.test.js.map +1 -0
- package/dist/esm/tests/unit/browserbase-session-accessors.test.js +20 -0
- package/dist/esm/tests/unit/browserbase-session-accessors.test.js.map +1 -1
- package/dist/esm/tests/unit/captcha-solver.test.d.ts +1 -0
- package/dist/esm/tests/unit/captcha-solver.test.js +152 -0
- package/dist/esm/tests/unit/captcha-solver.test.js.map +1 -0
- package/dist/esm/tests/unit/flowlogger-eventstore.test.js +1 -1
- package/dist/esm/tests/unit/flowlogger-eventstore.test.js.map +1 -1
- package/dist/esm/tests/unit/openai-cua-client.test.d.ts +1 -0
- package/dist/esm/tests/unit/openai-cua-client.test.js +69 -0
- package/dist/esm/tests/unit/openai-cua-client.test.js.map +1 -0
- package/dist/esm/tests/unit/prompt-observe-variables.test.d.ts +1 -0
- package/dist/esm/tests/unit/prompt-observe-variables.test.js +17 -0
- package/dist/esm/tests/unit/prompt-observe-variables.test.js.map +1 -0
- package/dist/esm/tests/unit/public-api/public-types.test.js.map +1 -1
- package/dist/esm/tests/unit/timeout-handlers.test.js +50 -0
- package/dist/esm/tests/unit/timeout-handlers.test.js.map +1 -1
- package/package.json +2 -2
|
@@ -4,8 +4,8 @@ export interface AgentSystemPromptOptions {
|
|
|
4
4
|
executionInstruction: string;
|
|
5
5
|
mode: AgentToolMode;
|
|
6
6
|
systemInstructions?: string;
|
|
7
|
-
/** Whether
|
|
8
|
-
|
|
7
|
+
/** Whether captchas are automatically solved by the browser environment */
|
|
8
|
+
captchasAutoSolve?: boolean;
|
|
9
9
|
/** Tools to exclude from the system prompt */
|
|
10
10
|
excludeTools?: string[];
|
|
11
11
|
/** Variables available to the agent for use in act/type tools */
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.buildAgentSystemPrompt = buildAgentSystemPrompt;
|
|
4
|
+
const captchaSolver_js_1 = require("../utils/captchaSolver.js");
|
|
5
|
+
const variables_js_1 = require("../utils/variables.js");
|
|
4
6
|
function buildToolsSection(isHybridMode, hasSearch, excludeTools) {
|
|
5
7
|
const excludeSet = new Set(excludeTools ?? []);
|
|
6
8
|
const hybridTools = [
|
|
@@ -74,7 +76,7 @@ function buildToolsSection(isHybridMode, hasSearch, excludeTools) {
|
|
|
74
76
|
return `<tools>\n${toolLines}\n </tools>`;
|
|
75
77
|
}
|
|
76
78
|
function buildAgentSystemPrompt(options) {
|
|
77
|
-
const { url, executionInstruction, mode, systemInstructions,
|
|
79
|
+
const { url, executionInstruction, mode, systemInstructions, captchasAutoSolve = false, excludeTools, variables, useSearch = false, } = options;
|
|
78
80
|
const localeDate = new Date().toLocaleDateString();
|
|
79
81
|
const isoDate = new Date().toISOString();
|
|
80
82
|
const cdata = (text) => `<![CDATA[${text}]]>`;
|
|
@@ -135,11 +137,10 @@ function buildAgentSystemPrompt(options) {
|
|
|
135
137
|
</secondary_tool>
|
|
136
138
|
</step_1>
|
|
137
139
|
</page_understanding_protocol>`;
|
|
138
|
-
// Roadblocks section only shown when
|
|
139
|
-
const roadblocksSection =
|
|
140
|
+
// Roadblocks section only shown when captchas are auto-solved
|
|
141
|
+
const roadblocksSection = captchasAutoSolve
|
|
140
142
|
? `<roadblocks>
|
|
141
|
-
<note
|
|
142
|
-
<captcha>If you see a captcha, use the wait tool. It will automatically be solved by our internal solver.</captcha>
|
|
143
|
+
<note>${captchaSolver_js_1.CAPTCHA_SYSTEM_PROMPT_NOTE}</note>
|
|
143
144
|
</roadblocks>`
|
|
144
145
|
: "";
|
|
145
146
|
// Build customInstructions block only if provided
|
|
@@ -150,17 +151,15 @@ function buildAgentSystemPrompt(options) {
|
|
|
150
151
|
const hasVariables = variables && Object.keys(variables).length > 0;
|
|
151
152
|
const variableToolsNote = isHybridMode
|
|
152
153
|
? "Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields."
|
|
153
|
-
: "Use %variableName% syntax in the act or fillForm tool's
|
|
154
|
+
: "Use %variableName% syntax in the act or fillForm tool's action fields.";
|
|
155
|
+
const variableEntries = (0, variables_js_1.getVariablePromptEntries)(variables);
|
|
154
156
|
const variablesSection = hasVariables
|
|
155
157
|
? `<variables>
|
|
156
158
|
<note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>
|
|
157
159
|
<usage>${variableToolsNote}</usage>
|
|
158
160
|
<example>To type a password, use: type %password% into the password field</example>
|
|
159
|
-
${
|
|
160
|
-
.map((
|
|
161
|
-
const description = typeof v === "object" && v !== null && "value" in v
|
|
162
|
-
? v.description
|
|
163
|
-
: undefined;
|
|
161
|
+
${variableEntries
|
|
162
|
+
.map(({ name, description }) => {
|
|
164
163
|
return description
|
|
165
164
|
? `<variable name="${name}">${description}</variable>`
|
|
166
165
|
: `<variable name="${name}" />`;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agentSystemPrompt.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/prompts/agentSystemPrompt.ts"],"names":[],"mappings":";;AAsHA,wDA6JC;AAvPD,SAAS,iBAAiB,CACxB,YAAqB,EACrB,SAAkB,EAClB,YAAuB;IAEvB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IAE/C,MAAM,WAAW,GAAqB;QACpC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,OAAO;YACb,WAAW,EACT,qFAAqF;SACxF;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EACT,2FAA2F;SAC9F;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EACT,0LAA0L;SAC7L;QACD,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,0BAA0B,EAAE;QAChE,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,8BAA8B,EAAE;QACrE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,mCAAmC;SACjD;QACD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,QAAQ,GAAqB;QACjC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,gDAAgD;SAC9D;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExD,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,QAAQ;YACd,WAAW,EACT,8IAA8I;SACjJ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7E,MAAM,SAAS,GAAG,aAAa;SAC5B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,SAAS,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,YAAY,SAAS,cAAc,CAAC;AAC7C,CAAC;AAED,SAAgB,sBAAsB,CACpC,OAAiC;IAEjC,MAAM,EACJ,GAAG,EACH,oBAAoB,EACpB,IAAI,EACJ,kBAAkB,EAClB,aAAa,GAAG,KAAK,EACrB,YAAY,EACZ,SAAS,EACT,SAAS,GAAG,KAAK,GAClB,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,IAAI,KAAK,CAAC;IAEtD,MAAM,YAAY,GAAG,IAAI,KAAK,QAAQ,CAAC;IACvC,MAAM,SAAS,GAAG,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAElE,yDAAyD;IACzD,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9E,iCAAiC;IACjC,MAAM,aAAa,GAAG,YAAY;QAChC,CAAC,CAAC;YACE,uIAAuI;YACvI,4GAA4G;YAC5G,sIAAsI;YACtI,uHAAuH;YACvH,sFAAsF;SACvF;QACH,CAAC,CAAC;YACE,2FAA2F;YAC3F,oJAAoJ;YACpJ,6HAA6H;YAC7H,mIAAmI;YACnI,wHAAwH;SACzH,CAAC;IAEN,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,mBAAmB,GAAG;sNACwL,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,UAAU;;;;KAIrQ,CAAC;IAEJ,oDAAoD;IACpD,MAAM,yBAAyB,GAAG,YAAY;QAC5C,CAAC,CAAC;;;;;;;;;;;;;iCAa2B;QAC7B,CAAC,CAAC;;;;;;;;;;;;;iCAa2B,CAAC;IAEhC,iFAAiF;IACjF,MAAM,iBAAiB,GAAG,aAAa;QACrC,CAAC,CAAC;;;gBAGU;QACZ,CAAC,CAAC,EAAE,CAAC;IAEP,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,kBAAkB;QAChD,CAAC,CAAC,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,2BAA2B;QAC7E,CAAC,CAAC,EAAE,CAAC;IAEP,yDAAyD;IACzD,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,gGAAgG;QAClG,CAAC,CAAC,8EAA8E,CAAC;IACnF,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC;;aAEO,iBAAiB;;MAExB,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC;aACxB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE;YACjB,MAAM,WAAW,GACf,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,IAAI,CAAC;gBACjD,CAAC,CAAC,CAAC,CAAC,WAAW;gBACf,CAAC,CAAC,SAAS,CAAC;YAChB,OAAO,WAAW;gBAChB,CAAC,CAAC,mBAAmB,IAAI,KAAK,WAAW,aAAa;gBACtD,CAAC,CAAC,mBAAmB,IAAI,MAAM,CAAC;QACpC,CAAC,CAAC;aACD,IAAI,CAAC,QAAQ,CAAC;eACN;QACX,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;IAEL,uBAAuB;YACf,KAAK,CAAC,oBAAoB,CAAC;iCACN,OAAO,KAAK,UAAU;;;;2DAII,GAAG;;;;;;;;;;;;IAY1D,yBAAyB;;;MAGvB,SAAS,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE;;IAEtG,YAAY;;MAEV,eAAe;MACf,mBAAmB;;IAErB,iBAAiB;IACjB,gBAAgB;;;;;;;;UAQV,CAAC;AACX,CAAC","sourcesContent":["import type { AgentToolMode, Variables } from \"../../types/public/agent.js\";\n\nexport interface AgentSystemPromptOptions {\n url: string;\n executionInstruction: string;\n mode: AgentToolMode;\n systemInstructions?: string;\n /** Whether running on Browserbase (enables captcha solver messaging) */\n isBrowserbase?: boolean;\n /** Tools to exclude from the system prompt */\n excludeTools?: string[];\n /** Variables available to the agent for use in act/type tools */\n variables?: Variables;\n /** Whether the search tool is enabled for this execution */\n useSearch?: boolean;\n}\n\n/**\n * Builds the system prompt for the agent based on the tool mode.\n *\n * @param options - The prompt configuration options\n * @returns The formatted system prompt string\n */\ninterface ToolDefinition {\n name: string;\n description: string;\n}\n\nfunction buildToolsSection(\n isHybridMode: boolean,\n hasSearch: boolean,\n excludeTools?: string[],\n): string {\n const excludeSet = new Set(excludeTools ?? []);\n\n const hybridTools: ToolDefinition[] = [\n {\n name: \"screenshot\",\n description: \"Take a compressed JPEG screenshot for quick visual context\",\n },\n {\n name: \"ariaTree\",\n description:\n \"Get an accessibility (ARIA) hybrid tree for full page context\",\n },\n {\n name: \"click\",\n description:\n \"Click on an element (PREFERRED - more reliable when element is visible in viewport)\",\n },\n {\n name: \"type\",\n description:\n \"Type text into an element (PREFERRED - more reliable when element is visible in viewport)\",\n },\n {\n name: \"act\",\n description:\n \"Perform a specific atomic action (click, type, etc.) - ONLY use when element is in ariaTree but NOT visible in screenshot. Less reliable but can interact with out-of-viewport elements.\",\n },\n { name: \"dragAndDrop\", description: \"Drag and drop an element\" },\n { name: \"clickAndHold\", description: \"Click and hold on an element\" },\n { name: \"keys\", description: \"Press a keyboard key\" },\n {\n name: \"fillFormVision\",\n description: \"Fill out a form using coordinates\",\n },\n { name: \"think\", description: \"Think about the task\" },\n { name: \"extract\", description: \"Extract structured data\" },\n { name: \"goto\", description: \"Navigate to a URL\" },\n { name: \"wait\", description: \"Wait for a specified time\" },\n { name: \"navback\", description: \"Navigate back in browser history\" },\n { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n ];\n\n const domTools: ToolDefinition[] = [\n {\n name: \"screenshot\",\n description: \"Take a compressed JPEG screenshot for quick visual context\",\n },\n {\n name: \"ariaTree\",\n description:\n \"Get an accessibility (ARIA) hybrid tree for full page context\",\n },\n {\n name: \"act\",\n description: \"Perform a specific atomic action (click, type)\",\n },\n { name: \"keys\", description: \"Press a keyboard key\" },\n { name: \"fillForm\", description: \"Fill out a form\" },\n { name: \"think\", description: \"Think about the task\" },\n { name: \"extract\", description: \"Extract structured data\" },\n { name: \"goto\", description: \"Navigate to a URL\" },\n { name: \"wait\", description: \"Wait for a specified time\" },\n { name: \"navback\", description: \"Navigate back in browser history\" },\n { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n ];\n\n const baseTools = isHybridMode ? hybridTools : domTools;\n\n if (hasSearch) {\n baseTools.push({\n name: \"search\",\n description:\n \"Perform a web search and return results. Prefer this over navigating to Google and searching within the page for reliability and efficiency.\",\n });\n }\n\n const filteredTools = baseTools.filter((tool) => !excludeSet.has(tool.name));\n\n const toolLines = filteredTools\n .map((tool) => ` <tool name=\"${tool.name}\">${tool.description}</tool>`)\n .join(\"\\n\");\n\n return `<tools>\\n${toolLines}\\n </tools>`;\n}\n\nexport function buildAgentSystemPrompt(\n options: AgentSystemPromptOptions,\n): string {\n const {\n url,\n executionInstruction,\n mode,\n systemInstructions,\n isBrowserbase = false,\n excludeTools,\n variables,\n useSearch = false,\n } = options;\n const localeDate = new Date().toLocaleDateString();\n const isoDate = new Date().toISOString();\n const cdata = (text: string) => `<![CDATA[${text}]]>`;\n\n const isHybridMode = mode === \"hybrid\";\n const hasSearch = useSearch || Boolean(process.env.BRAVE_API_KEY);\n\n // Tools section differs based on mode and excluded tools\n const toolsSection = buildToolsSection(isHybridMode, hasSearch, excludeTools);\n\n // Strategy differs based on mode\n const strategyItems = isHybridMode\n ? [\n `<item>Tool selection priority: Use specific tools (click, type) when elements are visible in viewport for maximum reliability.</item>`,\n `<item>Always use screenshot to get proper grounding of the coordinates you want to type/click into.</item>`,\n `<item>When interacting with an input, always use the type tool to type into the input, over clicking and then typing into it.</item>`,\n `<item>Use ariaTree as a secondary tool when elements aren't visible in screenshot or to get full page context.</item>`,\n `<item>Only use act when element is in ariaTree but NOT visible in screenshot.</item>`,\n ]\n : [\n `<item>Tool selection priority: Use act tool for all clicking and typing on a page.</item>`,\n `<item>Always check ariaTree first to understand full page content without scrolling - it shows all elements including those below the fold.</item>`,\n `<item>When interacting with an input, always use the act tool to type into the input, over clicking and then typing.</item>`,\n `<item>If an element is present in the ariaTree, use act to interact with it directly - this eliminates the need to scroll.</item>`,\n `<item>Use screenshot for visual confirmation when needed, but rely primarily on ariaTree for element detection.</item>`,\n ];\n\n const strategySection = strategyItems.join(\"\\n \");\n\n const commonStrategyItems = `\n <item>CRITICAL: Use extract ONLY when the task explicitly requires structured data output (e.g., \"get job listings\", \"extract product details\"). For reading page content or understanding elements, always use ${isHybridMode ? \"screenshot or ariaTree\" : \"ariaTree\"} instead - it's faster and more reliable.</item>\n <item>Keep actions atomic and verify outcomes before proceeding.</item>\n <item>For each action, provide clear reasoning about why you're taking that step.</item>\n <item>When you need to input text that could be entered character-by-character or through multiple separate inputs, prefer using the keys tool to type the entire sequence at once. This is more efficient for scenarios like verification codes split across multiple fields, or when virtual keyboards are present but direct typing would be faster.</item>\n `;\n\n // Page understanding protocol differs based on mode\n const pageUnderstandingProtocol = isHybridMode\n ? `<page_understanding_protocol>\n <step_1>\n <title>UNDERSTAND THE PAGE</title>\n <primary_tool>\n <name>screenshot</name>\n <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n </primary_tool>\n <secondary_tool>\n <name>ariaTree</name>\n <usage>Get complete page context before taking actions</usage>\n <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n </secondary_tool>\n </step_1>\n </page_understanding_protocol>`\n : `<page_understanding_protocol>\n <step_1>\n <title>UNDERSTAND THE PAGE</title>\n <primary_tool>\n <name>ariaTree</name>\n <usage>Get complete page context before taking actions</usage>\n <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n </primary_tool>\n <secondary_tool>\n <name>screenshot</name>\n <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n </secondary_tool>\n </step_1>\n </page_understanding_protocol>`;\n\n // Roadblocks section only shown when running on Browserbase (has captcha solver)\n const roadblocksSection = isBrowserbase\n ? `<roadblocks>\n <note>captchas, popups, etc.</note>\n <captcha>If you see a captcha, use the wait tool. It will automatically be solved by our internal solver.</captcha>\n </roadblocks>`\n : \"\";\n\n // Build customInstructions block only if provided\n const customInstructionsBlock = systemInstructions\n ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\\n `\n : \"\";\n\n // Build variables section only if variables are provided\n const hasVariables = variables && Object.keys(variables).length > 0;\n const variableToolsNote = isHybridMode\n ? \"Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields.\"\n : \"Use %variableName% syntax in the act or fillForm tool's value/action fields.\";\n const variablesSection = hasVariables\n ? `<variables>\n <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>\n <usage>${variableToolsNote}</usage>\n <example>To type a password, use: type %password% into the password field</example>\n ${Object.entries(variables)\n .map(([name, v]) => {\n const description =\n typeof v === \"object\" && v !== null && \"value\" in v\n ? v.description\n : undefined;\n return description\n ? `<variable name=\"${name}\">${description}</variable>`\n : `<variable name=\"${name}\" />`;\n })\n .join(\"\\n \")}\n </variables>`\n : \"\";\n\n return `<system>\n <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>\n ${customInstructionsBlock}<task>\n <goal>${cdata(executionInstruction)}</goal>\n <date display=\"local\" iso=\"${isoDate}\">${localeDate}</date>\n <note>You may think the date is different due to knowledge cutoff, but this is the actual date.</note>\n </task>\n <page>\n <startingUrl>you are starting your task on this url: ${url}</startingUrl>\n </page>\n <mindset>\n <note>Be very intentional about your action. The initial instruction is very important, and slight variations of the actual goal can lead to failures.</note>\n <importantNote>If something fails to meet a single condition of the task, move on from it rather than seeing if it meets other criteria. We only care that it meets all of it</importantNote>\n <note>When the task is complete, do not seek more information; you have completed the task.</note>\n </mindset>\n <guidelines>\n <item>Always start by understanding the current page state</item>\n <item>Use the screenshot tool to verify page state when needed</item>\n <item>Use appropriate tools for each action</item>\n </guidelines>\n ${pageUnderstandingProtocol}\n <navigation>\n <rule>If you are confident in the URL, navigate directly to it.</rule>\n ${hasSearch ? `<rule>If you are not confident in the URL, use the search tool to find it.</rule>` : ``}\n </navigation>\n ${toolsSection}\n <strategy>\n ${strategySection}\n ${commonStrategyItems}\n </strategy>\n ${roadblocksSection}\n ${variablesSection}\n <completion>\n <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>\n <examples>\n <example>If you were asked for specific flights, list the flights you found.</example>\n <example>If you were asked for information about a product, list the product information you were asked for.</example>\n </examples>\n </completion>\n</system>`;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"agentSystemPrompt.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/prompts/agentSystemPrompt.ts"],"names":[],"mappings":";;AAwHA,wDAyJC;AAhRD,gEAAuE;AACvE,wDAAiE;AA4BjE,SAAS,iBAAiB,CACxB,YAAqB,EACrB,SAAkB,EAClB,YAAuB;IAEvB,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IAE/C,MAAM,WAAW,GAAqB;QACpC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,OAAO;YACb,WAAW,EACT,qFAAqF;SACxF;QACD;YACE,IAAI,EAAE,MAAM;YACZ,WAAW,EACT,2FAA2F;SAC9F;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EACT,0LAA0L;SAC7L;QACD,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,0BAA0B,EAAE;QAChE,EAAE,IAAI,EAAE,cAAc,EAAE,WAAW,EAAE,8BAA8B,EAAE;QACrE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD;YACE,IAAI,EAAE,gBAAgB;YACtB,WAAW,EAAE,mCAAmC;SACjD;QACD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,QAAQ,GAAqB;QACjC;YACE,IAAI,EAAE,YAAY;YAClB,WAAW,EAAE,4DAA4D;SAC1E;QACD;YACE,IAAI,EAAE,UAAU;YAChB,WAAW,EACT,+DAA+D;SAClE;QACD;YACE,IAAI,EAAE,KAAK;YACX,WAAW,EAAE,gDAAgD;SAC9D;QACD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACrD,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,iBAAiB,EAAE;QACpD,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,sBAAsB,EAAE;QACtD,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,yBAAyB,EAAE;QAC3D,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;QAClD,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,2BAA2B,EAAE;QAC1D,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;QACpE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,qCAAqC,EAAE;KACvE,CAAC;IAEF,MAAM,SAAS,GAAG,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC;IAExD,IAAI,SAAS,EAAE,CAAC;QACd,SAAS,CAAC,IAAI,CAAC;YACb,IAAI,EAAE,QAAQ;YACd,WAAW,EACT,8IAA8I;SACjJ,CAAC,CAAC;IACL,CAAC;IAED,MAAM,aAAa,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAE7E,MAAM,SAAS,GAAG,aAAa;SAC5B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,mBAAmB,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,WAAW,SAAS,CAAC;SACzE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEd,OAAO,YAAY,SAAS,cAAc,CAAC;AAC7C,CAAC;AAED,SAAgB,sBAAsB,CACpC,OAAiC;IAEjC,MAAM,EACJ,GAAG,EACH,oBAAoB,EACpB,IAAI,EACJ,kBAAkB,EAClB,iBAAiB,GAAG,KAAK,EACzB,YAAY,EACZ,SAAS,EACT,SAAS,GAAG,KAAK,GAClB,GAAG,OAAO,CAAC;IACZ,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,kBAAkB,EAAE,CAAC;IACnD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,YAAY,IAAI,KAAK,CAAC;IAEtD,MAAM,YAAY,GAAG,IAAI,KAAK,QAAQ,CAAC;IACvC,MAAM,SAAS,GAAG,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAElE,yDAAyD;IACzD,MAAM,YAAY,GAAG,iBAAiB,CAAC,YAAY,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;IAE9E,iCAAiC;IACjC,MAAM,aAAa,GAAG,YAAY;QAChC,CAAC,CAAC;YACE,uIAAuI;YACvI,4GAA4G;YAC5G,sIAAsI;YACtI,uHAAuH;YACvH,sFAAsF;SACvF;QACH,CAAC,CAAC;YACE,2FAA2F;YAC3F,oJAAoJ;YACpJ,6HAA6H;YAC7H,mIAAmI;YACnI,wHAAwH;SACzH,CAAC;IAEN,MAAM,eAAe,GAAG,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAErD,MAAM,mBAAmB,GAAG;sNACwL,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,UAAU;;;;KAIrQ,CAAC;IAEJ,oDAAoD;IACpD,MAAM,yBAAyB,GAAG,YAAY;QAC5C,CAAC,CAAC;;;;;;;;;;;;;iCAa2B;QAC7B,CAAC,CAAC;;;;;;;;;;;;;iCAa2B,CAAC;IAEhC,8DAA8D;IAC9D,MAAM,iBAAiB,GAAG,iBAAiB;QACzC,CAAC,CAAC;YACM,6CAA0B;gBACtB;QACZ,CAAC,CAAC,EAAE,CAAC;IAEP,kDAAkD;IAClD,MAAM,uBAAuB,GAAG,kBAAkB;QAChD,CAAC,CAAC,uBAAuB,KAAK,CAAC,kBAAkB,CAAC,2BAA2B;QAC7E,CAAC,CAAC,EAAE,CAAC;IAEP,yDAAyD;IACzD,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,gGAAgG;QAClG,CAAC,CAAC,wEAAwE,CAAC;IAC7E,MAAM,eAAe,GAAG,IAAA,uCAAwB,EAAC,SAAS,CAAC,CAAC;IAC5D,MAAM,gBAAgB,GAAG,YAAY;QACnC,CAAC,CAAC;;aAEO,iBAAiB;;MAExB,eAAe;aACd,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE;YAC7B,OAAO,WAAW;gBAChB,CAAC,CAAC,mBAAmB,IAAI,KAAK,WAAW,aAAa;gBACtD,CAAC,CAAC,mBAAmB,IAAI,MAAM,CAAC;QACpC,CAAC,CAAC;aACD,IAAI,CAAC,QAAQ,CAAC;eACN;QACX,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO;;IAEL,uBAAuB;YACf,KAAK,CAAC,oBAAoB,CAAC;iCACN,OAAO,KAAK,UAAU;;;;2DAII,GAAG;;;;;;;;;;;;IAY1D,yBAAyB;;;MAGvB,SAAS,CAAC,CAAC,CAAC,mFAAmF,CAAC,CAAC,CAAC,EAAE;;IAEtG,YAAY;;MAEV,eAAe;MACf,mBAAmB;;IAErB,iBAAiB;IACjB,gBAAgB;;;;;;;;UAQV,CAAC;AACX,CAAC","sourcesContent":["import type { AgentToolMode, Variables } from \"../../types/public/agent.js\";\nimport { CAPTCHA_SYSTEM_PROMPT_NOTE } from \"../utils/captchaSolver.js\";\nimport { getVariablePromptEntries } from \"../utils/variables.js\";\n\nexport interface AgentSystemPromptOptions {\n url: string;\n executionInstruction: string;\n mode: AgentToolMode;\n systemInstructions?: string;\n /** Whether captchas are automatically solved by the browser environment */\n captchasAutoSolve?: boolean;\n /** Tools to exclude from the system prompt */\n excludeTools?: string[];\n /** Variables available to the agent for use in act/type tools */\n variables?: Variables;\n /** Whether the search tool is enabled for this execution */\n useSearch?: boolean;\n}\n\n/**\n * Builds the system prompt for the agent based on the tool mode.\n *\n * @param options - The prompt configuration options\n * @returns The formatted system prompt string\n */\ninterface ToolDefinition {\n name: string;\n description: string;\n}\n\nfunction buildToolsSection(\n isHybridMode: boolean,\n hasSearch: boolean,\n excludeTools?: string[],\n): string {\n const excludeSet = new Set(excludeTools ?? []);\n\n const hybridTools: ToolDefinition[] = [\n {\n name: \"screenshot\",\n description: \"Take a compressed JPEG screenshot for quick visual context\",\n },\n {\n name: \"ariaTree\",\n description:\n \"Get an accessibility (ARIA) hybrid tree for full page context\",\n },\n {\n name: \"click\",\n description:\n \"Click on an element (PREFERRED - more reliable when element is visible in viewport)\",\n },\n {\n name: \"type\",\n description:\n \"Type text into an element (PREFERRED - more reliable when element is visible in viewport)\",\n },\n {\n name: \"act\",\n description:\n \"Perform a specific atomic action (click, type, etc.) - ONLY use when element is in ariaTree but NOT visible in screenshot. Less reliable but can interact with out-of-viewport elements.\",\n },\n { name: \"dragAndDrop\", description: \"Drag and drop an element\" },\n { name: \"clickAndHold\", description: \"Click and hold on an element\" },\n { name: \"keys\", description: \"Press a keyboard key\" },\n {\n name: \"fillFormVision\",\n description: \"Fill out a form using coordinates\",\n },\n { name: \"think\", description: \"Think about the task\" },\n { name: \"extract\", description: \"Extract structured data\" },\n { name: \"goto\", description: \"Navigate to a URL\" },\n { name: \"wait\", description: \"Wait for a specified time\" },\n { name: \"navback\", description: \"Navigate back in browser history\" },\n { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n ];\n\n const domTools: ToolDefinition[] = [\n {\n name: \"screenshot\",\n description: \"Take a compressed JPEG screenshot for quick visual context\",\n },\n {\n name: \"ariaTree\",\n description:\n \"Get an accessibility (ARIA) hybrid tree for full page context\",\n },\n {\n name: \"act\",\n description: \"Perform a specific atomic action (click, type)\",\n },\n { name: \"keys\", description: \"Press a keyboard key\" },\n { name: \"fillForm\", description: \"Fill out a form\" },\n { name: \"think\", description: \"Think about the task\" },\n { name: \"extract\", description: \"Extract structured data\" },\n { name: \"goto\", description: \"Navigate to a URL\" },\n { name: \"wait\", description: \"Wait for a specified time\" },\n { name: \"navback\", description: \"Navigate back in browser history\" },\n { name: \"scroll\", description: \"Scroll the page x pixels up or down\" },\n ];\n\n const baseTools = isHybridMode ? hybridTools : domTools;\n\n if (hasSearch) {\n baseTools.push({\n name: \"search\",\n description:\n \"Perform a web search and return results. Prefer this over navigating to Google and searching within the page for reliability and efficiency.\",\n });\n }\n\n const filteredTools = baseTools.filter((tool) => !excludeSet.has(tool.name));\n\n const toolLines = filteredTools\n .map((tool) => ` <tool name=\"${tool.name}\">${tool.description}</tool>`)\n .join(\"\\n\");\n\n return `<tools>\\n${toolLines}\\n </tools>`;\n}\n\nexport function buildAgentSystemPrompt(\n options: AgentSystemPromptOptions,\n): string {\n const {\n url,\n executionInstruction,\n mode,\n systemInstructions,\n captchasAutoSolve = false,\n excludeTools,\n variables,\n useSearch = false,\n } = options;\n const localeDate = new Date().toLocaleDateString();\n const isoDate = new Date().toISOString();\n const cdata = (text: string) => `<![CDATA[${text}]]>`;\n\n const isHybridMode = mode === \"hybrid\";\n const hasSearch = useSearch || Boolean(process.env.BRAVE_API_KEY);\n\n // Tools section differs based on mode and excluded tools\n const toolsSection = buildToolsSection(isHybridMode, hasSearch, excludeTools);\n\n // Strategy differs based on mode\n const strategyItems = isHybridMode\n ? [\n `<item>Tool selection priority: Use specific tools (click, type) when elements are visible in viewport for maximum reliability.</item>`,\n `<item>Always use screenshot to get proper grounding of the coordinates you want to type/click into.</item>`,\n `<item>When interacting with an input, always use the type tool to type into the input, over clicking and then typing into it.</item>`,\n `<item>Use ariaTree as a secondary tool when elements aren't visible in screenshot or to get full page context.</item>`,\n `<item>Only use act when element is in ariaTree but NOT visible in screenshot.</item>`,\n ]\n : [\n `<item>Tool selection priority: Use act tool for all clicking and typing on a page.</item>`,\n `<item>Always check ariaTree first to understand full page content without scrolling - it shows all elements including those below the fold.</item>`,\n `<item>When interacting with an input, always use the act tool to type into the input, over clicking and then typing.</item>`,\n `<item>If an element is present in the ariaTree, use act to interact with it directly - this eliminates the need to scroll.</item>`,\n `<item>Use screenshot for visual confirmation when needed, but rely primarily on ariaTree for element detection.</item>`,\n ];\n\n const strategySection = strategyItems.join(\"\\n \");\n\n const commonStrategyItems = `\n <item>CRITICAL: Use extract ONLY when the task explicitly requires structured data output (e.g., \"get job listings\", \"extract product details\"). For reading page content or understanding elements, always use ${isHybridMode ? \"screenshot or ariaTree\" : \"ariaTree\"} instead - it's faster and more reliable.</item>\n <item>Keep actions atomic and verify outcomes before proceeding.</item>\n <item>For each action, provide clear reasoning about why you're taking that step.</item>\n <item>When you need to input text that could be entered character-by-character or through multiple separate inputs, prefer using the keys tool to type the entire sequence at once. This is more efficient for scenarios like verification codes split across multiple fields, or when virtual keyboards are present but direct typing would be faster.</item>\n `;\n\n // Page understanding protocol differs based on mode\n const pageUnderstandingProtocol = isHybridMode\n ? `<page_understanding_protocol>\n <step_1>\n <title>UNDERSTAND THE PAGE</title>\n <primary_tool>\n <name>screenshot</name>\n <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n </primary_tool>\n <secondary_tool>\n <name>ariaTree</name>\n <usage>Get complete page context before taking actions</usage>\n <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n </secondary_tool>\n </step_1>\n </page_understanding_protocol>`\n : `<page_understanding_protocol>\n <step_1>\n <title>UNDERSTAND THE PAGE</title>\n <primary_tool>\n <name>ariaTree</name>\n <usage>Get complete page context before taking actions</usage>\n <benefit>Eliminates the need to scroll and provides full accessible content</benefit>\n </primary_tool>\n <secondary_tool>\n <name>screenshot</name>\n <usage>Visual confirmation when needed. Ideally after navigating to a new page.</usage>\n </secondary_tool>\n </step_1>\n </page_understanding_protocol>`;\n\n // Roadblocks section only shown when captchas are auto-solved\n const roadblocksSection = captchasAutoSolve\n ? `<roadblocks>\n <note>${CAPTCHA_SYSTEM_PROMPT_NOTE}</note>\n </roadblocks>`\n : \"\";\n\n // Build customInstructions block only if provided\n const customInstructionsBlock = systemInstructions\n ? `<customInstructions>${cdata(systemInstructions)}</customInstructions>\\n `\n : \"\";\n\n // Build variables section only if variables are provided\n const hasVariables = variables && Object.keys(variables).length > 0;\n const variableToolsNote = isHybridMode\n ? \"Use %variableName% syntax in the type, fillFormVision, or act tool's value/text/action fields.\"\n : \"Use %variableName% syntax in the act or fillForm tool's action fields.\";\n const variableEntries = getVariablePromptEntries(variables);\n const variablesSection = hasVariables\n ? `<variables>\n <note>You have access to the following variables. Use %variableName% syntax to substitute variable values. This is especially important for sensitive data like passwords.</note>\n <usage>${variableToolsNote}</usage>\n <example>To type a password, use: type %password% into the password field</example>\n ${variableEntries\n .map(({ name, description }) => {\n return description\n ? `<variable name=\"${name}\">${description}</variable>`\n : `<variable name=\"${name}\" />`;\n })\n .join(\"\\n \")}\n </variables>`\n : \"\";\n\n return `<system>\n <identity>You are a web automation assistant using browser automation tools to accomplish the user's goal.</identity>\n ${customInstructionsBlock}<task>\n <goal>${cdata(executionInstruction)}</goal>\n <date display=\"local\" iso=\"${isoDate}\">${localeDate}</date>\n <note>You may think the date is different due to knowledge cutoff, but this is the actual date.</note>\n </task>\n <page>\n <startingUrl>you are starting your task on this url: ${url}</startingUrl>\n </page>\n <mindset>\n <note>Be very intentional about your action. The initial instruction is very important, and slight variations of the actual goal can lead to failures.</note>\n <importantNote>If something fails to meet a single condition of the task, move on from it rather than seeing if it meets other criteria. We only care that it meets all of it</importantNote>\n <note>When the task is complete, do not seek more information; you have completed the task.</note>\n </mindset>\n <guidelines>\n <item>Always start by understanding the current page state</item>\n <item>Use the screenshot tool to verify page state when needed</item>\n <item>Use appropriate tools for each action</item>\n </guidelines>\n ${pageUnderstandingProtocol}\n <navigation>\n <rule>If you are confident in the URL, navigate directly to it.</rule>\n ${hasSearch ? `<rule>If you are not confident in the URL, use the search tool to find it.</rule>` : ``}\n </navigation>\n ${toolsSection}\n <strategy>\n ${strategySection}\n ${commonStrategyItems}\n </strategy>\n ${roadblocksSection}\n ${variablesSection}\n <completion>\n <note>When you complete the task, explain any information that was found that was relevant to the original task.</note>\n <examples>\n <example>If you were asked for specific flights, list the flights you found.</example>\n <example>If you were asked for information about a product, list the product information you were asked for.</example>\n </examples>\n </completion>\n</system>`;\n}\n"]}
|
|
@@ -4,7 +4,6 @@ import type { AgentModelConfig, Variables } from "../../types/public/agent.js";
|
|
|
4
4
|
export declare const fillFormTool: (v3: V3, executionModel?: string | AgentModelConfig, variables?: Variables, toolTimeout?: number) => import("ai").Tool<{
|
|
5
5
|
fields: {
|
|
6
6
|
action: string;
|
|
7
|
-
value: string;
|
|
8
7
|
}[];
|
|
9
8
|
}, {
|
|
10
9
|
success: boolean;
|
|
@@ -6,18 +6,15 @@ const zod_1 = require("zod");
|
|
|
6
6
|
const sdkErrors_js_1 = require("../../types/public/sdkErrors.js");
|
|
7
7
|
const fillFormTool = (v3, executionModel, variables, toolTimeout) => {
|
|
8
8
|
const hasVariables = variables && Object.keys(variables).length > 0;
|
|
9
|
-
const
|
|
10
|
-
? `
|
|
11
|
-
: "
|
|
9
|
+
const actionDescription = hasVariables
|
|
10
|
+
? `Must follow the pattern: "type <exact value> into the <field name> <fieldType>". Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(", ")}. Examples: "type %email% into the email input", "type %password% into the password input"`
|
|
11
|
+
: 'Must follow the pattern: "type <exact value> into the <field name> <fieldType>". Examples: "type john@example.com into the email input", "type John into the first name input"';
|
|
12
12
|
return (0, ai_1.tool)({
|
|
13
|
-
description:
|
|
13
|
+
description: 'FORM FILL - MULTI-FIELD INPUT TOOL\nFill 2+ form inputs/textareas at once. Each action MUST include the exact text to type and the target field, e.g. "type john@example.com into the email field".',
|
|
14
14
|
inputSchema: zod_1.z.object({
|
|
15
15
|
fields: zod_1.z
|
|
16
16
|
.array(zod_1.z.object({
|
|
17
|
-
action: zod_1.z
|
|
18
|
-
.string()
|
|
19
|
-
.describe('Description of typing action, e.g. "type foo into the email field"'),
|
|
20
|
-
value: zod_1.z.string().describe(valueDescription),
|
|
17
|
+
action: zod_1.z.string().describe(actionDescription),
|
|
21
18
|
}))
|
|
22
19
|
.min(1, "Provide at least one field to fill"),
|
|
23
20
|
}),
|
|
@@ -38,8 +35,8 @@ const fillFormTool = (v3, executionModel, variables, toolTimeout) => {
|
|
|
38
35
|
.map((f) => f.action)
|
|
39
36
|
.join(", ")}`;
|
|
40
37
|
const observeOptions = executionModel
|
|
41
|
-
? { model: executionModel, timeout: toolTimeout }
|
|
42
|
-
: { timeout: toolTimeout };
|
|
38
|
+
? { model: executionModel, variables, timeout: toolTimeout }
|
|
39
|
+
: { variables, timeout: toolTimeout };
|
|
43
40
|
const observeResults = await v3.observe(instruction, observeOptions);
|
|
44
41
|
const completed = [];
|
|
45
42
|
const replayableActions = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAIxB,kEAA+D;AAExD,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,
|
|
1
|
+
{"version":3,"file":"fillform.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/tools/fillform.ts"],"names":[],"mappings":";;;AAAA,2BAA0B;AAC1B,6BAAwB;AAIxB,kEAA+D;AAExD,MAAM,YAAY,GAAG,CAC1B,EAAM,EACN,cAA0C,EAC1C,SAAqB,EACrB,WAAoB,EACpB,EAAE;IACF,MAAM,YAAY,GAAG,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACpE,MAAM,iBAAiB,GAAG,YAAY;QACpC,CAAC,CAAC,kJAAkJ,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,4FAA4F;QACjR,CAAC,CAAC,gLAAgL,CAAC;IAErL,OAAO,IAAA,SAAI,EAAC;QACV,WAAW,EACT,qMAAqM;QACvM,WAAW,EAAE,OAAC,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE,OAAC;iBACN,KAAK,CACJ,OAAC,CAAC,MAAM,CAAC;gBACP,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iBAAiB,CAAC;aAC/C,CAAC,CACH;iBACA,GAAG,CAAC,CAAC,EAAE,oCAAoC,CAAC;SAChD,CAAC;QACF,OAAO,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE;YAC5B,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC;oBACR,QAAQ,EAAE,OAAO;oBACjB,OAAO,EAAE,8BAA8B;oBACvC,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE;wBACT,SAAS,EAAE;4BACT,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;4BAC7B,IAAI,EAAE,QAAQ;yBACf;qBACF;iBACF,CAAC,CAAC;gBACH,MAAM,WAAW,GAAG,yDAAyD,MAAM;qBAChF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;qBACpB,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAEhB,MAAM,cAAc,GAAG,cAAc;oBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;oBAC5D,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;gBACxC,MAAM,cAAc,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;gBAErE,MAAM,SAAS,GAAG,EAAe,CAAC;gBAClC,MAAM,iBAAiB,GAAa,EAAE,CAAC;gBACvC,KAAK,MAAM,GAAG,IAAI,cAAc,EAAE,CAAC;oBACjC,MAAM,UAAU,GAAG,SAAS;wBAC1B,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE;wBACrC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;oBAC7B,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;oBAChD,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;wBACrC,iBAAiB,CAAC,IAAI,CAAC,GAAI,SAAS,CAAC,OAAoB,CAAC,CAAC;oBAC7D,CAAC;gBACH,CAAC;gBACD,EAAE,CAAC,qBAAqB,CAAC;oBACvB,IAAI,EAAE,UAAU;oBAChB,MAAM;oBACN,cAAc;oBACd,OAAO,EAAE,iBAAiB;iBAC3B,CAAC,CAAC;gBACH,OAAO;oBACL,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,SAAS;oBAClB,mBAAmB,EAAE,iBAAiB;iBACvC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,KAAK,YAAY,2BAAY,EAAE,CAAC;oBAClC,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,OAAO;oBACL,OAAO,EAAE,KAAK;oBACd,KAAK,EAAE,KAAK,EAAE,OAAO,IAAI,MAAM,CAAC,KAAK,CAAC;iBACvC,CAAC;YACJ,CAAC;QACH,CAAC;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AA/EW,QAAA,YAAY,gBA+EvB","sourcesContent":["import { tool } from \"ai\";\nimport { z } from \"zod\";\nimport type { V3 } from \"../../v3.js\";\nimport type { Action } from \"../../types/public/methods.js\";\nimport type { AgentModelConfig, Variables } from \"../../types/public/agent.js\";\nimport { TimeoutError } from \"../../types/public/sdkErrors.js\";\n\nexport const fillFormTool = (\n v3: V3,\n executionModel?: string | AgentModelConfig,\n variables?: Variables,\n toolTimeout?: number,\n) => {\n const hasVariables = variables && Object.keys(variables).length > 0;\n const actionDescription = hasVariables\n ? `Must follow the pattern: \"type <exact value> into the <field name> <fieldType>\". Use %variableName% to substitute a variable value. Available: ${Object.keys(variables).join(\", \")}. Examples: \"type %email% into the email input\", \"type %password% into the password input\"`\n : 'Must follow the pattern: \"type <exact value> into the <field name> <fieldType>\". Examples: \"type john@example.com into the email input\", \"type John into the first name input\"';\n\n return tool({\n description:\n 'FORM FILL - MULTI-FIELD INPUT TOOL\\nFill 2+ form inputs/textareas at once. Each action MUST include the exact text to type and the target field, e.g. \"type john@example.com into the email field\".',\n inputSchema: z.object({\n fields: z\n .array(\n z.object({\n action: z.string().describe(actionDescription),\n }),\n )\n .min(1, \"Provide at least one field to fill\"),\n }),\n execute: async ({ fields }) => {\n try {\n v3.logger({\n category: \"agent\",\n message: `Agent calling tool: fillForm`,\n level: 1,\n auxiliary: {\n arguments: {\n value: JSON.stringify(fields),\n type: \"object\",\n },\n },\n });\n const instruction = `Return observation results for the following actions: ${fields\n .map((f) => f.action)\n .join(\", \")}`;\n\n const observeOptions = executionModel\n ? { model: executionModel, variables, timeout: toolTimeout }\n : { variables, timeout: toolTimeout };\n const observeResults = await v3.observe(instruction, observeOptions);\n\n const completed = [] as unknown[];\n const replayableActions: Action[] = [];\n for (const res of observeResults) {\n const actOptions = variables\n ? { variables, timeout: toolTimeout }\n : { timeout: toolTimeout };\n const actResult = await v3.act(res, actOptions);\n completed.push(actResult);\n if (Array.isArray(actResult.actions)) {\n replayableActions.push(...(actResult.actions as Action[]));\n }\n }\n v3.recordAgentReplayStep({\n type: \"fillForm\",\n fields,\n observeResults,\n actions: replayableActions,\n });\n return {\n success: true,\n actions: completed,\n playwrightArguments: replayableActions,\n };\n } catch (error) {\n if (error instanceof TimeoutError) {\n throw error;\n }\n return {\n success: false,\n error: error?.message ?? String(error),\n };\n }\n },\n });\n};\n"]}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { Page } from "../../understudy/page.js";
|
|
2
|
+
/** Injected into the agent message stream after a successful captcha solve. */
|
|
3
|
+
export declare const CAPTCHA_SOLVED_MSG = "A captcha was automatically detected and solved \u2014 no further interaction with the captcha is needed, even if it does not visually appear solved. Do not click the captcha checkbox, widget, or challenge again. Continue with your task.";
|
|
4
|
+
/** Injected into the agent message stream when the captcha solver fails. */
|
|
5
|
+
export declare const CAPTCHA_ERRORED_MSG = "A captcha was detected but the automatic captcha solver failed to solve it. You may need to try a different approach or navigate around the captcha.";
|
|
6
|
+
/** Appended to the system prompt (DOM/hybrid agents) when captchas auto-solve. */
|
|
7
|
+
export declare const CAPTCHA_SYSTEM_PROMPT_NOTE = "Captchas on this page are automatically detected and solved by the browser environment. Do not interact with or attempt to solve any captchas yourself \u2014 they will be handled for you. Do not click the captcha checkbox, widget, or challenge again after it has been solved, even if it still looks unresolved. Continue with your task as if the captcha does not exist.";
|
|
8
|
+
/** Appended to the CUA system prompt when captchas auto-solve. */
|
|
9
|
+
export declare const CAPTCHA_CUA_SYSTEM_PROMPT_NOTE = "\n\nCaptchas on this page are automatically detected and solved by the browser environment. Do not interact with or attempt to solve any captchas yourself \u2014 they will be handled for you. Continue with your task as if the captcha does not exist.";
|
|
10
|
+
/**
|
|
11
|
+
* Tracks Browserbase captcha solver state via console messages and provides
|
|
12
|
+
* a blocking `waitIfSolving()` that agents call before each step/action.
|
|
13
|
+
*
|
|
14
|
+
* Accepts a page-provider callback so the listener is automatically
|
|
15
|
+
* re-attached when the active page changes (e.g. popup / new tab).
|
|
16
|
+
*
|
|
17
|
+
* All concurrent callers of `waitIfSolving()` share the same underlying
|
|
18
|
+
* promise, so multiple waiters are safely resolved together.
|
|
19
|
+
*/
|
|
20
|
+
export declare class CaptchaSolver {
|
|
21
|
+
private solving;
|
|
22
|
+
private _solvedSinceLastConsume;
|
|
23
|
+
private _erroredSinceLastConsume;
|
|
24
|
+
private listener;
|
|
25
|
+
private attachedPage;
|
|
26
|
+
private pageProvider;
|
|
27
|
+
/** Shared promise that all concurrent waitIfSolving() callers await. */
|
|
28
|
+
private waitPromise;
|
|
29
|
+
/** Resolves the shared waitPromise. */
|
|
30
|
+
private resolveWait;
|
|
31
|
+
/** Timeout handle for the 90s deadline. */
|
|
32
|
+
private waitTimer;
|
|
33
|
+
/**
|
|
34
|
+
* Initialise with a callback that returns the current active page.
|
|
35
|
+
* The listener is lazily (re-)attached whenever the active page changes.
|
|
36
|
+
*/
|
|
37
|
+
init(pageProvider: () => Promise<Page>): void;
|
|
38
|
+
/** Whether a captcha solve is currently in progress. */
|
|
39
|
+
isSolving(): boolean;
|
|
40
|
+
/**
|
|
41
|
+
* Ensure the console listener is attached to the current active page.
|
|
42
|
+
* If the active page has changed since the last call, the old listener
|
|
43
|
+
* is removed and a new one is installed.
|
|
44
|
+
*/
|
|
45
|
+
ensureAttached(): Promise<void>;
|
|
46
|
+
/**
|
|
47
|
+
* Returns a promise that resolves immediately if no captcha is being
|
|
48
|
+
* solved, or blocks until the solver finishes, errors, or the 90s
|
|
49
|
+
* timeout is reached.
|
|
50
|
+
*
|
|
51
|
+
* Also re-attaches the listener to the current active page if it has
|
|
52
|
+
* changed since the last call.
|
|
53
|
+
*
|
|
54
|
+
* All concurrent callers share the same promise, so no waiter is
|
|
55
|
+
* orphaned.
|
|
56
|
+
*/
|
|
57
|
+
waitIfSolving(): Promise<void>;
|
|
58
|
+
/**
|
|
59
|
+
* Returns and resets the solve event flags.
|
|
60
|
+
* Call after `waitIfSolving()` to check whether a captcha was solved
|
|
61
|
+
* (or errored) since the last consume. This captures events even if
|
|
62
|
+
* the solve completed between two `waitIfSolving()` calls.
|
|
63
|
+
*/
|
|
64
|
+
consumeSolveResult(): {
|
|
65
|
+
solved: boolean;
|
|
66
|
+
errored: boolean;
|
|
67
|
+
};
|
|
68
|
+
/**
|
|
69
|
+
* Remove the console listener and reset all state.
|
|
70
|
+
*/
|
|
71
|
+
dispose(): void;
|
|
72
|
+
/** Remove the console listener from the currently attached page. */
|
|
73
|
+
private detachListener;
|
|
74
|
+
/** Resolve the shared wait promise and clear the timeout. */
|
|
75
|
+
private settle;
|
|
76
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CaptchaSolver = exports.CAPTCHA_CUA_SYSTEM_PROMPT_NOTE = exports.CAPTCHA_SYSTEM_PROMPT_NOTE = exports.CAPTCHA_ERRORED_MSG = exports.CAPTCHA_SOLVED_MSG = void 0;
|
|
4
|
+
const SOLVING_STARTED = "browserbase-solving-started";
|
|
5
|
+
const SOLVING_FINISHED = "browserbase-solving-finished";
|
|
6
|
+
const SOLVING_ERRORED = "browserbase-solving-errored";
|
|
7
|
+
/** Maximum time (ms) to wait for the captcha solver before giving up. */
|
|
8
|
+
const SOLVE_TIMEOUT_MS = 90_000;
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
// Shared captcha notification strings
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
/** Injected into the agent message stream after a successful captcha solve. */
|
|
13
|
+
exports.CAPTCHA_SOLVED_MSG = "A captcha was automatically detected and solved — no further interaction with the captcha is needed, even if it does not visually appear solved. Do not click the captcha checkbox, widget, or challenge again. Continue with your task.";
|
|
14
|
+
/** Injected into the agent message stream when the captcha solver fails. */
|
|
15
|
+
exports.CAPTCHA_ERRORED_MSG = "A captcha was detected but the automatic captcha solver failed to solve it. You may need to try a different approach or navigate around the captcha.";
|
|
16
|
+
/** Appended to the system prompt (DOM/hybrid agents) when captchas auto-solve. */
|
|
17
|
+
exports.CAPTCHA_SYSTEM_PROMPT_NOTE = "Captchas on this page are automatically detected and solved by the browser environment. Do not interact with or attempt to solve any captchas yourself — they will be handled for you. Do not click the captcha checkbox, widget, or challenge again after it has been solved, even if it still looks unresolved. Continue with your task as if the captcha does not exist.";
|
|
18
|
+
/** Appended to the CUA system prompt when captchas auto-solve. */
|
|
19
|
+
exports.CAPTCHA_CUA_SYSTEM_PROMPT_NOTE = "\n\nCaptchas on this page are automatically detected and solved by the browser environment. Do not interact with or attempt to solve any captchas yourself — they will be handled for you. Continue with your task as if the captcha does not exist.";
|
|
20
|
+
/**
|
|
21
|
+
* Tracks Browserbase captcha solver state via console messages and provides
|
|
22
|
+
* a blocking `waitIfSolving()` that agents call before each step/action.
|
|
23
|
+
*
|
|
24
|
+
* Accepts a page-provider callback so the listener is automatically
|
|
25
|
+
* re-attached when the active page changes (e.g. popup / new tab).
|
|
26
|
+
*
|
|
27
|
+
* All concurrent callers of `waitIfSolving()` share the same underlying
|
|
28
|
+
* promise, so multiple waiters are safely resolved together.
|
|
29
|
+
*/
|
|
30
|
+
class CaptchaSolver {
|
|
31
|
+
solving = false;
|
|
32
|
+
_solvedSinceLastConsume = false;
|
|
33
|
+
_erroredSinceLastConsume = false;
|
|
34
|
+
listener = null;
|
|
35
|
+
attachedPage = null;
|
|
36
|
+
pageProvider = null;
|
|
37
|
+
/** Shared promise that all concurrent waitIfSolving() callers await. */
|
|
38
|
+
waitPromise = null;
|
|
39
|
+
/** Resolves the shared waitPromise. */
|
|
40
|
+
resolveWait = null;
|
|
41
|
+
/** Timeout handle for the 90s deadline. */
|
|
42
|
+
waitTimer = null;
|
|
43
|
+
/**
|
|
44
|
+
* Initialise with a callback that returns the current active page.
|
|
45
|
+
* The listener is lazily (re-)attached whenever the active page changes.
|
|
46
|
+
*/
|
|
47
|
+
init(pageProvider) {
|
|
48
|
+
this.pageProvider = pageProvider;
|
|
49
|
+
}
|
|
50
|
+
/** Whether a captcha solve is currently in progress. */
|
|
51
|
+
isSolving() {
|
|
52
|
+
return this.solving;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Ensure the console listener is attached to the current active page.
|
|
56
|
+
* If the active page has changed since the last call, the old listener
|
|
57
|
+
* is removed and a new one is installed.
|
|
58
|
+
*/
|
|
59
|
+
async ensureAttached() {
|
|
60
|
+
if (!this.pageProvider)
|
|
61
|
+
return;
|
|
62
|
+
const page = await this.pageProvider();
|
|
63
|
+
if (page === this.attachedPage)
|
|
64
|
+
return;
|
|
65
|
+
// Detach from the old page
|
|
66
|
+
this.detachListener();
|
|
67
|
+
this.attachedPage = page;
|
|
68
|
+
this.listener = (msg) => {
|
|
69
|
+
const text = msg.text();
|
|
70
|
+
if (text === SOLVING_STARTED) {
|
|
71
|
+
this.solving = true;
|
|
72
|
+
}
|
|
73
|
+
else if (text === SOLVING_FINISHED) {
|
|
74
|
+
this.solving = false;
|
|
75
|
+
this._solvedSinceLastConsume = true;
|
|
76
|
+
this.settle();
|
|
77
|
+
}
|
|
78
|
+
else if (text === SOLVING_ERRORED) {
|
|
79
|
+
this.solving = false;
|
|
80
|
+
this._erroredSinceLastConsume = true;
|
|
81
|
+
this.settle();
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
page.on("console", this.listener);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Returns a promise that resolves immediately if no captcha is being
|
|
88
|
+
* solved, or blocks until the solver finishes, errors, or the 90s
|
|
89
|
+
* timeout is reached.
|
|
90
|
+
*
|
|
91
|
+
* Also re-attaches the listener to the current active page if it has
|
|
92
|
+
* changed since the last call.
|
|
93
|
+
*
|
|
94
|
+
* All concurrent callers share the same promise, so no waiter is
|
|
95
|
+
* orphaned.
|
|
96
|
+
*/
|
|
97
|
+
async waitIfSolving() {
|
|
98
|
+
await this.ensureAttached();
|
|
99
|
+
if (!this.solving)
|
|
100
|
+
return;
|
|
101
|
+
// Return the existing shared promise if one is already pending
|
|
102
|
+
if (this.waitPromise)
|
|
103
|
+
return this.waitPromise;
|
|
104
|
+
this.waitPromise = new Promise((resolve) => {
|
|
105
|
+
this.resolveWait = resolve;
|
|
106
|
+
this.waitTimer = setTimeout(() => {
|
|
107
|
+
this.solving = false;
|
|
108
|
+
this._erroredSinceLastConsume = true;
|
|
109
|
+
this.settle();
|
|
110
|
+
}, SOLVE_TIMEOUT_MS);
|
|
111
|
+
});
|
|
112
|
+
return this.waitPromise;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Returns and resets the solve event flags.
|
|
116
|
+
* Call after `waitIfSolving()` to check whether a captcha was solved
|
|
117
|
+
* (or errored) since the last consume. This captures events even if
|
|
118
|
+
* the solve completed between two `waitIfSolving()` calls.
|
|
119
|
+
*/
|
|
120
|
+
consumeSolveResult() {
|
|
121
|
+
const result = {
|
|
122
|
+
solved: this._solvedSinceLastConsume,
|
|
123
|
+
errored: this._erroredSinceLastConsume,
|
|
124
|
+
};
|
|
125
|
+
this._solvedSinceLastConsume = false;
|
|
126
|
+
this._erroredSinceLastConsume = false;
|
|
127
|
+
return result;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Remove the console listener and reset all state.
|
|
131
|
+
*/
|
|
132
|
+
dispose() {
|
|
133
|
+
this.detachListener();
|
|
134
|
+
this.attachedPage = null;
|
|
135
|
+
this.pageProvider = null;
|
|
136
|
+
this.solving = false;
|
|
137
|
+
this._solvedSinceLastConsume = false;
|
|
138
|
+
this._erroredSinceLastConsume = false;
|
|
139
|
+
this.settle();
|
|
140
|
+
}
|
|
141
|
+
// ------------------------------------------------------------------
|
|
142
|
+
// Internal helpers
|
|
143
|
+
// ------------------------------------------------------------------
|
|
144
|
+
/** Remove the console listener from the currently attached page. */
|
|
145
|
+
detachListener() {
|
|
146
|
+
if (this.attachedPage && this.listener) {
|
|
147
|
+
this.attachedPage.off("console", this.listener);
|
|
148
|
+
}
|
|
149
|
+
this.listener = null;
|
|
150
|
+
// If a solve was in progress, mark it as errored so consumers
|
|
151
|
+
// know it was interrupted (consistent with the timeout path).
|
|
152
|
+
if (this.solving) {
|
|
153
|
+
this._erroredSinceLastConsume = true;
|
|
154
|
+
}
|
|
155
|
+
// Reset solving state so waiters aren't stuck waiting for events
|
|
156
|
+
// that can never arrive from the detached page.
|
|
157
|
+
this.solving = false;
|
|
158
|
+
this.settle();
|
|
159
|
+
}
|
|
160
|
+
/** Resolve the shared wait promise and clear the timeout. */
|
|
161
|
+
settle() {
|
|
162
|
+
if (this.waitTimer) {
|
|
163
|
+
clearTimeout(this.waitTimer);
|
|
164
|
+
this.waitTimer = null;
|
|
165
|
+
}
|
|
166
|
+
if (this.resolveWait) {
|
|
167
|
+
const resolve = this.resolveWait;
|
|
168
|
+
this.resolveWait = null;
|
|
169
|
+
this.waitPromise = null;
|
|
170
|
+
resolve();
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
exports.CaptchaSolver = CaptchaSolver;
|
|
175
|
+
//# sourceMappingURL=captchaSolver.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"captchaSolver.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/captchaSolver.ts"],"names":[],"mappings":";;;AAGA,MAAM,eAAe,GAAG,6BAA6B,CAAC;AACtD,MAAM,gBAAgB,GAAG,8BAA8B,CAAC;AACxD,MAAM,eAAe,GAAG,6BAA6B,CAAC;AAEtD,yEAAyE;AACzE,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,8EAA8E;AAC9E,sCAAsC;AACtC,8EAA8E;AAE9E,+EAA+E;AAClE,QAAA,kBAAkB,GAC7B,0OAA0O,CAAC;AAE7O,4EAA4E;AAC/D,QAAA,mBAAmB,GAC9B,sJAAsJ,CAAC;AAEzJ,kFAAkF;AACrE,QAAA,0BAA0B,GACrC,6WAA6W,CAAC;AAEhX,kEAAkE;AACrD,QAAA,8BAA8B,GACzC,sPAAsP,CAAC;AAEzP;;;;;;;;;GASG;AACH,MAAa,aAAa;IAChB,OAAO,GAAG,KAAK,CAAC;IAChB,uBAAuB,GAAG,KAAK,CAAC;IAChC,wBAAwB,GAAG,KAAK,CAAC;IACjC,QAAQ,GAA2C,IAAI,CAAC;IACxD,YAAY,GAAgB,IAAI,CAAC;IACjC,YAAY,GAAiC,IAAI,CAAC;IAE1D,wEAAwE;IAChE,WAAW,GAAyB,IAAI,CAAC;IACjD,uCAAuC;IAC/B,WAAW,GAAwB,IAAI,CAAC;IAChD,2CAA2C;IACnC,SAAS,GAAyC,IAAI,CAAC;IAE/D;;;OAGG;IACH,IAAI,CAAC,YAAiC;QACpC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,wDAAwD;IACxD,SAAS;QACP,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO;QAC/B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACvC,IAAI,IAAI,KAAK,IAAI,CAAC,YAAY;YAAE,OAAO;QAEvC,2BAA2B;QAC3B,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,CAAC,GAAmB,EAAE,EAAE;YACtC,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;YACxB,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;gBAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,CAAC;iBAAM,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;gBACrC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,IAAI,CAAC,uBAAuB,GAAG,IAAI,CAAC;gBACpC,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;iBAAM,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;gBACpC,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;gBACrC,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC;QACH,CAAC,CAAC;QACF,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,aAAa;QACjB,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAE5B,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAE1B,+DAA+D;QAC/D,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAE9C,IAAI,CAAC,WAAW,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC/C,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC;YAC3B,IAAI,CAAC,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC/B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;gBACrC,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,CAAC,EAAE,gBAAgB,CAAC,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAED;;;;;OAKG;IACH,kBAAkB;QAChB,MAAM,MAAM,GAAG;YACb,MAAM,EAAE,IAAI,CAAC,uBAAuB;YACpC,OAAO,EAAE,IAAI,CAAC,wBAAwB;SACvC,CAAC;QACF,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;QACrC,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QACtC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,cAAc,EAAE,CAAC;QACtB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,uBAAuB,GAAG,KAAK,CAAC;QACrC,IAAI,CAAC,wBAAwB,GAAG,KAAK,CAAC;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,qEAAqE;IACrE,mBAAmB;IACnB,qEAAqE;IAErE,oEAAoE;IAC5D,cAAc;QACpB,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACvC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClD,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,8DAA8D;QAC9D,8DAA8D;QAC9D,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC;QACvC,CAAC;QACD,iEAAiE;QACjE,gDAAgD;QAChD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,MAAM,EAAE,CAAC;IAChB,CAAC;IAED,6DAA6D;IACrD,MAAM;QACZ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC7B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC;YACjC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;CACF;AAzJD,sCAyJC","sourcesContent":["import type { Page } from \"../../understudy/page.js\";\nimport type { ConsoleMessage } from \"../../understudy/consoleMessage.js\";\n\nconst SOLVING_STARTED = \"browserbase-solving-started\";\nconst SOLVING_FINISHED = \"browserbase-solving-finished\";\nconst SOLVING_ERRORED = \"browserbase-solving-errored\";\n\n/** Maximum time (ms) to wait for the captcha solver before giving up. */\nconst SOLVE_TIMEOUT_MS = 90_000;\n\n// ---------------------------------------------------------------------------\n// Shared captcha notification strings\n// ---------------------------------------------------------------------------\n\n/** Injected into the agent message stream after a successful captcha solve. */\nexport const CAPTCHA_SOLVED_MSG =\n \"A captcha was automatically detected and solved — no further interaction with the captcha is needed, even if it does not visually appear solved. Do not click the captcha checkbox, widget, or challenge again. Continue with your task.\";\n\n/** Injected into the agent message stream when the captcha solver fails. */\nexport const CAPTCHA_ERRORED_MSG =\n \"A captcha was detected but the automatic captcha solver failed to solve it. You may need to try a different approach or navigate around the captcha.\";\n\n/** Appended to the system prompt (DOM/hybrid agents) when captchas auto-solve. */\nexport const CAPTCHA_SYSTEM_PROMPT_NOTE =\n \"Captchas on this page are automatically detected and solved by the browser environment. Do not interact with or attempt to solve any captchas yourself — they will be handled for you. Do not click the captcha checkbox, widget, or challenge again after it has been solved, even if it still looks unresolved. Continue with your task as if the captcha does not exist.\";\n\n/** Appended to the CUA system prompt when captchas auto-solve. */\nexport const CAPTCHA_CUA_SYSTEM_PROMPT_NOTE =\n \"\\n\\nCaptchas on this page are automatically detected and solved by the browser environment. Do not interact with or attempt to solve any captchas yourself — they will be handled for you. Continue with your task as if the captcha does not exist.\";\n\n/**\n * Tracks Browserbase captcha solver state via console messages and provides\n * a blocking `waitIfSolving()` that agents call before each step/action.\n *\n * Accepts a page-provider callback so the listener is automatically\n * re-attached when the active page changes (e.g. popup / new tab).\n *\n * All concurrent callers of `waitIfSolving()` share the same underlying\n * promise, so multiple waiters are safely resolved together.\n */\nexport class CaptchaSolver {\n private solving = false;\n private _solvedSinceLastConsume = false;\n private _erroredSinceLastConsume = false;\n private listener: ((msg: ConsoleMessage) => void) | null = null;\n private attachedPage: Page | null = null;\n private pageProvider: (() => Promise<Page>) | null = null;\n\n /** Shared promise that all concurrent waitIfSolving() callers await. */\n private waitPromise: Promise<void> | null = null;\n /** Resolves the shared waitPromise. */\n private resolveWait: (() => void) | null = null;\n /** Timeout handle for the 90s deadline. */\n private waitTimer: ReturnType<typeof setTimeout> | null = null;\n\n /**\n * Initialise with a callback that returns the current active page.\n * The listener is lazily (re-)attached whenever the active page changes.\n */\n init(pageProvider: () => Promise<Page>): void {\n this.pageProvider = pageProvider;\n }\n\n /** Whether a captcha solve is currently in progress. */\n isSolving(): boolean {\n return this.solving;\n }\n\n /**\n * Ensure the console listener is attached to the current active page.\n * If the active page has changed since the last call, the old listener\n * is removed and a new one is installed.\n */\n async ensureAttached(): Promise<void> {\n if (!this.pageProvider) return;\n const page = await this.pageProvider();\n if (page === this.attachedPage) return;\n\n // Detach from the old page\n this.detachListener();\n\n this.attachedPage = page;\n this.listener = (msg: ConsoleMessage) => {\n const text = msg.text();\n if (text === SOLVING_STARTED) {\n this.solving = true;\n } else if (text === SOLVING_FINISHED) {\n this.solving = false;\n this._solvedSinceLastConsume = true;\n this.settle();\n } else if (text === SOLVING_ERRORED) {\n this.solving = false;\n this._erroredSinceLastConsume = true;\n this.settle();\n }\n };\n page.on(\"console\", this.listener);\n }\n\n /**\n * Returns a promise that resolves immediately if no captcha is being\n * solved, or blocks until the solver finishes, errors, or the 90s\n * timeout is reached.\n *\n * Also re-attaches the listener to the current active page if it has\n * changed since the last call.\n *\n * All concurrent callers share the same promise, so no waiter is\n * orphaned.\n */\n async waitIfSolving(): Promise<void> {\n await this.ensureAttached();\n\n if (!this.solving) return;\n\n // Return the existing shared promise if one is already pending\n if (this.waitPromise) return this.waitPromise;\n\n this.waitPromise = new Promise<void>((resolve) => {\n this.resolveWait = resolve;\n this.waitTimer = setTimeout(() => {\n this.solving = false;\n this._erroredSinceLastConsume = true;\n this.settle();\n }, SOLVE_TIMEOUT_MS);\n });\n\n return this.waitPromise;\n }\n\n /**\n * Returns and resets the solve event flags.\n * Call after `waitIfSolving()` to check whether a captcha was solved\n * (or errored) since the last consume. This captures events even if\n * the solve completed between two `waitIfSolving()` calls.\n */\n consumeSolveResult(): { solved: boolean; errored: boolean } {\n const result = {\n solved: this._solvedSinceLastConsume,\n errored: this._erroredSinceLastConsume,\n };\n this._solvedSinceLastConsume = false;\n this._erroredSinceLastConsume = false;\n return result;\n }\n\n /**\n * Remove the console listener and reset all state.\n */\n dispose(): void {\n this.detachListener();\n this.attachedPage = null;\n this.pageProvider = null;\n this.solving = false;\n this._solvedSinceLastConsume = false;\n this._erroredSinceLastConsume = false;\n this.settle();\n }\n\n // ------------------------------------------------------------------\n // Internal helpers\n // ------------------------------------------------------------------\n\n /** Remove the console listener from the currently attached page. */\n private detachListener(): void {\n if (this.attachedPage && this.listener) {\n this.attachedPage.off(\"console\", this.listener);\n }\n this.listener = null;\n // If a solve was in progress, mark it as errored so consumers\n // know it was interrupted (consistent with the timeout path).\n if (this.solving) {\n this._erroredSinceLastConsume = true;\n }\n // Reset solving state so waiters aren't stuck waiting for events\n // that can never arrive from the detached page.\n this.solving = false;\n this.settle();\n }\n\n /** Resolve the shared wait promise and clear the timeout. */\n private settle(): void {\n if (this.waitTimer) {\n clearTimeout(this.waitTimer);\n this.waitTimer = null;\n }\n if (this.resolveWait) {\n const resolve = this.resolveWait;\n this.resolveWait = null;\n this.waitPromise = null;\n resolve();\n }\n }\n}\n"]}
|
|
@@ -9,6 +9,11 @@ export declare function resolveVariableValue(v: VariableValue): string;
|
|
|
9
9
|
* Returns undefined for simple primitive values.
|
|
10
10
|
*/
|
|
11
11
|
export declare function getVariableDescription(v: VariableValue): string | undefined;
|
|
12
|
+
export interface VariablePromptEntry {
|
|
13
|
+
name: string;
|
|
14
|
+
description?: string;
|
|
15
|
+
}
|
|
16
|
+
export declare function getVariablePromptEntries(variables?: Variables): VariablePromptEntry[];
|
|
12
17
|
/**
|
|
13
18
|
* Substitutes %variableName% tokens in text with resolved variable values.
|
|
14
19
|
* Works with both simple and rich variable formats.
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.resolveVariableValue = resolveVariableValue;
|
|
4
4
|
exports.getVariableDescription = getVariableDescription;
|
|
5
|
+
exports.getVariablePromptEntries = getVariablePromptEntries;
|
|
5
6
|
exports.substituteVariables = substituteVariables;
|
|
6
7
|
exports.flattenVariables = flattenVariables;
|
|
7
8
|
/**
|
|
@@ -24,6 +25,14 @@ function getVariableDescription(v) {
|
|
|
24
25
|
}
|
|
25
26
|
return undefined;
|
|
26
27
|
}
|
|
28
|
+
function getVariablePromptEntries(variables) {
|
|
29
|
+
if (!variables)
|
|
30
|
+
return [];
|
|
31
|
+
return Object.entries(variables).map(([name, value]) => ({
|
|
32
|
+
name,
|
|
33
|
+
description: getVariableDescription(value),
|
|
34
|
+
}));
|
|
35
|
+
}
|
|
27
36
|
/**
|
|
28
37
|
* Substitutes %variableName% tokens in text with resolved variable values.
|
|
29
38
|
* Works with both simple and rich variable formats.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"variables.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/variables.ts"],"names":[],"mappings":";;AAMA,oDAKC;AAMD,wDAKC;AAMD,kDAWC;AAMD,4CASC;
|
|
1
|
+
{"version":3,"file":"variables.js","sourceRoot":"","sources":["../../../../../../lib/v3/agent/utils/variables.ts"],"names":[],"mappings":";;AAMA,oDAKC;AAMD,wDAKC;AAOD,4DAQC;AAMD,kDAWC;AAMD,4CASC;AAnED;;;GAGG;AACH,SAAgB,oBAAoB,CAAC,CAAgB;IACnD,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACxD,OAAO,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,SAAgB,sBAAsB,CAAC,CAAgB;IACrD,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACxD,OAAO,CAAC,CAAC,WAAW,CAAC;IACvB,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAOD,SAAgB,wBAAwB,CACtC,SAAqB;IAErB,IAAI,CAAC,SAAS;QAAE,OAAO,EAAE,CAAC;IAC1B,OAAO,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QACvD,IAAI;QACJ,WAAW,EAAE,sBAAsB,CAAC,KAAK,CAAC;KAC3C,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,SAAgB,mBAAmB,CACjC,IAAY,EACZ,SAAqB;IAErB,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,MAAM,GAAG,IAAI,CAAC;IAClB,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,IAAI,GAAG,GAAG,CAAC;QACzB,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAC9B,SAAqB;IAErB,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACxE,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC","sourcesContent":["import type { Variables, VariableValue } from \"../../types/public/agent.js\";\n\n/**\n * Resolves a VariableValue to its primitive string value.\n * Handles both simple primitives (\"secret\") and rich objects ({ value: \"secret\", description: \"...\" }).\n */\nexport function resolveVariableValue(v: VariableValue): string {\n if (typeof v === \"object\" && v !== null && \"value\" in v) {\n return String(v.value);\n }\n return String(v);\n}\n\n/**\n * Extracts the optional description from a VariableValue.\n * Returns undefined for simple primitive values.\n */\nexport function getVariableDescription(v: VariableValue): string | undefined {\n if (typeof v === \"object\" && v !== null && \"value\" in v) {\n return v.description;\n }\n return undefined;\n}\n\nexport interface VariablePromptEntry {\n name: string;\n description?: string;\n}\n\nexport function getVariablePromptEntries(\n variables?: Variables,\n): VariablePromptEntry[] {\n if (!variables) return [];\n return Object.entries(variables).map(([name, value]) => ({\n name,\n description: getVariableDescription(value),\n }));\n}\n\n/**\n * Substitutes %variableName% tokens in text with resolved variable values.\n * Works with both simple and rich variable formats.\n */\nexport function substituteVariables(\n text: string,\n variables?: Variables,\n): string {\n if (!variables) return text;\n let result = text;\n for (const [key, v] of Object.entries(variables)) {\n const token = `%${key}%`;\n result = result.split(token).join(resolveVariableValue(v));\n }\n return result;\n}\n\n/**\n * Flattens Variables to Record<string, string> for internal consumers\n * that only need key→value mappings (e.g., actHandler, cache replay).\n */\nexport function flattenVariables(\n variables?: Variables,\n): Record<string, string> | undefined {\n if (!variables || Object.keys(variables).length === 0) return undefined;\n const result: Record<string, string> = {};\n for (const [key, v] of Object.entries(variables)) {\n result[key] = resolveVariableValue(v);\n }\n return result;\n}\n"]}
|
package/dist/cjs/lib/v3/api.js
CHANGED
|
@@ -89,7 +89,6 @@ class StagehandAPIClient {
|
|
|
89
89
|
method: "POST",
|
|
90
90
|
body: JSON.stringify(requestBody),
|
|
91
91
|
});
|
|
92
|
-
console.log(JSON.stringify(requestBody, null, 2));
|
|
93
92
|
if (sessionResponse.status === 401) {
|
|
94
93
|
throw new index_js_1.StagehandAPIUnauthorizedError("Unauthorized. Ensure you provided a valid API key.");
|
|
95
94
|
}
|
|
@@ -240,9 +239,6 @@ class StagehandAPIClient {
|
|
|
240
239
|
frameId,
|
|
241
240
|
shouldCache,
|
|
242
241
|
};
|
|
243
|
-
console.log(JSON.stringify(requestBody, null, 2));
|
|
244
|
-
console.log(JSON.stringify(requestBody, null, 2));
|
|
245
|
-
console.log(JSON.stringify(requestBody, null, 2));
|
|
246
242
|
const result = await this.execute({
|
|
247
243
|
method: "agentExecute",
|
|
248
244
|
args: requestBody,
|
|
@@ -579,16 +575,13 @@ class StagehandAPIClient {
|
|
|
579
575
|
else {
|
|
580
576
|
baseUrl = getApiUrlForRegion(this.region);
|
|
581
577
|
}
|
|
582
|
-
const
|
|
583
|
-
const fetchOptions = {
|
|
578
|
+
const response = await this.fetchWithCookies(`${baseUrl}${path}`, {
|
|
584
579
|
...options,
|
|
585
580
|
headers: {
|
|
586
581
|
...defaultHeaders,
|
|
587
582
|
...options.headers,
|
|
588
583
|
},
|
|
589
|
-
};
|
|
590
|
-
console.log("Fetch request:", JSON.stringify({ url: fetchUrl, ...fetchOptions }, null, 2));
|
|
591
|
-
const response = await this.fetchWithCookies(fetchUrl, fetchOptions);
|
|
584
|
+
});
|
|
592
585
|
return response;
|
|
593
586
|
}
|
|
594
587
|
}
|