minimal-agent 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/package.json +60 -4
- package/plugins/computer_use/.claude-plugin/plugin.json +1 -1
- package/plugins/cron-runner/src/commands.js +4 -4
- package/plugins/cron-runner/src/commands.ts +4 -4
- package/plugins/workflow-runner/.claude-plugin/plugin.json +2 -2
- package/plugins/workflow-runner/commands/workflow.md +57 -90
- package/plugins/workflow-runner/commands/workflows.md +4 -4
- package/plugins/workflow-runner/plugin.js +24 -13
- package/plugins/workflow-runner/plugin.ts +28 -13
- package/plugins/workflow-runner/src/discovery.js +160 -0
- package/plugins/workflow-runner/src/discovery.ts +194 -0
- package/plugins/workflow-runner/src/goalAnchor.js +53 -0
- package/plugins/workflow-runner/src/goalAnchor.ts +59 -0
- package/plugins/workflow-runner/src/index.js +197 -206
- package/plugins/workflow-runner/src/index.ts +248 -249
- package/plugins/workflow-runner/src/inputCoercion.js +40 -0
- package/plugins/workflow-runner/src/inputCoercion.ts +43 -0
- package/plugins/workflow-runner/src/journal.js +224 -0
- package/plugins/workflow-runner/src/journal.ts +274 -0
- package/plugins/workflow-runner/src/jsonSchema.js +132 -0
- package/plugins/workflow-runner/src/jsonSchema.ts +162 -0
- package/plugins/workflow-runner/src/moduleResolution.js +64 -0
- package/plugins/workflow-runner/src/moduleResolution.ts +71 -0
- package/plugins/workflow-runner/src/resourceInputs.js +220 -0
- package/plugins/workflow-runner/src/resourceInputs.ts +300 -0
- package/plugins/workflow-runner/src/runner.js +400 -489
- package/plugins/workflow-runner/src/runner.ts +488 -560
- package/plugins/workflow-runner/src/runtime/config.js +46 -0
- package/plugins/workflow-runner/src/runtime/config.ts +69 -0
- package/plugins/workflow-runner/src/runtime/context.js +1304 -0
- package/plugins/workflow-runner/src/runtime/context.ts +1786 -0
- package/plugins/workflow-runner/src/runtime/exec.js +34 -0
- package/plugins/workflow-runner/src/runtime/exec.ts +53 -0
- package/plugins/workflow-runner/src/runtime/flow.js +8 -0
- package/plugins/workflow-runner/src/runtime/flow.ts +12 -0
- package/plugins/workflow-runner/src/runtime/fs.js +27 -0
- package/plugins/workflow-runner/src/runtime/fs.ts +41 -0
- package/plugins/workflow-runner/src/runtime/model.js +6 -0
- package/plugins/workflow-runner/src/runtime/model.ts +8 -0
- package/plugins/workflow-runner/src/runtime/output.js +7 -0
- package/plugins/workflow-runner/src/runtime/output.ts +21 -0
- package/plugins/workflow-runner/src/runtime/scheduler.js +141 -0
- package/plugins/workflow-runner/src/runtime/scheduler.ts +166 -0
- package/plugins/workflow-runner/src/runtime/state.js +16 -0
- package/plugins/workflow-runner/src/runtime/state.ts +19 -0
- package/plugins/workflow-runner/src/runtime/tools.js +3 -0
- package/plugins/workflow-runner/src/runtime/tools.ts +6 -0
- package/plugins/workflow-runner/src/sandbox.js +141 -0
- package/plugins/workflow-runner/src/sandbox.ts +182 -0
- package/plugins/workflow-runner/src/scriptGuard.d.mts +38 -0
- package/plugins/workflow-runner/src/scriptGuard.mjs +449 -0
- package/plugins/workflow-runner/src/taskIdentity.js +151 -307
- package/plugins/workflow-runner/src/taskIdentity.ts +207 -355
- package/plugins/workflow-runner/src/toolRuntime.js +29 -8
- package/plugins/workflow-runner/src/toolRuntime.ts +36 -11
- package/plugins/workflow-runner/src/types.js +7 -211
- package/plugins/workflow-runner/src/types.ts +154 -623
- package/plugins/workflow-runner/src/util.js +47 -0
- package/plugins/workflow-runner/src/util.ts +51 -0
- package/plugins/workflow-runner/src/workflowContext.js +20 -0
- package/plugins/workflow-runner/src/workflowContext.ts +58 -0
- package/plugins/workflow-runner/src/workflowDependencies.js +36 -0
- package/plugins/workflow-runner/src/workflowDependencies.ts +59 -0
- package/plugins/workflow-runner/src/workflowFingerprint.js +118 -0
- package/plugins/workflow-runner/src/workflowFingerprint.ts +122 -0
- package/plugins/workflow-runner/src/workflowState.js +474 -802
- package/plugins/workflow-runner/src/workflowState.ts +575 -970
- package/plugins/workflow-runner/test/api.test.ts +223 -0
- package/plugins/workflow-runner/test/apiCapabilities.test.ts +614 -0
- package/plugins/workflow-runner/test/apiConcurrencyGate.test.ts +104 -0
- package/plugins/workflow-runner/test/apiEcho.test.ts +210 -0
- package/plugins/workflow-runner/test/apiReadText.test.ts +178 -0
- package/plugins/workflow-runner/test/apiResourceLedger.test.ts +184 -0
- package/plugins/workflow-runner/test/childWorkflow.test.ts +304 -0
- package/plugins/workflow-runner/test/discovery.test.ts +234 -0
- package/plugins/workflow-runner/test/goalAnchor.test.ts +106 -0
- package/plugins/workflow-runner/test/goalController.test.ts +317 -0
- package/plugins/workflow-runner/test/journalLanes.test.ts +198 -0
- package/plugins/workflow-runner/test/listResources.test.ts +106 -0
- package/plugins/workflow-runner/test/resourceInputs.test.ts +294 -0
- package/plugins/workflow-runner/test/sandbox.test.ts +126 -0
- package/plugins/workflow-runner/test/scheduler.test.ts +70 -0
- package/plugins/workflow-runner/test/scriptGuard.test.ts +276 -0
- package/plugins/workflow-runner/test/skillDocSync.test.ts +78 -0
- package/plugins/workflow-runner/test/taskIdentity.test.ts +288 -179
- package/plugins/workflow-runner/test/workflowFingerprint.test.ts +106 -0
- package/plugins/workflow-runner/test/workflowState.test.ts +429 -0
- package/skills/pptx/examples/assets/cover-bg.svg +23 -0
- package/skills/pptx/examples/assets/icon-spark.svg +5 -0
- package/skills/ui-design/SKILL.md +5 -2
- package/skills/ui-design/references/layout-composition.md +1 -1
- package/skills/workflow-creator/SKILL.md +344 -0
- package/skills/workflow-creator/assets/template/workflow.mjs +48 -0
- package/skills/workflow-creator/references/api.md +216 -0
- package/skills/workflow-creator/references/migration.md +103 -0
- package/skills/workflow-creator/references/patterns.md +779 -0
- package/skills/workflow-creator/scripts/list-resources.mjs +205 -0
- package/skills/workflow-creator/scripts/scaffold.mjs +148 -0
- package/skills/workflow-creator/scripts/validate.mjs +275 -0
- package/skills/workflow-creator/scripts/workflow-creator.test.mjs +82 -0
- package/src/advisor/advisorCoordinator.js +503 -0
- package/src/advisor/advisorCoordinator.ts +657 -0
- package/src/cli/configMenu.tsx +0 -1
- package/src/cli/print.js +27 -6
- package/src/cli/print.ts +48 -4
- package/src/cli/streamJson.js +5 -6
- package/src/cli/streamJson.ts +8 -4
- package/src/cli/wizards/advisorWizard.js +4 -2
- package/src/cli/wizards/advisorWizard.tsx +8 -2
- package/src/cli/wizards/contextAdvisorWizard.js +5 -4
- package/src/cli/wizards/contextAdvisorWizard.tsx +7 -6
- package/src/collaboration/taskFlow.js +115 -0
- package/src/collaboration/taskFlow.ts +169 -0
- package/src/config/configDefaults.js +41 -1
- package/src/config/configDefaults.ts +43 -1
- package/src/config/configDiagnostics.ts +2 -2
- package/src/config/configFile.js +40 -9
- package/src/config/configFile.ts +60 -11
- package/src/config/configService.js +419 -74
- package/src/config/configService.ts +703 -145
- package/src/config/numericLimits.js +22 -0
- package/src/config/numericLimits.ts +24 -0
- package/src/config.js +1 -0
- package/src/config.ts +3 -2
- package/src/context/budgetPlan.js +1 -1
- package/src/context/budgetPlan.ts +1 -1
- package/src/context/compact.js +7 -2
- package/src/context/compact.ts +9 -2
- package/src/context/microCompactLite.js +212 -13
- package/src/context/microCompactLite.ts +261 -12
- package/src/context/sessionRegistry.js +8 -1
- package/src/context/sessionRegistry.ts +18 -5
- package/src/context/toolResultArchive.js +72 -0
- package/src/context/toolResultArchive.ts +95 -0
- package/src/goal/errors.js +8 -0
- package/src/goal/errors.ts +18 -0
- package/src/goal/goalRuntime.js +344 -0
- package/src/goal/goalRuntime.ts +466 -0
- package/src/goal/goalStore.js +200 -0
- package/src/goal/goalStore.ts +219 -0
- package/src/goal/index.js +4 -0
- package/src/goal/index.ts +4 -0
- package/src/goal/statusMapping.js +71 -0
- package/src/goal/statusMapping.ts +95 -0
- package/src/goal/types.js +7 -0
- package/src/goal/types.ts +179 -0
- package/src/goal/workflowController.js +306 -0
- package/src/goal/workflowController.ts +360 -0
- package/src/harness/evidenceLedger.js +2 -0
- package/src/harness/evidenceLedger.ts +1 -0
- package/src/harness/stallGuard.js +9 -3
- package/src/harness/stallGuard.ts +7 -3
- package/src/images/imageResolver.js +161 -0
- package/src/images/imageResolver.ts +189 -0
- package/src/infra/verification.js +34 -5
- package/src/infra/verification.ts +39 -4
- package/src/llm/client.js +79 -21
- package/src/llm/client.ts +182 -35
- package/src/llm/gatewayClient.js +201 -0
- package/src/llm/gatewayClient.ts +299 -0
- package/src/llm/messageProvenance.js +46 -0
- package/src/llm/messageProvenance.ts +58 -0
- package/src/llm/modelProvider.js +1 -0
- package/src/llm/modelProvider.ts +22 -0
- package/src/llm/providers/anthropicFormat.js +108 -1
- package/src/llm/providers/anthropicFormat.ts +115 -4
- package/src/llm/providers/openaiFormat.js +9 -0
- package/src/llm/providers/openaiFormat.ts +12 -3
- package/src/loop.js +319 -54
- package/src/loop.ts +373 -57
- package/src/main.js +4 -1
- package/src/main.tsx +4 -1
- package/src/managedExecution/index.js +2 -0
- package/src/managedExecution/index.ts +2 -0
- package/src/managedExecution/registry.js +15 -0
- package/src/managedExecution/registry.ts +20 -0
- package/src/managedExecution/types.js +1 -0
- package/src/managedExecution/types.ts +86 -0
- package/src/modelRef.js +9 -0
- package/src/modelRef.ts +31 -0
- package/src/multimodal.js +14 -28
- package/src/multimodal.ts +17 -33
- package/src/net/lanDiscovery.js +65 -0
- package/src/net/lanDiscovery.ts +82 -0
- package/src/plugin-sdk.js +28 -3
- package/src/plugin-sdk.ts +39 -6
- package/src/plugins/pluginApi.ts +16 -1
- package/src/plugins/pluginLoader.js +4 -0
- package/src/plugins/pluginLoader.ts +6 -0
- package/src/plugins/pluginRunner.js +139 -7
- package/src/plugins/pluginRunner.ts +173 -9
- package/src/plugins/slashFallback.js +113 -0
- package/src/plugins/slashFallback.ts +137 -0
- package/src/prompts/policyRegistry.js +12 -32
- package/src/prompts/policyRegistry.ts +15 -32
- package/src/prompts/promptAssembly.js +103 -12
- package/src/prompts/promptAssembly.ts +114 -12
- package/src/prompts/skillList.js +21 -3
- package/src/prompts/skillList.ts +27 -2
- package/src/prompts/stableSystemPrompt.js +18 -0
- package/src/prompts/stableSystemPrompt.ts +33 -0
- package/src/runtime/agentRuntimeContext.js +8 -2
- package/src/runtime/agentRuntimeContext.ts +17 -4
- package/src/session/runAgentTurn.js +78 -12
- package/src/session/runAgentTurn.ts +105 -23
- package/src/session/runTurn.js +10 -9
- package/src/session/runTurn.ts +17 -8
- package/src/session/sessionState.ts +2 -0
- package/src/team/auth.js +186 -0
- package/src/team/auth.ts +298 -0
- package/src/team/cli.js +105 -0
- package/src/team/cli.ts +122 -0
- package/src/team/onlineToolContract.js +64 -0
- package/src/team/onlineToolContract.ts +109 -0
- package/src/team/toolPolicy.js +36 -0
- package/src/team/toolPolicy.ts +75 -0
- package/src/tools/advisor/advisor.js +90 -46
- package/src/tools/advisor/advisor.ts +100 -47
- package/src/tools/advisor/advisorDecision.js +71 -0
- package/src/tools/advisor/advisorDecision.ts +85 -0
- package/src/tools/agent/agent.js +18 -5
- package/src/tools/agent/agent.ts +21 -7
- package/src/tools/agent/agentControl.js +246 -0
- package/src/tools/agent/agentControl.ts +284 -0
- package/src/tools/agent/agentTaskService.js +40 -1
- package/src/tools/agent/agentTaskService.ts +44 -1
- package/src/tools/agent/agentTypes.js +14 -2
- package/src/tools/agent/agentTypes.ts +15 -2
- package/src/tools/applypatch/applyPatch.js +362 -0
- package/src/tools/applypatch/applyPatch.ts +428 -0
- package/src/tools/bash/bash.js +22 -15
- package/src/tools/bash/bash.ts +23 -17
- package/src/tools/capabilities.js +10 -5
- package/src/tools/capabilities.ts +10 -5
- package/src/tools/computeruse/computerIntent.js +32 -0
- package/src/tools/computeruse/computerIntent.ts +46 -0
- package/src/tools/computeruse/telemetryLog.js +12 -0
- package/src/tools/computeruse/telemetryLog.ts +16 -0
- package/src/tools/computeruse/vision/telemetry.js +1 -0
- package/src/tools/computeruse/vision/telemetry.ts +1 -0
- package/src/tools/effectiveToolSet.js +4 -0
- package/src/tools/effectiveToolSet.ts +4 -0
- package/src/tools/index.js +21 -2
- package/src/tools/index.ts +33 -2
- package/src/tools/presentresult/presentresult.js +6 -15
- package/src/tools/presentresult/presentresult.ts +6 -15
- package/src/tools/read/read.js +62 -9
- package/src/tools/read/read.ts +63 -9
- package/src/tools/shared/fileUtils.js +1 -2
- package/src/tools/shared/fileUtils.ts +1 -2
- package/src/tools/shared/toolOutputStore.js +23 -1
- package/src/tools/shared/toolOutputStore.ts +22 -1
- package/src/tools/skill/skill.js +9 -3
- package/src/tools/skill/skill.ts +8 -4
- package/src/tools/toolPromptGuides.js +117 -0
- package/src/tools/toolPromptGuides.ts +137 -0
- package/src/tools/webbrowser/agentBrowserPath.js +19 -9
- package/src/tools/webbrowser/agentBrowserPath.ts +70 -28
- package/src/tools/webbrowser/browser.js +27 -11
- package/src/tools/webbrowser/browser.ts +68 -24
- package/src/tools/webbrowser/browserAutomation.js +329 -0
- package/src/tools/webbrowser/browserAutomation.ts +467 -0
- package/src/tools/webbrowser/browserAxState.js +97 -0
- package/src/tools/webbrowser/browserAxState.ts +142 -0
- package/src/tools/webbrowser/browserBatch.js +1578 -0
- package/src/tools/webbrowser/browserBatch.ts +2382 -0
- package/src/tools/webbrowser/browserCdp.js +208 -0
- package/src/tools/webbrowser/browserCdp.ts +255 -0
- package/src/tools/webbrowser/browserConditions.js +24 -0
- package/src/tools/webbrowser/browserConditions.ts +45 -0
- package/src/tools/webbrowser/browserContent.js +281 -0
- package/src/tools/webbrowser/browserContent.ts +438 -0
- package/src/tools/webbrowser/browserControl.js +131 -34
- package/src/tools/webbrowser/browserControl.ts +210 -56
- package/src/tools/webbrowser/browserDaemon.js +22 -14
- package/src/tools/webbrowser/browserDaemon.ts +54 -21
- package/src/tools/webbrowser/browserFrames.js +47 -0
- package/src/tools/webbrowser/browserFrames.ts +54 -0
- package/src/tools/webbrowser/browserInteractionResolver.js +17 -0
- package/src/tools/webbrowser/browserInteractionResolver.ts +40 -0
- package/src/tools/webbrowser/browserLocator.js +140 -0
- package/src/tools/webbrowser/browserLocator.ts +194 -0
- package/src/tools/webbrowser/browserOutcome.js +10 -17
- package/src/tools/webbrowser/browserOutcome.ts +48 -31
- package/src/tools/webbrowser/browserRecovery.js +58 -0
- package/src/tools/webbrowser/browserRecovery.ts +83 -0
- package/src/tools/webbrowser/browserRunRegistry.js +35 -3
- package/src/tools/webbrowser/browserRunRegistry.ts +91 -15
- package/src/tools/webbrowser/browserService.js +28 -2
- package/src/tools/webbrowser/browserService.ts +49 -9
- package/src/tools/webbrowser/browserStream.js +132 -33
- package/src/tools/webbrowser/browserStream.ts +224 -50
- package/src/tools/webbrowser/browserTabs.js +48 -15
- package/src/tools/webbrowser/browserTabs.ts +178 -96
- package/src/tools/webbrowser/webbrowser.js +679 -144
- package/src/tools/webbrowser/webbrowser.ts +975 -236
- package/src/tools/webfetch/preapproved.js +0 -2
- package/src/tools/webfetch/preapproved.ts +0 -2
- package/src/tools/websearch/websearch.js +129 -8
- package/src/tools/websearch/websearch.ts +138 -6
- package/src/types.ts +440 -38
- package/src/ui/App.js +377 -58
- package/src/ui/App.tsx +576 -111
- package/src/ui/LiveArea.js +14 -8
- package/src/ui/LiveArea.tsx +14 -10
- package/src/ui/MessageList.js +63 -20
- package/src/ui/MessageList.tsx +101 -16
- package/src/ui/ModelPicker.js +7 -4
- package/src/ui/ModelPicker.tsx +55 -8
- package/src/ui/hooks/useChat.js +152 -163
- package/src/ui/hooks/useChat.ts +396 -334
- package/src/ui/loginCommand.js +56 -0
- package/src/ui/loginCommand.ts +63 -0
- package/src/ui/toolProgress.js +41 -0
- package/src/ui/toolProgress.ts +44 -0
- package/src/web/browserBridge.js +20 -2
- package/src/web/browserBridge.ts +20 -2
- package/src/web/protocol.ts +1897 -194
- package/workflows/adaptive-research/workflow.mjs +86 -0
- package/workflows/e2e-capabilities/workflow.mjs +102 -0
- package/workflows/e2e-llm/workflow.mjs +52 -0
- package/workflows/e2e-smoke/workflow.mjs +59 -0
- package/workflows/e2e-vision-schema/README.md +48 -0
- package/workflows/e2e-vision-schema/workflow.mjs +149 -0
- package/workflows/e2e-write-greet/workflow.mjs +37 -0
- package/workflows/file-census/README.md +14 -0
- package/workflows/file-census/scripts/.gitkeep +2 -0
- package/workflows/file-census/workflow.mjs +110 -0
- package/workflows/goal/README.md +166 -0
- package/workflows/goal/prompts/audit.md +57 -0
- package/workflows/goal/prompts/contract.md +56 -0
- package/workflows/goal/prompts/iterate.md +37 -0
- package/workflows/goal/workflow.mjs +501 -0
- package/workflows/illustrate/README.md +90 -0
- package/workflows/illustrate/assets/engines.json +20 -0
- package/workflows/illustrate/assets/extract-baidu.js +21 -0
- package/workflows/illustrate/assets/extract-bing.js +33 -0
- package/workflows/illustrate/assets/extract-google.js +31 -0
- package/workflows/illustrate/workflow.mjs +301 -0
- package/workflows/plan/workflow.mjs +389 -0
- package/workflows/question-crops/README.md +35 -0
- package/workflows/question-crops/prompts/inventory.md +23 -0
- package/workflows/question-crops/scripts/image-ops.mjs +305 -0
- package/workflows/question-crops/scripts/image-ops.test.mjs +95 -0
- package/workflows/question-crops/workflow.mjs +172 -0
- package/workflows/ralph-loop/workflow.mjs +162 -0
- package/workflows/research/README.md +74 -0
- package/workflows/research/prompts/s0-goal.md +43 -0
- package/workflows/research/prompts/s1-early-exit.md +22 -0
- package/workflows/research/prompts/s1-map.md +58 -0
- package/workflows/research/prompts/s2-options.md +53 -0
- package/workflows/research/prompts/s3-deep-dive.md +56 -0
- package/workflows/research/prompts/s4-verify.md +74 -0
- package/workflows/research/prompts/s5-synthesis.md +81 -0
- package/workflows/research/prompts/s5b-fill-gaps.md +32 -0
- package/workflows/research/prompts/s6-package.md +32 -0
- package/workflows/research/scripts/build-report.mjs +98 -0
- package/workflows/research/workflow.mjs +288 -0
- package/workflows/wechat-reading-shots/prompts/collect-params.md +16 -0
- package/workflows/wechat-reading-shots/prompts/evaluate.md +20 -0
- package/workflows/wechat-reading-shots/prompts/open-book.md +28 -0
- package/workflows/wechat-reading-shots/prompts/read-page.md +13 -0
- package/workflows/wechat-reading-shots/prompts/turn-batch.md +24 -0
- package/workflows/wechat-reading-shots/test/control-flow.test.ts +329 -0
- package/workflows/wechat-reading-shots/workflow.mjs +266 -0
- package/workflows/wrong-question-docx/README.md +55 -0
- package/workflows/wrong-question-docx/prompts/figure-qa.md +8 -0
- package/workflows/wrong-question-docx/prompts/transcribe.md +25 -0
- package/workflows/wrong-question-docx/scripts/workbook.mjs +857 -0
- package/workflows/wrong-question-docx/test/control-flow.test.ts +165 -0
- package/workflows/wrong-question-docx/test/workbook.test.mjs +303 -0
- package/workflows/wrong-question-docx/workflow.mjs +485 -0
- package/plugins/workflow-runner/src/agentCommon.js +0 -354
- package/plugins/workflow-runner/src/agentCommon.ts +0 -435
- package/plugins/workflow-runner/src/controlFlow/branch.js +0 -40
- package/plugins/workflow-runner/src/controlFlow/branch.ts +0 -46
- package/plugins/workflow-runner/src/controlFlow/loop.js +0 -211
- package/plugins/workflow-runner/src/controlFlow/loop.ts +0 -248
- package/plugins/workflow-runner/src/controlFlow/pause.js +0 -129
- package/plugins/workflow-runner/src/controlFlow/pause.ts +0 -141
- package/plugins/workflow-runner/src/controlFlow/signal.js +0 -21
- package/plugins/workflow-runner/src/controlFlow/signal.ts +0 -29
- package/plugins/workflow-runner/src/controlFlow/types.js +0 -18
- package/plugins/workflow-runner/src/controlFlow/types.ts +0 -63
- package/plugins/workflow-runner/src/dslManifest.js +0 -72
- package/plugins/workflow-runner/src/dslManifest.ts +0 -127
- package/plugins/workflow-runner/src/expressions.js +0 -639
- package/plugins/workflow-runner/src/expressions.ts +0 -640
- package/plugins/workflow-runner/src/loader.js +0 -573
- package/plugins/workflow-runner/src/loader.ts +0 -721
- package/plugins/workflow-runner/src/multimodal.js +0 -38
- package/plugins/workflow-runner/src/multimodal.ts +0 -64
- package/plugins/workflow-runner/src/recovery.js +0 -126
- package/plugins/workflow-runner/src/recovery.ts +0 -156
- package/plugins/workflow-runner/src/stepExecutors/agent.js +0 -173
- package/plugins/workflow-runner/src/stepExecutors/agent.ts +0 -234
- package/plugins/workflow-runner/src/stepExecutors/assert.js +0 -42
- package/plugins/workflow-runner/src/stepExecutors/assert.ts +0 -52
- package/plugins/workflow-runner/src/stepExecutors/echo.js +0 -31
- package/plugins/workflow-runner/src/stepExecutors/echo.ts +0 -38
- package/plugins/workflow-runner/src/stepExecutors/llm.js +0 -99
- package/plugins/workflow-runner/src/stepExecutors/llm.ts +0 -133
- package/plugins/workflow-runner/src/stepExecutors/set.js +0 -49
- package/plugins/workflow-runner/src/stepExecutors/set.ts +0 -56
- package/plugins/workflow-runner/src/stepExecutors/skill.js +0 -111
- package/plugins/workflow-runner/src/stepExecutors/skill.ts +0 -140
- package/plugins/workflow-runner/src/stepExecutors/tool.js +0 -227
- package/plugins/workflow-runner/src/stepExecutors/tool.ts +0 -274
- package/plugins/workflow-runner/src/stepExecutors/verify.js +0 -61
- package/plugins/workflow-runner/src/stepExecutors/verify.ts +0 -72
- package/plugins/workflow-runner/src/stepPipeline.js +0 -277
- package/plugins/workflow-runner/src/stepPipeline.ts +0 -366
- package/plugins/workflow-runner/test/agent.test.ts +0 -587
- package/plugins/workflow-runner/test/agentLlmError.test.ts +0 -373
- package/plugins/workflow-runner/test/bashBlacklist.sweep.test.ts +0 -65
- package/plugins/workflow-runner/test/captureMisbind.test.ts +0 -99
- package/plugins/workflow-runner/test/cli.e2e.test.ts +0 -167
- package/plugins/workflow-runner/test/context-files.test.ts +0 -320
- package/plugins/workflow-runner/test/e2e.test.ts +0 -370
- package/plugins/workflow-runner/test/e2eNewLayout.test.ts +0 -314
- package/plugins/workflow-runner/test/engineRobustness.test.ts +0 -234
- package/plugins/workflow-runner/test/expressions.test.ts +0 -439
- package/plugins/workflow-runner/test/foundation.test.ts +0 -126
- package/plugins/workflow-runner/test/graceful.test.ts +0 -139
- package/plugins/workflow-runner/test/greedyInputs.load.test.ts +0 -47
- package/plugins/workflow-runner/test/hostCapability.test.ts +0 -108
- package/plugins/workflow-runner/test/illustrate.load.test.ts +0 -183
- package/plugins/workflow-runner/test/jsonPayload.test.ts +0 -83
- package/plugins/workflow-runner/test/loader.test.ts +0 -617
- package/plugins/workflow-runner/test/maxTurnsRetryDelay.test.ts +0 -33
- package/plugins/workflow-runner/test/multimodal.test.ts +0 -342
- package/plugins/workflow-runner/test/output-schema.test.ts +0 -213
- package/plugins/workflow-runner/test/parseWorkflowArgs.test.ts +0 -106
- package/plugins/workflow-runner/test/pluginRunner.isolation.test.ts +0 -269
- package/plugins/workflow-runner/test/ppt.load.test.ts +0 -1220
- package/plugins/workflow-runner/test/ralphLoop.e2e.test.ts +0 -221
- package/plugins/workflow-runner/test/recovery.test.ts +0 -265
- package/plugins/workflow-runner/test/research.load.test.ts +0 -328
- package/plugins/workflow-runner/test/runner.test.ts +0 -911
- package/plugins/workflow-runner/test/runnerPreflight.test.ts +0 -436
- package/plugins/workflow-runner/test/skillTurnDone.test.ts +0 -71
- package/plugins/workflow-runner/test/stateLifecycle.test.ts +0 -198
- package/plugins/workflow-runner/test/stepPipeline.test.ts +0 -114
- package/plugins/workflow-runner/test/taskArbitration.test.ts +0 -354
- package/plugins/workflow-runner/test/taskLifecycle.test.ts +0 -324
- package/plugins/workflow-runner/test/toolArtifactOverwrite.test.ts +0 -221
- package/plugins/workflow-runner/test/toolCapabilityBoundary.test.ts +0 -29
- package/plugins/workflow-runner/test/toolRuntimeIntegration.test.ts +0 -168
- package/plugins/workflow-runner/test/webppt.load.test.ts +0 -250
- package/plugins/workflow-runner/test/word.load.test.ts +0 -208
- package/plugins/workflow-runner/test/workflows.smoke.test.ts +0 -76
- package/plugins/workflow-runner/test/yamlGuardConditions.test.ts +0 -207
- package/src/advisorState.js +0 -22
- package/src/cli/configWizard.js +0 -293
- package/src/tools/toolProgressBridge.js +0 -48
- package/workflows/adaptive-research.yaml +0 -50
- package/workflows/book-review-short.yaml +0 -125
- package/workflows/e2e-write-greet.yaml +0 -31
- package/workflows/illustrate.yaml +0 -227
- package/workflows/plan.yaml +0 -470
- package/workflows/ppt.yaml +0 -1670
- package/workflows/ralph-loop.yaml +0 -201
- package/workflows/research.yaml +0 -873
- package/workflows/schema.json +0 -176
- package/workflows/webppt.yaml +0 -496
- package/workflows/wechat-reading-shots.yaml +0 -315
- package/workflows/word.yaml +0 -720
- package/workflows/youtube-shorts.yaml +0 -202
- /package/workflows/{research.md → research/DESIGN.md} +0 -0
|
@@ -25,15 +25,24 @@
|
|
|
25
25
|
import { z } from 'zod';
|
|
26
26
|
|
|
27
27
|
import { randomUUID } from 'node:crypto';
|
|
28
|
-
import { realpath } from 'node:fs/promises';
|
|
29
|
-
import { isAbsolute, relative, resolve } from 'node:path';
|
|
28
|
+
import { mkdir, realpath } from 'node:fs/promises';
|
|
29
|
+
import { isAbsolute, join, relative, resolve } from 'node:path';
|
|
30
30
|
|
|
31
|
-
import
|
|
31
|
+
import { getWorkingDir } from '../../bootstrap/workingDir.js';
|
|
32
32
|
import { configService } from '../../config/configService.js';
|
|
33
|
-
import {
|
|
33
|
+
import type {
|
|
34
|
+
ImageInputSpec,
|
|
35
|
+
Tool,
|
|
36
|
+
ToolCallContext,
|
|
37
|
+
ToolResult,
|
|
38
|
+
} from '../../types.js';
|
|
34
39
|
import { toToolParameters } from '../../utils/zodToJson.js';
|
|
40
|
+
import { DEFAULT_MAX_RESULT_SIZE_CHARS } from '../types.js';
|
|
35
41
|
import { resolveAgentBrowserPath } from './agentBrowserPath.js';
|
|
36
42
|
import {
|
|
43
|
+
type AgentBrowserRun,
|
|
44
|
+
type AgentBrowserRunner,
|
|
45
|
+
BROWSER_SESSION,
|
|
37
46
|
browserArtifactDir,
|
|
38
47
|
createArtifactPath,
|
|
39
48
|
isAllowedBrowserScheme,
|
|
@@ -41,31 +50,45 @@ import {
|
|
|
41
50
|
sessionArgs,
|
|
42
51
|
spawnRunner,
|
|
43
52
|
validateArtifactOutputPath,
|
|
44
|
-
BROWSER_SESSION,
|
|
45
|
-
type AgentBrowserRun,
|
|
46
|
-
type AgentBrowserRunner,
|
|
47
53
|
} from './browser.js';
|
|
48
|
-
import {
|
|
49
|
-
import { ensureDaemon, stopDaemon, getStreamPort } from './browserDaemon.js';
|
|
50
|
-
import { getBrowserStream, type BrowserStream } from './browserStream.js';
|
|
51
|
-
import { browserService } from './browserService.js';
|
|
52
|
-
import { detectBrowserChallenge } from './browserControl.js';
|
|
54
|
+
import { browserDownloadDir } from './browserAutomation.js';
|
|
53
55
|
import {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
56
|
+
type BrowserPageAsset,
|
|
57
|
+
CLIPBOARD_READ_EXPRESSION,
|
|
58
|
+
CLIPBOARD_READ_TEXT_EXPRESSION,
|
|
59
|
+
PAGE_ASSETS_EXPRESSION,
|
|
60
|
+
browserAssetFetchExpression,
|
|
61
|
+
bundlePageAssets,
|
|
62
|
+
clipboardWriteExpression,
|
|
63
|
+
clipboardWriteTextExpression,
|
|
64
|
+
pageContentExpression,
|
|
65
|
+
} from './browserContent.js';
|
|
66
|
+
import { detectBrowserChallenge } from './browserControl.js';
|
|
67
|
+
import { ensureDaemon, getStreamPort, stopDaemon } from './browserDaemon.js';
|
|
68
|
+
import { FRAME_INFO_EXPRESSION } from './browserFrames.js';
|
|
57
69
|
import {
|
|
70
|
+
BROWSER_FAILURE_LIMIT,
|
|
71
|
+
VERIFIED_BROWSER_ACTIONS,
|
|
58
72
|
browserAttempts,
|
|
59
73
|
budgetSnapshot,
|
|
60
74
|
classifyPageEffect,
|
|
61
75
|
describeEffect,
|
|
62
76
|
differingLineIndexes,
|
|
63
77
|
suggestSelectorCandidates,
|
|
64
|
-
BROWSER_FAILURE_LIMIT,
|
|
65
|
-
VERIFIED_BROWSER_ACTIONS,
|
|
66
78
|
} from './browserOutcome.js';
|
|
79
|
+
import {
|
|
80
|
+
recoverableAgentBrowserRunFailure,
|
|
81
|
+
withBrowserRecovery,
|
|
82
|
+
} from './browserRecovery.js';
|
|
83
|
+
import {
|
|
84
|
+
browserRunRegistry,
|
|
85
|
+
isLikelyIrreversibleBrowserTarget,
|
|
86
|
+
} from './browserRunRegistry.js';
|
|
87
|
+
import { browserService } from './browserService.js';
|
|
88
|
+
import { type BrowserStream, getBrowserStream } from './browserStream.js';
|
|
67
89
|
import {
|
|
68
90
|
BrowserTabError,
|
|
91
|
+
claimConversationTabById,
|
|
69
92
|
createConversationTab,
|
|
70
93
|
ensureConversationTab,
|
|
71
94
|
listBrowserTabs,
|
|
@@ -83,7 +106,14 @@ const CHALLENGE_PROBE_ACTIONS = new Set([
|
|
|
83
106
|
'dialogAccept',
|
|
84
107
|
]);
|
|
85
108
|
// C4:驱动可见 AI 指针的交互动作(有明确元素目标)。navigate/press/dialogAccept 无元素目标不列入。
|
|
86
|
-
const POINTER_ACTIONS = new Set([
|
|
109
|
+
const POINTER_ACTIONS = new Set([
|
|
110
|
+
'click',
|
|
111
|
+
'fill',
|
|
112
|
+
'type',
|
|
113
|
+
'hover',
|
|
114
|
+
'select',
|
|
115
|
+
'drag',
|
|
116
|
+
]);
|
|
87
117
|
// 关键节点截图核验:这些是「页面到达/读取评估」的确定性检查点,成功后自动截当前可见画面喂视觉模型自我核验,
|
|
88
118
|
// 根治「AI 以为成功了、实际页面不是那回事」。不是每个动作都截(点击/输入等中间步不截),故不会「一直截图」。
|
|
89
119
|
const KEY_NODE_SHOT_ACTIONS = new Set(['navigate', 'snapshot']);
|
|
@@ -161,7 +191,12 @@ async function viewportPoint(
|
|
|
161
191
|
} catch {
|
|
162
192
|
return null;
|
|
163
193
|
}
|
|
164
|
-
if (
|
|
194
|
+
if (
|
|
195
|
+
!Number.isFinite(width) ||
|
|
196
|
+
!Number.isFinite(height) ||
|
|
197
|
+
width <= 0 ||
|
|
198
|
+
height <= 0
|
|
199
|
+
) {
|
|
165
200
|
return null;
|
|
166
201
|
}
|
|
167
202
|
// 夹在视口内:越界坐标在 CDP 侧不报错,只会点到边缘,症状是"点了但没反应"。
|
|
@@ -192,11 +227,16 @@ async function runPointerAt(
|
|
|
192
227
|
code: 1,
|
|
193
228
|
stdout: '',
|
|
194
229
|
stderr:
|
|
195
|
-
'无法取得视口尺寸,坐标点击已放弃 —— 与其按一个猜出来的尺寸点在错误位置,'
|
|
196
|
-
|
|
230
|
+
'无法取得视口尺寸,坐标点击已放弃 —— 与其按一个猜出来的尺寸点在错误位置,' +
|
|
231
|
+
'不如不点。请确认页面已加载(先 navigate 或 snapshot),或改用选择器定位。',
|
|
197
232
|
};
|
|
198
233
|
}
|
|
199
|
-
const moved = await run([
|
|
234
|
+
const moved = await run([
|
|
235
|
+
'mouse',
|
|
236
|
+
'move',
|
|
237
|
+
String(point.px),
|
|
238
|
+
String(point.py),
|
|
239
|
+
]);
|
|
200
240
|
if (moved.code !== 0) return moved;
|
|
201
241
|
if (kind === 'hover') {
|
|
202
242
|
return { ...moved, stdout: `hovered at (${point.px}, ${point.py})` };
|
|
@@ -205,7 +245,10 @@ async function runPointerAt(
|
|
|
205
245
|
if (down.code !== 0) return down;
|
|
206
246
|
const up = await run(['mouse', 'up']);
|
|
207
247
|
if (up.code !== 0) return up;
|
|
208
|
-
return {
|
|
248
|
+
return {
|
|
249
|
+
...up,
|
|
250
|
+
stdout: `clicked at (${point.px}, ${point.py}) [视口比例 ${input.x}, ${input.y}]`,
|
|
251
|
+
};
|
|
209
252
|
}
|
|
210
253
|
|
|
211
254
|
/**
|
|
@@ -233,9 +276,16 @@ async function resolveLiveUrl(
|
|
|
233
276
|
}
|
|
234
277
|
}
|
|
235
278
|
|
|
236
|
-
function requireAllowedDeveloperOrigin(
|
|
279
|
+
function requireAllowedDeveloperOrigin(
|
|
280
|
+
url: string,
|
|
281
|
+
allowedOrigins: string[],
|
|
282
|
+
): string | null {
|
|
237
283
|
let origin: string;
|
|
238
|
-
try {
|
|
284
|
+
try {
|
|
285
|
+
origin = new URL(url).origin;
|
|
286
|
+
} catch {
|
|
287
|
+
return '当前页面没有可授权的 HTTP(S) 站点';
|
|
288
|
+
}
|
|
239
289
|
if (!/^https?:\/\//i.test(origin) || !allowedOrigins.includes(origin)) {
|
|
240
290
|
return `Developer Mode 尚未授权当前站点:${origin}。请在“设置 → 浏览器”添加该站点。`;
|
|
241
291
|
}
|
|
@@ -252,7 +302,16 @@ const inputSchema = z.object({
|
|
|
252
302
|
'forward',
|
|
253
303
|
'reload',
|
|
254
304
|
'snapshot',
|
|
305
|
+
'frameInfo',
|
|
255
306
|
'getContent',
|
|
307
|
+
'pageAssets',
|
|
308
|
+
'listPageAssets',
|
|
309
|
+
'bundlePageAssets',
|
|
310
|
+
'exportContent',
|
|
311
|
+
'clipboardReadText',
|
|
312
|
+
'clipboardWriteText',
|
|
313
|
+
'clipboardRead',
|
|
314
|
+
'clipboardWrite',
|
|
256
315
|
'screenshot',
|
|
257
316
|
'click',
|
|
258
317
|
'fill',
|
|
@@ -296,13 +355,27 @@ const inputSchema = z.object({
|
|
|
296
355
|
'traceStop',
|
|
297
356
|
'reactTree',
|
|
298
357
|
'reactInspect',
|
|
358
|
+
'download',
|
|
359
|
+
'doubleClick',
|
|
360
|
+
'check',
|
|
361
|
+
'uncheck',
|
|
362
|
+
'setChecked',
|
|
363
|
+
'getState',
|
|
364
|
+
'waitForURL',
|
|
365
|
+
'waitForDownload',
|
|
366
|
+
'waitForFileChooser',
|
|
299
367
|
])
|
|
300
368
|
.describe('Browser action to perform'),
|
|
301
|
-
url: z
|
|
369
|
+
url: z
|
|
370
|
+
.string()
|
|
371
|
+
.optional()
|
|
372
|
+
.describe('URL to navigate to (for navigate/tabNew)'),
|
|
302
373
|
selector: z
|
|
303
374
|
.string()
|
|
304
375
|
.optional()
|
|
305
|
-
.describe(
|
|
376
|
+
.describe(
|
|
377
|
+
'Element selector: CSS selector, @eN ref from snapshot, or data-ab-pick-ref from pick',
|
|
378
|
+
),
|
|
306
379
|
/**
|
|
307
380
|
* 视口比例坐标(0–1000),`click` / `hover` 在**没有可用选择器**时的兜底定位。
|
|
308
381
|
*
|
|
@@ -314,19 +387,74 @@ const inputSchema = z.object({
|
|
|
314
387
|
*
|
|
315
388
|
* 与 ComputerUse 的坐标系**刻意一致**(同为 0–1000),模型不必为两个工具切换心智。
|
|
316
389
|
*/
|
|
317
|
-
x: z
|
|
318
|
-
.
|
|
319
|
-
|
|
320
|
-
.
|
|
390
|
+
x: z
|
|
391
|
+
.number()
|
|
392
|
+
.min(0)
|
|
393
|
+
.max(1000)
|
|
394
|
+
.optional()
|
|
395
|
+
.describe(
|
|
396
|
+
'视口比例横坐标 0-1000(左=0,右=1000)。仅在没有可用选择器时用;必须与 y 同时提供',
|
|
397
|
+
),
|
|
398
|
+
y: z
|
|
399
|
+
.number()
|
|
400
|
+
.min(0)
|
|
401
|
+
.max(1000)
|
|
402
|
+
.optional()
|
|
403
|
+
.describe(
|
|
404
|
+
'视口比例纵坐标 0-1000(上=0,下=1000)。仅在没有可用选择器时用;必须与 x 同时提供',
|
|
405
|
+
),
|
|
321
406
|
value: z.string().optional().describe('Text value for fill/type/select'),
|
|
407
|
+
clipboardText: z
|
|
408
|
+
.string()
|
|
409
|
+
.optional()
|
|
410
|
+
.describe('Text to write to the browser profile clipboard'),
|
|
411
|
+
clipboardItems: z
|
|
412
|
+
.array(z.object({ type: z.string().min(1), base64: z.string() }))
|
|
413
|
+
.optional()
|
|
414
|
+
.describe('Binary clipboard items for clipboardWrite, encoded as base64'),
|
|
415
|
+
exportFormat: z
|
|
416
|
+
.enum([
|
|
417
|
+
'text',
|
|
418
|
+
'txt',
|
|
419
|
+
'markdown',
|
|
420
|
+
'md',
|
|
421
|
+
'html',
|
|
422
|
+
'json',
|
|
423
|
+
'page',
|
|
424
|
+
'google-workspace',
|
|
425
|
+
'youtube-transcript',
|
|
426
|
+
'pdf',
|
|
427
|
+
'docx',
|
|
428
|
+
'xlsx',
|
|
429
|
+
'csv',
|
|
430
|
+
'pptx',
|
|
431
|
+
])
|
|
432
|
+
.optional()
|
|
433
|
+
.describe('Format for exportContent'),
|
|
434
|
+
bundleDir: z
|
|
435
|
+
.string()
|
|
436
|
+
.optional()
|
|
437
|
+
.describe(
|
|
438
|
+
'Directory under the workspace or browser artifact root for bundlePageAssets',
|
|
439
|
+
),
|
|
322
440
|
values: z.array(z.string()).optional().describe('Values for multi-select'),
|
|
323
|
-
key: z
|
|
441
|
+
key: z
|
|
442
|
+
.string()
|
|
443
|
+
.optional()
|
|
444
|
+
.describe(
|
|
445
|
+
'Key to press (for press): Enter, Tab, Escape, ArrowUp/Down/Left/Right, etc.',
|
|
446
|
+
),
|
|
324
447
|
js: z.string().optional().describe('JavaScript to run (for eval)'),
|
|
325
448
|
path: z
|
|
326
449
|
.string()
|
|
327
450
|
.optional()
|
|
328
|
-
.describe(
|
|
329
|
-
|
|
451
|
+
.describe(
|
|
452
|
+
'Output file path for screenshot/pdf/recording (optional; defaults to temp file)',
|
|
453
|
+
),
|
|
454
|
+
files: z
|
|
455
|
+
.array(z.string())
|
|
456
|
+
.optional()
|
|
457
|
+
.describe('File paths to upload (for upload)'),
|
|
330
458
|
annotate: z
|
|
331
459
|
.boolean()
|
|
332
460
|
.optional()
|
|
@@ -339,18 +467,48 @@ const inputSchema = z.object({
|
|
|
339
467
|
.enum(['up', 'down', 'top', 'bottom'])
|
|
340
468
|
.optional()
|
|
341
469
|
.describe('scroll direction'),
|
|
342
|
-
scrollX: z
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
470
|
+
scrollX: z
|
|
471
|
+
.number()
|
|
472
|
+
.optional()
|
|
473
|
+
.describe('scroll pixels horizontally (positive=right, negative=left)'),
|
|
474
|
+
scrollY: z
|
|
475
|
+
.number()
|
|
476
|
+
.optional()
|
|
477
|
+
.describe('scroll pixels vertically (positive=down, negative=up)'),
|
|
478
|
+
waitMs: z
|
|
479
|
+
.number()
|
|
480
|
+
.int()
|
|
481
|
+
.positive()
|
|
482
|
+
.optional()
|
|
483
|
+
.describe('Wait time in milliseconds'),
|
|
484
|
+
waitSelector: z
|
|
485
|
+
.string()
|
|
486
|
+
.optional()
|
|
487
|
+
.describe('Wait until this selector appears'),
|
|
488
|
+
waitText: z
|
|
489
|
+
.string()
|
|
490
|
+
.optional()
|
|
491
|
+
.describe('Wait until this text appears on page'),
|
|
347
492
|
waitLoad: z
|
|
348
493
|
.enum(['load', 'domcontentloaded', 'networkidle'])
|
|
349
494
|
.optional()
|
|
350
495
|
.describe(
|
|
351
496
|
"Wait until the page reaches a load state after navigation. 'load' / 'domcontentloaded' settle quickly; 'networkidle' can wait up to ~25s on pages with long-lived connections.",
|
|
352
497
|
),
|
|
498
|
+
waitUrl: z
|
|
499
|
+
.string()
|
|
500
|
+
.optional()
|
|
501
|
+
.describe('Wait until the current URL matches this pattern'),
|
|
353
502
|
tabId: z.string().optional().describe('Tab ID for tab operations'),
|
|
503
|
+
retention: z
|
|
504
|
+
.enum(['ephemeral', 'conversation', 'handoff', 'deliverable'])
|
|
505
|
+
.optional()
|
|
506
|
+
.describe('Retention policy for the claimed tab'),
|
|
507
|
+
tabTitle: z
|
|
508
|
+
.string()
|
|
509
|
+
.max(200)
|
|
510
|
+
.optional()
|
|
511
|
+
.describe('Human-readable title for a retained tab'),
|
|
354
512
|
cookies: z
|
|
355
513
|
.array(
|
|
356
514
|
z.object({
|
|
@@ -362,19 +520,53 @@ const inputSchema = z.object({
|
|
|
362
520
|
)
|
|
363
521
|
.optional()
|
|
364
522
|
.describe('Cookies to set (for cookiesSet)'),
|
|
365
|
-
viewportWidth: z
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
523
|
+
viewportWidth: z
|
|
524
|
+
.number()
|
|
525
|
+
.int()
|
|
526
|
+
.positive()
|
|
527
|
+
.optional()
|
|
528
|
+
.describe('Viewport width'),
|
|
529
|
+
viewportHeight: z
|
|
530
|
+
.number()
|
|
531
|
+
.int()
|
|
532
|
+
.positive()
|
|
533
|
+
.optional()
|
|
534
|
+
.describe('Viewport height'),
|
|
535
|
+
devicePreset: z
|
|
536
|
+
.string()
|
|
537
|
+
.optional()
|
|
538
|
+
.describe('Device preset name (iPhone 15, iPad, Pixel 9, etc.)'),
|
|
539
|
+
colorScheme: z
|
|
540
|
+
.enum(['light', 'dark', 'no-preference'])
|
|
541
|
+
.optional()
|
|
542
|
+
.describe('Color scheme preference'),
|
|
369
543
|
offline: z.boolean().optional().describe('Enable offline mode'),
|
|
370
|
-
mode: z
|
|
371
|
-
|
|
544
|
+
mode: z
|
|
545
|
+
.enum(['browse', 'assist', 'auto'])
|
|
546
|
+
.optional()
|
|
547
|
+
.describe(
|
|
548
|
+
'Deprecated compatibility field; supervised Desktop control is automatic',
|
|
549
|
+
),
|
|
550
|
+
dialogText: z
|
|
551
|
+
.string()
|
|
552
|
+
.optional()
|
|
553
|
+
.describe('Text to enter in prompt dialog (for dialogAccept)'),
|
|
372
554
|
pickX: z.number().optional().describe('X coordinate for pickElement'),
|
|
373
555
|
pickY: z.number().optional().describe('Y coordinate for pickElement'),
|
|
374
556
|
markLabel: z.string().optional().describe('Label for marked element'),
|
|
375
|
-
markColor: z
|
|
557
|
+
markColor: z
|
|
558
|
+
.string()
|
|
559
|
+
.optional()
|
|
560
|
+
.describe('Color for marked element highlight'),
|
|
376
561
|
dragTo: z.string().optional().describe('Target selector for drag action'),
|
|
377
|
-
|
|
562
|
+
checked: z
|
|
563
|
+
.boolean()
|
|
564
|
+
.optional()
|
|
565
|
+
.describe('Desired checkbox state for setChecked'),
|
|
566
|
+
reactId: z
|
|
567
|
+
.string()
|
|
568
|
+
.optional()
|
|
569
|
+
.describe('React fiber id returned by reactTree'),
|
|
378
570
|
timeout: z
|
|
379
571
|
.number()
|
|
380
572
|
.int()
|
|
@@ -388,20 +580,53 @@ const inputSchema = z.object({
|
|
|
388
580
|
// 探测 → 元素盒子 → 前置快照 → 动作 → 稳定重探 → 可选噪声基线」,这个预算盖住
|
|
389
581
|
// 全部。描述必须说清楚,否则调用方按「单个动作」估的值会把前后探测一起掐掉。
|
|
390
582
|
.describe(
|
|
391
|
-
'Timeout in milliseconds for the whole action sequence — including the pre-action probe, '
|
|
392
|
-
|
|
583
|
+
'Timeout in milliseconds for the whole action sequence — including the pre-action probe, ' +
|
|
584
|
+
'the action itself, and the post-action settle/verify probes (default: 120000, max: 300000)',
|
|
393
585
|
),
|
|
394
586
|
intent: z
|
|
395
587
|
.enum(['normal', 'submit'])
|
|
396
588
|
.optional()
|
|
397
|
-
.describe(
|
|
589
|
+
.describe(
|
|
590
|
+
'Use submit for the final irreversible action, such as publish, purchase, delete, or form submission',
|
|
591
|
+
),
|
|
398
592
|
});
|
|
399
593
|
type WebBrowserInput = z.infer<typeof inputSchema>;
|
|
400
594
|
|
|
401
595
|
const OBSERVE_ACTIONS = new Set<WebBrowserInput['action']>([
|
|
402
|
-
'snapshot',
|
|
403
|
-
'
|
|
404
|
-
'
|
|
596
|
+
'snapshot',
|
|
597
|
+
'frameInfo',
|
|
598
|
+
'getContent',
|
|
599
|
+
'pageAssets',
|
|
600
|
+
'listPageAssets',
|
|
601
|
+
'bundlePageAssets',
|
|
602
|
+
'exportContent',
|
|
603
|
+
'clipboardReadText',
|
|
604
|
+
'clipboardRead',
|
|
605
|
+
'screenshot',
|
|
606
|
+
'tabList',
|
|
607
|
+
'streamInfo',
|
|
608
|
+
'console',
|
|
609
|
+
'errors',
|
|
610
|
+
'networkRequests',
|
|
611
|
+
'getStyles',
|
|
612
|
+
'vitals',
|
|
613
|
+
'reactTree',
|
|
614
|
+
'reactInspect',
|
|
615
|
+
]);
|
|
616
|
+
|
|
617
|
+
// These capabilities use fixed internal expressions. They are not an escape
|
|
618
|
+
// hatch for arbitrary JavaScript; caller supplied code remains the `eval`
|
|
619
|
+
// action and keeps its Developer Mode gate.
|
|
620
|
+
const INTERNAL_SCRIPT_ACTIONS = new Set<WebBrowserInput['action']>([
|
|
621
|
+
'pageAssets',
|
|
622
|
+
'exportContent',
|
|
623
|
+
'clipboardReadText',
|
|
624
|
+
'clipboardWriteText',
|
|
625
|
+
'clipboardRead',
|
|
626
|
+
'clipboardWrite',
|
|
627
|
+
'frameInfo',
|
|
628
|
+
'listPageAssets',
|
|
629
|
+
'bundlePageAssets',
|
|
405
630
|
]);
|
|
406
631
|
|
|
407
632
|
const parameters = toToolParameters(inputSchema);
|
|
@@ -444,7 +669,12 @@ Navigation:
|
|
|
444
669
|
|
|
445
670
|
Page Reading:
|
|
446
671
|
snapshot - Accessibility tree with @eN refs (LLM gold standard for seeing page).
|
|
672
|
+
frameInfo - List the current page's iframe/frame handles and URLs.
|
|
447
673
|
getContent [selector] - Plain text of page or element (default body).
|
|
674
|
+
pageAssets - List images, media, styles, scripts, fonts and icons referenced by the page.
|
|
675
|
+
exportContent - Export page, Google Workspace or YouTube transcript content as structured data.
|
|
676
|
+
clipboardReadText - Read text from the browser profile clipboard.
|
|
677
|
+
clipboardWriteText - Write clipboardText to the browser profile clipboard.
|
|
448
678
|
screenshot [--annotate] [--fullPage] [--path] - Save PNG; --annotate numbers interactive elements.
|
|
449
679
|
pdf [path] - Save page as PDF.
|
|
450
680
|
eval <js> - Run JavaScript, return result.
|
|
@@ -526,12 +756,14 @@ const defaultDeps: WebBrowserDeps = {
|
|
|
526
756
|
* 表驱动,新增一个写文件的动作却忘了登记,会在 `call()` 里拿不到已解析路径而立刻暴露,
|
|
527
757
|
* 而不是悄悄绕过校验。
|
|
528
758
|
*/
|
|
529
|
-
export const ARTIFACT_ACTIONS: Record<string, { prefix: string; ext: string }> =
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
}
|
|
759
|
+
export const ARTIFACT_ACTIONS: Record<string, { prefix: string; ext: string }> =
|
|
760
|
+
{
|
|
761
|
+
screenshot: { prefix: 'browser-screenshot', ext: 'png' },
|
|
762
|
+
pdf: { prefix: 'browser-page', ext: 'pdf' },
|
|
763
|
+
recordStart: { prefix: 'recording', ext: 'webm' },
|
|
764
|
+
traceStop: { prefix: 'browser-trace', ext: 'json' },
|
|
765
|
+
download: { prefix: 'browser-download', ext: 'download' },
|
|
766
|
+
};
|
|
535
767
|
|
|
536
768
|
/**
|
|
537
769
|
* @param resolvedOutPath 已经过 `validateArtifactOutputPath` 校验的落盘路径。
|
|
@@ -541,14 +773,24 @@ export const ARTIFACT_ACTIONS: Record<string, { prefix: string; ext: string }> =
|
|
|
541
773
|
function buildSubArgs(
|
|
542
774
|
input: WebBrowserInput,
|
|
543
775
|
resolvedOutPath?: string,
|
|
544
|
-
):
|
|
776
|
+
):
|
|
777
|
+
| {
|
|
778
|
+
args: string[];
|
|
779
|
+
outPath?: string;
|
|
780
|
+
metaAction?: string;
|
|
781
|
+
pointerAction?: 'click' | 'hover';
|
|
782
|
+
}
|
|
783
|
+
| { error: string } {
|
|
545
784
|
const { action } = input;
|
|
546
785
|
|
|
547
786
|
switch (action) {
|
|
548
787
|
case 'start': {
|
|
549
788
|
const args: string[] = [];
|
|
550
789
|
if (input.viewportWidth && input.viewportHeight) {
|
|
551
|
-
args.push(
|
|
790
|
+
args.push(
|
|
791
|
+
'--viewport',
|
|
792
|
+
`${input.viewportWidth}x${input.viewportHeight}`,
|
|
793
|
+
);
|
|
552
794
|
}
|
|
553
795
|
return { args, metaAction: 'start' };
|
|
554
796
|
}
|
|
@@ -574,13 +816,47 @@ function buildSubArgs(
|
|
|
574
816
|
|
|
575
817
|
case 'snapshot':
|
|
576
818
|
return { args: ['snapshot'] };
|
|
819
|
+
case 'frameInfo':
|
|
820
|
+
return { args: ['eval', FRAME_INFO_EXPRESSION] };
|
|
577
821
|
|
|
578
822
|
case 'getContent':
|
|
579
823
|
return { args: ['get', 'text', input.selector ?? 'body'] };
|
|
580
824
|
|
|
825
|
+
case 'pageAssets':
|
|
826
|
+
case 'listPageAssets':
|
|
827
|
+
return { args: ['eval', PAGE_ASSETS_EXPRESSION] };
|
|
828
|
+
case 'bundlePageAssets':
|
|
829
|
+
return { args: ['eval', PAGE_ASSETS_EXPRESSION] };
|
|
830
|
+
case 'exportContent':
|
|
831
|
+
return {
|
|
832
|
+
args: [
|
|
833
|
+
'eval',
|
|
834
|
+
pageContentExpression(input.exportFormat ?? 'page', input.selector),
|
|
835
|
+
],
|
|
836
|
+
};
|
|
837
|
+
case 'clipboardReadText':
|
|
838
|
+
return { args: ['eval', CLIPBOARD_READ_TEXT_EXPRESSION] };
|
|
839
|
+
case 'clipboardWriteText':
|
|
840
|
+
if (input.clipboardText === undefined)
|
|
841
|
+
return { error: 'clipboardWriteText requires clipboardText' };
|
|
842
|
+
return {
|
|
843
|
+
args: ['eval', clipboardWriteTextExpression(input.clipboardText)],
|
|
844
|
+
};
|
|
845
|
+
case 'clipboardRead':
|
|
846
|
+
return { args: ['eval', CLIPBOARD_READ_EXPRESSION] };
|
|
847
|
+
case 'clipboardWrite':
|
|
848
|
+
if (!input.clipboardItems?.length)
|
|
849
|
+
return { error: 'clipboardWrite requires clipboardItems' };
|
|
850
|
+
return {
|
|
851
|
+
args: ['eval', clipboardWriteExpression(input.clipboardItems)],
|
|
852
|
+
};
|
|
853
|
+
|
|
581
854
|
case 'screenshot': {
|
|
582
855
|
const out = resolvedOutPath ?? input.path;
|
|
583
|
-
if (!out)
|
|
856
|
+
if (!out)
|
|
857
|
+
return {
|
|
858
|
+
error: `${action} 需要已解析的输出路径(由 call() 校验后传入)`,
|
|
859
|
+
};
|
|
584
860
|
const args = ['screenshot', out];
|
|
585
861
|
if (input.annotate) args.push('--annotate');
|
|
586
862
|
if (input.fullPage) args.push('--full');
|
|
@@ -592,10 +868,30 @@ function buildSubArgs(
|
|
|
592
868
|
// 那三条必须在同一把锁内连发,中途被别的会话插进来就点在别的地方了。
|
|
593
869
|
if (!input.selector) {
|
|
594
870
|
if (hasPointCoords(input)) return { args: [], pointerAction: 'click' };
|
|
595
|
-
return {
|
|
871
|
+
return {
|
|
872
|
+
error:
|
|
873
|
+
'click 需要 selector;没有可用选择器时可改用视口比例坐标 x/y(0-1000)',
|
|
874
|
+
};
|
|
596
875
|
}
|
|
597
876
|
return { args: ['click', input.selector] };
|
|
598
877
|
|
|
878
|
+
case 'doubleClick':
|
|
879
|
+
if (!input.selector) return { error: 'doubleClick requires selector' };
|
|
880
|
+
return { args: ['dblclick', input.selector] };
|
|
881
|
+
case 'check':
|
|
882
|
+
if (!input.selector) return { error: 'check requires selector' };
|
|
883
|
+
return { args: ['check', input.selector] };
|
|
884
|
+
case 'uncheck':
|
|
885
|
+
if (!input.selector) return { error: 'uncheck requires selector' };
|
|
886
|
+
return { args: ['uncheck', input.selector] };
|
|
887
|
+
case 'setChecked':
|
|
888
|
+
if (!input.selector || input.checked === undefined)
|
|
889
|
+
return { error: 'setChecked requires selector and checked' };
|
|
890
|
+
return { args: [input.checked ? 'check' : 'uncheck', input.selector] };
|
|
891
|
+
case 'getState':
|
|
892
|
+
if (!input.selector) return { error: 'getState requires selector' };
|
|
893
|
+
return { args: ['is', 'visible', input.selector] };
|
|
894
|
+
|
|
599
895
|
case 'fill':
|
|
600
896
|
if (!input.selector || input.value === undefined)
|
|
601
897
|
return { error: 'fill requires selector and value parameters' };
|
|
@@ -609,14 +905,17 @@ function buildSubArgs(
|
|
|
609
905
|
case 'hover':
|
|
610
906
|
if (!input.selector) {
|
|
611
907
|
if (hasPointCoords(input)) return { args: [], pointerAction: 'hover' };
|
|
612
|
-
return {
|
|
908
|
+
return {
|
|
909
|
+
error:
|
|
910
|
+
'hover 需要 selector;没有可用选择器时可改用视口比例坐标 x/y(0-1000)',
|
|
911
|
+
};
|
|
613
912
|
}
|
|
614
913
|
return { args: ['hover', input.selector] };
|
|
615
914
|
|
|
616
915
|
case 'scroll': {
|
|
617
916
|
const args = ['scroll'];
|
|
618
917
|
if (input.selector) args.push(input.selector);
|
|
619
|
-
if (input.direction) args.push(
|
|
918
|
+
if (input.direction) args.push(`--${input.direction}`);
|
|
620
919
|
if (input.scrollY) args.push('--by-y', String(input.scrollY));
|
|
621
920
|
if (input.scrollX) args.push('--by-x', String(input.scrollX));
|
|
622
921
|
return { args };
|
|
@@ -624,14 +923,18 @@ function buildSubArgs(
|
|
|
624
923
|
|
|
625
924
|
case 'select': {
|
|
626
925
|
if (!input.selector || (input.value === undefined && !input.values))
|
|
627
|
-
return {
|
|
926
|
+
return {
|
|
927
|
+
error: 'select requires selector and value/values parameters',
|
|
928
|
+
};
|
|
628
929
|
const vals = input.values ?? [input.value!];
|
|
629
930
|
return { args: ['select', input.selector, ...vals] };
|
|
630
931
|
}
|
|
631
932
|
|
|
632
933
|
case 'press':
|
|
633
934
|
if (!input.key) return { error: 'press requires key parameter' };
|
|
634
|
-
return {
|
|
935
|
+
return {
|
|
936
|
+
args: ['press', input.key, ...(input.selector ? [input.selector] : [])],
|
|
937
|
+
};
|
|
635
938
|
|
|
636
939
|
case 'wait': {
|
|
637
940
|
// agent-browser `wait` grammar: bare positional `wait <ms>` = plain sleep,
|
|
@@ -642,8 +945,21 @@ function buildSubArgs(
|
|
|
642
945
|
if (input.waitText) return { args: ['wait', '--text', input.waitText] };
|
|
643
946
|
if (input.waitLoad) return { args: ['wait', '--load', input.waitLoad] };
|
|
644
947
|
if (input.waitMs) return { args: ['wait', String(input.waitMs)] };
|
|
645
|
-
return {
|
|
948
|
+
return {
|
|
949
|
+
error: 'wait requires waitMs / waitSelector / waitText / waitLoad',
|
|
950
|
+
};
|
|
646
951
|
}
|
|
952
|
+
case 'waitForURL':
|
|
953
|
+
if (!input.waitUrl && !input.url)
|
|
954
|
+
return { error: 'waitForURL requires waitUrl or url' };
|
|
955
|
+
return { args: ['wait', '--url', input.waitUrl ?? input.url!] };
|
|
956
|
+
case 'waitForDownload':
|
|
957
|
+
return { args: ['wait', '--download'] };
|
|
958
|
+
case 'waitForFileChooser':
|
|
959
|
+
return {
|
|
960
|
+
error:
|
|
961
|
+
'waitForFileChooser is available in BrowserBatch, where it can be atomically paired with its triggering action',
|
|
962
|
+
};
|
|
647
963
|
|
|
648
964
|
case 'eval':
|
|
649
965
|
if (!input.js) return { error: 'eval requires js parameter' };
|
|
@@ -651,7 +967,10 @@ function buildSubArgs(
|
|
|
651
967
|
|
|
652
968
|
case 'pdf': {
|
|
653
969
|
const out = resolvedOutPath ?? input.path;
|
|
654
|
-
if (!out)
|
|
970
|
+
if (!out)
|
|
971
|
+
return {
|
|
972
|
+
error: `${action} 需要已解析的输出路径(由 call() 校验后传入)`,
|
|
973
|
+
};
|
|
655
974
|
return { args: ['pdf', out], outPath: out };
|
|
656
975
|
}
|
|
657
976
|
|
|
@@ -659,9 +978,15 @@ function buildSubArgs(
|
|
|
659
978
|
return { args: ['close'] };
|
|
660
979
|
|
|
661
980
|
case 'tabNew':
|
|
662
|
-
return {
|
|
981
|
+
return {
|
|
982
|
+
args: input.url
|
|
983
|
+
? ['tab', 'new', normalizeUrl(input.url)]
|
|
984
|
+
: ['tab', 'new'],
|
|
985
|
+
};
|
|
663
986
|
case 'tabClose':
|
|
664
|
-
return {
|
|
987
|
+
return {
|
|
988
|
+
args: input.tabId ? ['tab', 'close', input.tabId] : ['tab', 'close'],
|
|
989
|
+
};
|
|
665
990
|
case 'tabSwitch':
|
|
666
991
|
if (!input.tabId) return { error: 'tabSwitch requires tabId parameter' };
|
|
667
992
|
return { args: ['tab', input.tabId] };
|
|
@@ -673,15 +998,26 @@ function buildSubArgs(
|
|
|
673
998
|
case 'cookiesSet': {
|
|
674
999
|
if (!input.cookies || input.cookies.length === 0)
|
|
675
1000
|
return { error: 'cookiesSet requires cookies array' };
|
|
676
|
-
return {
|
|
1001
|
+
return {
|
|
1002
|
+
args: ['cookies', 'set', '--json', JSON.stringify(input.cookies)],
|
|
1003
|
+
};
|
|
677
1004
|
}
|
|
678
1005
|
case 'cookiesClear':
|
|
679
1006
|
return { args: ['cookies', 'clear'] };
|
|
680
1007
|
|
|
681
1008
|
case 'setViewport': {
|
|
682
1009
|
if (!input.viewportWidth || !input.viewportHeight)
|
|
683
|
-
return {
|
|
684
|
-
|
|
1010
|
+
return {
|
|
1011
|
+
error: 'setViewport requires viewportWidth and viewportHeight',
|
|
1012
|
+
};
|
|
1013
|
+
return {
|
|
1014
|
+
args: [
|
|
1015
|
+
'set',
|
|
1016
|
+
'viewport',
|
|
1017
|
+
String(input.viewportWidth),
|
|
1018
|
+
String(input.viewportHeight),
|
|
1019
|
+
],
|
|
1020
|
+
};
|
|
685
1021
|
}
|
|
686
1022
|
|
|
687
1023
|
case 'setDevice': {
|
|
@@ -689,7 +1025,11 @@ function buildSubArgs(
|
|
|
689
1025
|
if (input.devicePreset) {
|
|
690
1026
|
args.push(input.devicePreset);
|
|
691
1027
|
} else if (input.viewportWidth && input.viewportHeight) {
|
|
692
|
-
args.push(
|
|
1028
|
+
args.push(
|
|
1029
|
+
'--viewport',
|
|
1030
|
+
String(input.viewportWidth),
|
|
1031
|
+
String(input.viewportHeight),
|
|
1032
|
+
);
|
|
693
1033
|
}
|
|
694
1034
|
return { args };
|
|
695
1035
|
}
|
|
@@ -697,20 +1037,28 @@ function buildSubArgs(
|
|
|
697
1037
|
case 'setMedia': {
|
|
698
1038
|
const args = ['set', 'media'];
|
|
699
1039
|
if (input.colorScheme) args.push('--color-scheme', input.colorScheme);
|
|
700
|
-
if (input.offline !== undefined)
|
|
1040
|
+
if (input.offline !== undefined)
|
|
1041
|
+
args.push('--offline', input.offline ? 'on' : 'off');
|
|
701
1042
|
return { args };
|
|
702
1043
|
}
|
|
703
1044
|
|
|
704
1045
|
case 'recordStart': {
|
|
705
1046
|
const out = resolvedOutPath ?? input.path;
|
|
706
|
-
if (!out)
|
|
1047
|
+
if (!out)
|
|
1048
|
+
return {
|
|
1049
|
+
error: `${action} 需要已解析的输出路径(由 call() 校验后传入)`,
|
|
1050
|
+
};
|
|
707
1051
|
return { args: ['record', 'start', out], outPath: out };
|
|
708
1052
|
}
|
|
709
1053
|
case 'recordStop':
|
|
710
1054
|
return { args: ['record', 'stop'] };
|
|
711
1055
|
|
|
712
1056
|
case 'dialogAccept':
|
|
713
|
-
return {
|
|
1057
|
+
return {
|
|
1058
|
+
args: input.dialogText
|
|
1059
|
+
? ['dialog', 'accept', '--text', input.dialogText]
|
|
1060
|
+
: ['dialog', 'accept'],
|
|
1061
|
+
};
|
|
714
1062
|
case 'dialogDismiss':
|
|
715
1063
|
return { args: ['dialog', 'dismiss'] };
|
|
716
1064
|
|
|
@@ -720,7 +1068,8 @@ function buildSubArgs(
|
|
|
720
1068
|
return { args: [], metaAction: 'pickElement' };
|
|
721
1069
|
}
|
|
722
1070
|
case 'markElement': {
|
|
723
|
-
if (!input.selector)
|
|
1071
|
+
if (!input.selector)
|
|
1072
|
+
return { error: 'markElement requires selector parameter' };
|
|
724
1073
|
return { args: [], metaAction: 'markElement' };
|
|
725
1074
|
}
|
|
726
1075
|
case 'clearMarks':
|
|
@@ -732,25 +1081,45 @@ function buildSubArgs(
|
|
|
732
1081
|
return { args: ['upload', input.selector, ...input.files] };
|
|
733
1082
|
}
|
|
734
1083
|
|
|
1084
|
+
case 'download': {
|
|
1085
|
+
if (!input.selector) return { error: 'download requires selector' };
|
|
1086
|
+
const out = resolvedOutPath ?? input.path;
|
|
1087
|
+
if (!out) return { error: 'download requires output path' };
|
|
1088
|
+
return { args: ['download', input.selector, out], outPath: out };
|
|
1089
|
+
}
|
|
1090
|
+
|
|
735
1091
|
case 'drag': {
|
|
736
1092
|
if (!input.selector || !input.dragTo)
|
|
737
|
-
return {
|
|
1093
|
+
return {
|
|
1094
|
+
error: 'drag requires selector (from) and dragTo (to) parameters',
|
|
1095
|
+
};
|
|
738
1096
|
return { args: ['drag', input.selector, input.dragTo] };
|
|
739
1097
|
}
|
|
740
|
-
case 'console':
|
|
741
|
-
|
|
742
|
-
case '
|
|
743
|
-
|
|
744
|
-
case '
|
|
745
|
-
|
|
1098
|
+
case 'console':
|
|
1099
|
+
return { args: ['console'] };
|
|
1100
|
+
case 'errors':
|
|
1101
|
+
return { args: ['errors'] };
|
|
1102
|
+
case 'networkRequests':
|
|
1103
|
+
return { args: ['network', 'requests', '--json'] };
|
|
1104
|
+
case 'getStyles':
|
|
1105
|
+
return { args: ['get', 'styles', input.selector ?? 'body', '--json'] };
|
|
1106
|
+
case 'vitals':
|
|
1107
|
+
return { args: ['vitals', ...(input.url ? [input.url] : []), '--json'] };
|
|
1108
|
+
case 'traceStart':
|
|
1109
|
+
return { args: ['trace', 'start'] };
|
|
746
1110
|
case 'traceStop': {
|
|
747
1111
|
const out = resolvedOutPath ?? input.path;
|
|
748
|
-
if (!out)
|
|
1112
|
+
if (!out)
|
|
1113
|
+
return {
|
|
1114
|
+
error: `${action} 需要已解析的输出路径(由 call() 校验后传入)`,
|
|
1115
|
+
};
|
|
749
1116
|
return { args: ['trace', 'stop', out], outPath: out };
|
|
750
1117
|
}
|
|
751
|
-
case 'reactTree':
|
|
1118
|
+
case 'reactTree':
|
|
1119
|
+
return { args: ['react', 'tree'] };
|
|
752
1120
|
case 'reactInspect':
|
|
753
|
-
if (!input.reactId)
|
|
1121
|
+
if (!input.reactId)
|
|
1122
|
+
return { error: 'reactInspect requires reactId parameter' };
|
|
754
1123
|
return { args: ['react', 'inspect', input.reactId] };
|
|
755
1124
|
|
|
756
1125
|
default:
|
|
@@ -780,7 +1149,8 @@ async function handleMetaAction(
|
|
|
780
1149
|
switch (action) {
|
|
781
1150
|
case 'start': {
|
|
782
1151
|
const info = await ensureDaemon(BROWSER_SESSION, { signal });
|
|
783
|
-
const streamPort =
|
|
1152
|
+
const streamPort =
|
|
1153
|
+
info.streamPort ?? (await getStreamPort(BROWSER_SESSION));
|
|
784
1154
|
stream.setMode('browse');
|
|
785
1155
|
// actor 点亮媒体面:受监督宿主下把截屏流连上,面板从此刻起就能显示 agent 的画面,
|
|
786
1156
|
// 不再依赖面板自启动(曾因面板 browser/start 竞态/连到坏 daemon 而「开着却全黑」)。
|
|
@@ -805,7 +1175,10 @@ async function handleMetaAction(
|
|
|
805
1175
|
// ⚠️ 这只覆盖「显式 stop」这一条路径;进程被 kill、宿主直接丢弃会话时仍然
|
|
806
1176
|
// 留残留——彻底收口需要宿主在会话结束时主动调 `forgetConversation`。
|
|
807
1177
|
const key = ctx.browserRun?.conversationKey;
|
|
808
|
-
if (key) {
|
|
1178
|
+
if (key) {
|
|
1179
|
+
browserRunRegistry.forgetConversation(key);
|
|
1180
|
+
browserAttempts.forget(key);
|
|
1181
|
+
}
|
|
809
1182
|
return { ok: true, content: '[stop] browser stopped' };
|
|
810
1183
|
}
|
|
811
1184
|
case 'streamInfo': {
|
|
@@ -827,7 +1200,11 @@ async function handleMetaAction(
|
|
|
827
1200
|
const port = stream.getPort();
|
|
828
1201
|
return {
|
|
829
1202
|
ok: true,
|
|
830
|
-
content: JSON.stringify({
|
|
1203
|
+
content: JSON.stringify({
|
|
1204
|
+
connected: stream.isConnected(),
|
|
1205
|
+
port,
|
|
1206
|
+
status,
|
|
1207
|
+
}),
|
|
831
1208
|
};
|
|
832
1209
|
}
|
|
833
1210
|
case 'setMode': {
|
|
@@ -837,15 +1214,30 @@ async function handleMetaAction(
|
|
|
837
1214
|
}
|
|
838
1215
|
case 'pickElement': {
|
|
839
1216
|
const el = await stream.pickElement(input.pickX!, input.pickY!);
|
|
840
|
-
if (!el)
|
|
841
|
-
|
|
1217
|
+
if (!el)
|
|
1218
|
+
return {
|
|
1219
|
+
ok: false,
|
|
1220
|
+
error: 'pickElement: no element found at coordinates',
|
|
1221
|
+
};
|
|
1222
|
+
stream.markElement({
|
|
1223
|
+
ref: el.ref,
|
|
1224
|
+
tag: el.tag,
|
|
1225
|
+
text: el.text,
|
|
1226
|
+
boundingBox: el.boundingBox,
|
|
1227
|
+
label: el.ref,
|
|
1228
|
+
});
|
|
842
1229
|
return {
|
|
843
1230
|
ok: true,
|
|
844
1231
|
content: JSON.stringify(el),
|
|
845
1232
|
};
|
|
846
1233
|
}
|
|
847
1234
|
case 'markElement': {
|
|
848
|
-
const el = {
|
|
1235
|
+
const el = {
|
|
1236
|
+
ref: input.selector!,
|
|
1237
|
+
tag: input.selector!,
|
|
1238
|
+
label: input.markLabel,
|
|
1239
|
+
color: input.markColor,
|
|
1240
|
+
};
|
|
849
1241
|
stream.markElement(el);
|
|
850
1242
|
return { ok: true, content: `[markElement] marked ${input.selector}` };
|
|
851
1243
|
}
|
|
@@ -865,27 +1257,45 @@ async function handleMetaAction(
|
|
|
865
1257
|
function describeAction(input: WebBrowserInput): string {
|
|
866
1258
|
const target = input.selector ?? input.url ?? '';
|
|
867
1259
|
switch (input.action) {
|
|
868
|
-
case 'navigate':
|
|
869
|
-
|
|
870
|
-
case '
|
|
871
|
-
|
|
872
|
-
case '
|
|
873
|
-
|
|
874
|
-
case '
|
|
875
|
-
|
|
876
|
-
case '
|
|
877
|
-
|
|
878
|
-
case '
|
|
879
|
-
|
|
880
|
-
case '
|
|
881
|
-
|
|
882
|
-
case '
|
|
883
|
-
|
|
1260
|
+
case 'navigate':
|
|
1261
|
+
return `正在打开 ${input.url ?? ''}`.trim();
|
|
1262
|
+
case 'click':
|
|
1263
|
+
return `正在点击 ${target}`.trim();
|
|
1264
|
+
case 'fill':
|
|
1265
|
+
return `正在填写 ${input.selector ?? ''}`.trim();
|
|
1266
|
+
case 'type':
|
|
1267
|
+
return `正在输入到 ${input.selector ?? ''}`.trim();
|
|
1268
|
+
case 'hover':
|
|
1269
|
+
return `正在悬停 ${input.selector ?? ''}`.trim();
|
|
1270
|
+
case 'select':
|
|
1271
|
+
return `正在选择 ${input.selector ?? ''}`.trim();
|
|
1272
|
+
case 'press':
|
|
1273
|
+
return `正在按键 ${input.key ?? ''}`.trim();
|
|
1274
|
+
case 'scroll':
|
|
1275
|
+
return '正在滚动页面';
|
|
1276
|
+
case 'drag':
|
|
1277
|
+
return `正在拖拽 ${input.selector ?? ''}`.trim();
|
|
1278
|
+
case 'upload':
|
|
1279
|
+
return '正在上传文件';
|
|
1280
|
+
case 'back':
|
|
1281
|
+
return '正在后退';
|
|
1282
|
+
case 'forward':
|
|
1283
|
+
return '正在前进';
|
|
1284
|
+
case 'reload':
|
|
1285
|
+
return '正在刷新页面';
|
|
1286
|
+
case 'screenshot':
|
|
1287
|
+
return '正在截图';
|
|
1288
|
+
case 'pdf':
|
|
1289
|
+
return '正在导出 PDF';
|
|
1290
|
+
default:
|
|
1291
|
+
return `正在执行 ${input.action}`;
|
|
884
1292
|
}
|
|
885
1293
|
}
|
|
886
1294
|
|
|
887
1295
|
/** C4:把工具动作映射成指针动效类型(协议 action 枚举)。 */
|
|
888
|
-
function pointerActionFor(
|
|
1296
|
+
function pointerActionFor(
|
|
1297
|
+
action: WebBrowserInput['action'],
|
|
1298
|
+
): 'move' | 'click' | 'type' | 'hover' {
|
|
889
1299
|
if (action === 'fill' || action === 'type') return 'type';
|
|
890
1300
|
if (action === 'hover') return 'hover';
|
|
891
1301
|
if (action === 'drag') return 'move';
|
|
@@ -900,7 +1310,10 @@ async function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
|
900
1310
|
signal?.removeEventListener('abort', onAbort);
|
|
901
1311
|
resolve();
|
|
902
1312
|
}, ms);
|
|
903
|
-
const onAbort = () => {
|
|
1313
|
+
const onAbort = () => {
|
|
1314
|
+
clearTimeout(timer);
|
|
1315
|
+
resolve();
|
|
1316
|
+
};
|
|
904
1317
|
signal?.addEventListener('abort', onAbort, { once: true });
|
|
905
1318
|
});
|
|
906
1319
|
}
|
|
@@ -909,7 +1322,12 @@ async function sleep(ms: number, signal?: AbortSignal): Promise<void> {
|
|
|
909
1322
|
* Pure visual enhancement - failures are silently ignored (never block the action). */
|
|
910
1323
|
function flashTargetElement(stream: BrowserStream, selector: string): void {
|
|
911
1324
|
try {
|
|
912
|
-
stream.markElement({
|
|
1325
|
+
stream.markElement({
|
|
1326
|
+
ref: selector,
|
|
1327
|
+
tag: selector,
|
|
1328
|
+
color: '#22c55e',
|
|
1329
|
+
label: '',
|
|
1330
|
+
});
|
|
913
1331
|
setTimeout(() => stream.unmarkElement(selector), 350);
|
|
914
1332
|
} catch {
|
|
915
1333
|
// Highlight is cosmetic only; failures must never break the action.
|
|
@@ -922,7 +1340,12 @@ function parsePointerCenter(
|
|
|
922
1340
|
): { x: number; y: number } | null {
|
|
923
1341
|
try {
|
|
924
1342
|
if (result.code !== 0) return null;
|
|
925
|
-
const box = JSON.parse(result.stdout.trim()) as {
|
|
1343
|
+
const box = JSON.parse(result.stdout.trim()) as {
|
|
1344
|
+
x?: number;
|
|
1345
|
+
y?: number;
|
|
1346
|
+
width?: number;
|
|
1347
|
+
height?: number;
|
|
1348
|
+
};
|
|
926
1349
|
const x = Number(box.x);
|
|
927
1350
|
const y = Number(box.y);
|
|
928
1351
|
const w = Number(box.width);
|
|
@@ -970,7 +1393,10 @@ export async function settleSnapshot(
|
|
|
970
1393
|
}
|
|
971
1394
|
|
|
972
1395
|
/** C1:成功且拿到探测快照时,把「操作后页面快照」并入动作结果,省掉模型再单独调 snapshot 的往返。 */
|
|
973
|
-
function withPostActionSnapshot(
|
|
1396
|
+
function withPostActionSnapshot(
|
|
1397
|
+
base: ToolResult,
|
|
1398
|
+
snapshot: string,
|
|
1399
|
+
): ToolResult {
|
|
974
1400
|
if (!base.ok || !snapshot) return base;
|
|
975
1401
|
// 成功路径同样要走预算:一次 navigate 后的整页快照在真实站点上轻松几万字符,
|
|
976
1402
|
// 而模型下一步真正会用到的只是那些带 ref 的可操作行。
|
|
@@ -985,7 +1411,8 @@ async function call(
|
|
|
985
1411
|
ctx: ToolCallContext = {},
|
|
986
1412
|
): Promise<ToolResult> {
|
|
987
1413
|
const browserConfig = await configService.browser();
|
|
988
|
-
if (!browserConfig.enabled)
|
|
1414
|
+
if (!browserConfig.enabled)
|
|
1415
|
+
return { ok: false, error: '内置浏览器已在设置中关闭' };
|
|
989
1416
|
|
|
990
1417
|
// 顺序是有讲究的:**纯参数校验排在一切副作用之前**。
|
|
991
1418
|
// 解析二进制、起 daemon、问页面 URL 都是有代价的动作,参数本来就不合法时不该发生
|
|
@@ -994,12 +1421,14 @@ async function call(
|
|
|
994
1421
|
|
|
995
1422
|
// `javascript:` 是在页面里执行代码,必须走受门禁约束的 `eval`,不能靠一个看起来像
|
|
996
1423
|
// "打开网址"的动作绕过去;`data:` 可构造带脚本的文档,等价同一件事。
|
|
997
|
-
if (
|
|
998
|
-
|
|
1424
|
+
if (
|
|
1425
|
+
(input.action === 'navigate' || input.action === 'tabNew') &&
|
|
1426
|
+
input.url &&
|
|
1427
|
+
!isAllowedBrowserScheme(input.url)
|
|
1428
|
+
) {
|
|
999
1429
|
return {
|
|
1000
1430
|
ok: false,
|
|
1001
|
-
error: `[${input.action}] 不支持的地址协议:${input.url.split(':')[0]}:。仅允许 http/https/about/file
|
|
1002
|
-
+ '要在页面里执行脚本请用 eval(需 Developer Mode 授权)。',
|
|
1431
|
+
error: `[${input.action}] 不支持的地址协议:${input.url.split(':')[0]}:。仅允许 http/https/about/file;要在页面里执行脚本请用 eval(需 Developer Mode 授权)。`,
|
|
1003
1432
|
};
|
|
1004
1433
|
}
|
|
1005
1434
|
|
|
@@ -1014,13 +1443,45 @@ async function call(
|
|
|
1014
1443
|
...(ctx.browserRun?.allowedUploadRoots ?? []),
|
|
1015
1444
|
];
|
|
1016
1445
|
const checked = await validateArtifactOutputPath(input.path, roots);
|
|
1017
|
-
if (!checked.ok)
|
|
1446
|
+
if (!checked.ok)
|
|
1447
|
+
return { ok: false, error: `[${input.action}] ${checked.error}` };
|
|
1018
1448
|
resolvedOutPath = checked.path;
|
|
1449
|
+
} else if (input.action === 'download') {
|
|
1450
|
+
const dir = browserDownloadDir(
|
|
1451
|
+
ctx.browserRun?.conversationKey,
|
|
1452
|
+
ctx.browserRun?.runId ?? ctx.toolCallId,
|
|
1453
|
+
);
|
|
1454
|
+
await mkdir(dir, { recursive: true, mode: 0o700 });
|
|
1455
|
+
resolvedOutPath = join(
|
|
1456
|
+
dir,
|
|
1457
|
+
`download-${Date.now()}-${randomUUID().slice(0, 8)}.download`,
|
|
1458
|
+
);
|
|
1019
1459
|
} else {
|
|
1020
1460
|
resolvedOutPath = await createArtifactPath(artifact.prefix, artifact.ext);
|
|
1021
1461
|
}
|
|
1022
1462
|
}
|
|
1023
1463
|
|
|
1464
|
+
// Asset bundling writes a manifest and fetched files. Validate its root
|
|
1465
|
+
// before starting the browser or evaluating the page, just like the other
|
|
1466
|
+
// browser write actions.
|
|
1467
|
+
if (input.action === 'bundlePageAssets') {
|
|
1468
|
+
if (!input.bundleDir)
|
|
1469
|
+
return { ok: false, error: 'bundlePageAssets requires bundleDir' };
|
|
1470
|
+
const roots = [browserArtifactDir(), getWorkingDir()];
|
|
1471
|
+
const target = await realpath(input.bundleDir).catch(() =>
|
|
1472
|
+
resolve(input.bundleDir!),
|
|
1473
|
+
);
|
|
1474
|
+
const allowed = roots.some((root) => {
|
|
1475
|
+
const child = relative(root, target);
|
|
1476
|
+
return child === '' || (!child.startsWith('..') && !isAbsolute(child));
|
|
1477
|
+
});
|
|
1478
|
+
if (!allowed)
|
|
1479
|
+
return {
|
|
1480
|
+
ok: false,
|
|
1481
|
+
error: `资产打包目录不在浏览器产物或工作区范围内:${input.bundleDir}`,
|
|
1482
|
+
};
|
|
1483
|
+
}
|
|
1484
|
+
|
|
1024
1485
|
const built = buildSubArgs(input, resolvedOutPath);
|
|
1025
1486
|
if ('error' in built) return { ok: false, error: built.error };
|
|
1026
1487
|
|
|
@@ -1039,17 +1500,24 @@ async function call(
|
|
|
1039
1500
|
'简单网页读取可改用 WebFetch(无需浏览器)。',
|
|
1040
1501
|
};
|
|
1041
1502
|
}
|
|
1042
|
-
|
|
1043
1503
|
// Developer Mode 的开关本身是纯配置判定,没有副作用,在这里挡掉最省事。
|
|
1044
1504
|
// 但**站点 origin 判定不在这里**——它需要「当前真实 URL」,而那取决于本次动作会
|
|
1045
1505
|
// 落在哪个标签页,此刻还没认领。它挪进了下面的动作序列内(见 `requireLiveOrigin`)。
|
|
1046
|
-
if (
|
|
1047
|
-
|
|
1506
|
+
if (
|
|
1507
|
+
DEVELOPER_MODE_ACTIONS.has(input.action) &&
|
|
1508
|
+
!INTERNAL_SCRIPT_ACTIONS.has(input.action) &&
|
|
1509
|
+
!browserConfig.developerMode
|
|
1510
|
+
) {
|
|
1511
|
+
return {
|
|
1512
|
+
ok: false,
|
|
1513
|
+
error: '完整浏览器调试未授权。请先在“设置 → 浏览器”开启 Developer Mode。',
|
|
1514
|
+
};
|
|
1048
1515
|
}
|
|
1049
1516
|
|
|
1050
1517
|
const stream = getBrowserStream(BROWSER_SESSION);
|
|
1051
1518
|
const runId = ctx.browserRun?.runId ?? ctx.toolCallId ?? randomUUID();
|
|
1052
|
-
const conversationKey =
|
|
1519
|
+
const conversationKey =
|
|
1520
|
+
ctx.browserRun?.conversationKey ?? `headless:${process.pid}`;
|
|
1053
1521
|
const browserContext = browserRunRegistry.ensure({
|
|
1054
1522
|
conversationKey,
|
|
1055
1523
|
runId,
|
|
@@ -1059,36 +1527,65 @@ async function call(
|
|
|
1059
1527
|
allowedUploadRoots: ctx.browserRun?.allowedUploadRoots,
|
|
1060
1528
|
});
|
|
1061
1529
|
try {
|
|
1062
|
-
browserRunRegistry.requireGrant(
|
|
1530
|
+
browserRunRegistry.requireGrant(
|
|
1531
|
+
browserContext,
|
|
1532
|
+
OBSERVE_ACTIONS.has(input.action) ? 'observe' : 'interact',
|
|
1533
|
+
);
|
|
1063
1534
|
if (input.action === 'upload') {
|
|
1064
1535
|
browserRunRegistry.requireGrant(browserContext, 'upload');
|
|
1065
|
-
const roots = await Promise.all(
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1536
|
+
const roots = await Promise.all(
|
|
1537
|
+
browserContext.allowedUploadRoots.map(async (root) => {
|
|
1538
|
+
try {
|
|
1539
|
+
return await realpath(root);
|
|
1540
|
+
} catch {
|
|
1541
|
+
return resolve(root);
|
|
1542
|
+
}
|
|
1543
|
+
}),
|
|
1544
|
+
);
|
|
1545
|
+
if (roots.length === 0)
|
|
1546
|
+
return { ok: false, error: '当前浏览器任务没有获批的文件上传范围' };
|
|
1069
1547
|
for (const file of input.files ?? []) {
|
|
1070
1548
|
let canonical: string;
|
|
1071
|
-
try {
|
|
1072
|
-
|
|
1549
|
+
try {
|
|
1550
|
+
canonical = await realpath(file);
|
|
1551
|
+
} catch {
|
|
1552
|
+
return { ok: false, error: `上传文件不存在或不可读取:${file}` };
|
|
1553
|
+
}
|
|
1073
1554
|
const allowed = roots.some((root) => {
|
|
1074
1555
|
const rel = relative(root, canonical);
|
|
1075
1556
|
return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel));
|
|
1076
1557
|
});
|
|
1077
|
-
if (!allowed)
|
|
1558
|
+
if (!allowed)
|
|
1559
|
+
return { ok: false, error: `上传文件超出当前任务批准范围:${file}` };
|
|
1078
1560
|
}
|
|
1079
1561
|
}
|
|
1080
|
-
if (input.intent === 'submit')
|
|
1562
|
+
if (input.intent === 'submit')
|
|
1563
|
+
browserRunRegistry.requireGrant(browserContext, 'submit');
|
|
1081
1564
|
} catch (error) {
|
|
1082
|
-
return {
|
|
1565
|
+
return {
|
|
1566
|
+
ok: false,
|
|
1567
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1568
|
+
};
|
|
1083
1569
|
}
|
|
1084
1570
|
// 仅当宿主声明「有可见浏览器 + 真人可接管」(Desktop)才允许遇关卡阻塞等人;
|
|
1085
1571
|
// CLI -p / TUI / gateway 不设 browserRun → supervised=false → 据实非阻塞返回,绝不挂起。
|
|
1086
1572
|
const supervised = ctx.browserRun?.supervised === true;
|
|
1087
|
-
const visibleAgentAction = ![
|
|
1573
|
+
const visibleAgentAction = ![
|
|
1574
|
+
'snapshot',
|
|
1575
|
+
'getContent',
|
|
1576
|
+
'console',
|
|
1577
|
+
'errors',
|
|
1578
|
+
'networkRequests',
|
|
1579
|
+
'getStyles',
|
|
1580
|
+
'vitals',
|
|
1581
|
+
].includes(input.action);
|
|
1088
1582
|
|
|
1089
1583
|
// 预备步(认领会话标签页)专用控制器:受父级中断 + 超时约束,但不被「接管」打断(快速一次性)。
|
|
1090
1584
|
const preCtrl = new AbortController();
|
|
1091
|
-
let preTimer: ReturnType<typeof setTimeout> | null = setTimeout(
|
|
1585
|
+
let preTimer: ReturnType<typeof setTimeout> | null = setTimeout(
|
|
1586
|
+
() => preCtrl.abort(),
|
|
1587
|
+
input.timeout ?? 120_000,
|
|
1588
|
+
);
|
|
1092
1589
|
const abortPreFromParent = () => preCtrl.abort(signal?.reason);
|
|
1093
1590
|
if (signal) {
|
|
1094
1591
|
if (signal.aborted) preCtrl.abort();
|
|
@@ -1123,27 +1620,70 @@ async function call(
|
|
|
1123
1620
|
else signal.addEventListener('abort', onParent, { once: true });
|
|
1124
1621
|
}
|
|
1125
1622
|
// subscribe 立即以当前状态回调一次:动作开始时是 running(不掐);中途进入 waiting_human 才掐断。
|
|
1126
|
-
const unsub = stream.subscribeControl((state) => {
|
|
1623
|
+
const unsub = stream.subscribeControl((state) => {
|
|
1624
|
+
if (state.status === 'waiting_human') ac.abort();
|
|
1625
|
+
});
|
|
1127
1626
|
try {
|
|
1128
|
-
return await
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1627
|
+
return await withBrowserRecovery(
|
|
1628
|
+
() =>
|
|
1629
|
+
browserService.exclusive(
|
|
1630
|
+
{
|
|
1631
|
+
sessionName: BROWSER_SESSION,
|
|
1632
|
+
bin,
|
|
1633
|
+
runner: deps.run,
|
|
1634
|
+
signal: ac.signal,
|
|
1635
|
+
},
|
|
1636
|
+
async (run) => {
|
|
1637
|
+
let tabEnsured = false;
|
|
1638
|
+
// 标签页只在本序列的第一条命令前切一次:同一把锁内没人能把它切走,
|
|
1639
|
+
// 每条都切纯属浪费一次子进程往返。
|
|
1640
|
+
const scoped = async (
|
|
1641
|
+
args: string[],
|
|
1642
|
+
): Promise<AgentBrowserRun> => {
|
|
1643
|
+
if (claimedTabId && !tabEnsured) {
|
|
1644
|
+
const switched = await run([
|
|
1645
|
+
...sessionArgs(BROWSER_SESSION),
|
|
1646
|
+
'tab',
|
|
1647
|
+
claimedTabId,
|
|
1648
|
+
]);
|
|
1649
|
+
if (switched.code !== 0) {
|
|
1650
|
+
const recoverable =
|
|
1651
|
+
recoverableAgentBrowserRunFailure(switched);
|
|
1652
|
+
if (recoverable) throw recoverable;
|
|
1653
|
+
return switched;
|
|
1654
|
+
}
|
|
1655
|
+
tabEnsured = true;
|
|
1656
|
+
}
|
|
1657
|
+
const result = await run([
|
|
1658
|
+
...sessionArgs(BROWSER_SESSION),
|
|
1659
|
+
...args,
|
|
1660
|
+
]);
|
|
1661
|
+
const recoverable = recoverableAgentBrowserRunFailure(result);
|
|
1662
|
+
if (recoverable) throw recoverable;
|
|
1663
|
+
return result;
|
|
1664
|
+
};
|
|
1665
|
+
return body(scoped);
|
|
1666
|
+
},
|
|
1667
|
+
),
|
|
1668
|
+
{
|
|
1669
|
+
signal: ac.signal,
|
|
1670
|
+
reconnect: async () => {
|
|
1671
|
+
await deps.ensureDaemon(BROWSER_SESSION, { signal: ac.signal });
|
|
1672
|
+
},
|
|
1673
|
+
rebindTab: async () => {
|
|
1674
|
+
if (!claimedTabId) return;
|
|
1675
|
+
const rebound = await deps.run(
|
|
1676
|
+
bin,
|
|
1677
|
+
[...sessionArgs(BROWSER_SESSION), 'tab', claimedTabId],
|
|
1678
|
+
ac.signal,
|
|
1679
|
+
);
|
|
1680
|
+
if (rebound.code !== 0)
|
|
1681
|
+
throw new Error(
|
|
1682
|
+
rebound.stderr || '浏览器标签页已失效,无法重新绑定',
|
|
1683
|
+
);
|
|
1684
|
+
},
|
|
1685
|
+
},
|
|
1686
|
+
);
|
|
1147
1687
|
} finally {
|
|
1148
1688
|
clearTimeout(to);
|
|
1149
1689
|
signal?.removeEventListener('abort', onParent);
|
|
@@ -1155,14 +1695,18 @@ async function call(
|
|
|
1155
1695
|
const runAction = (args: string[]): Promise<AgentBrowserRun> =>
|
|
1156
1696
|
runSequence((run) => run(args));
|
|
1157
1697
|
|
|
1158
|
-
stream.beginAgentAction({ runId, conversationKey, action: input.action });
|
|
1159
1698
|
let pointerShown = false; // C4:本次是否点亮了 AI 指针(finally 淡出用)
|
|
1160
1699
|
|
|
1161
|
-
// ⚠️
|
|
1162
|
-
//
|
|
1163
|
-
//
|
|
1164
|
-
//
|
|
1700
|
+
// ⚠️ 取锁是 try 的**第一行**。这把锁是进程级单例,唯一的释放路径是下面的 finally,
|
|
1701
|
+
// 所以「拿到锁却没能进 try」必须在结构上不可能发生 —— 而不是靠后来者记得别往
|
|
1702
|
+
// begin 与 try 之间插代码。历史上这里正是 begin 在外、旁白(ctx.reportProgress,
|
|
1703
|
+
// 宿主注入的回调)也在外:它一抛错就跳过整个 try/finally,锁再不释放,此后该进程内
|
|
1704
|
+
// **任何**会话都撞「正在执行另一个会话的原子操作」,且现场已经没了。
|
|
1705
|
+
//
|
|
1706
|
+
// 取锁本身失败(锁被别人持有)时 finally 照常执行 finishAgentAction,这是安全的:
|
|
1707
|
+
// finishAgent 校验 state.runId === runId,持有者是别人就不会被误释放。
|
|
1165
1708
|
try {
|
|
1709
|
+
stream.beginAgentAction({ runId, conversationKey, action: input.action });
|
|
1166
1710
|
if (visibleAgentAction) {
|
|
1167
1711
|
stream.setAgentActing(true, input.action, input.selector ?? input.url);
|
|
1168
1712
|
ctx.reportProgress?.(describeAction(input)); // C3:每个可见动作发一句人话旁白到对话/状态行
|
|
@@ -1176,15 +1720,21 @@ async function call(
|
|
|
1176
1720
|
// different conversation and actions silently cross session boundaries.
|
|
1177
1721
|
const globals = sessionArgs(BROWSER_SESSION);
|
|
1178
1722
|
if (input.action === 'tabNew') {
|
|
1179
|
-
const tabId = await createConversationTab(
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1723
|
+
const tabId = await createConversationTab(
|
|
1724
|
+
{
|
|
1725
|
+
context: browserContext,
|
|
1726
|
+
sessionName: BROWSER_SESSION,
|
|
1727
|
+
bin,
|
|
1728
|
+
globals,
|
|
1729
|
+
runner: deps.run,
|
|
1730
|
+
signal: preCtrl.signal,
|
|
1731
|
+
},
|
|
1732
|
+
input.url ? normalizeUrl(input.url) : 'about:blank',
|
|
1733
|
+
);
|
|
1734
|
+
if (preTimer) {
|
|
1735
|
+
clearTimeout(preTimer);
|
|
1736
|
+
preTimer = null;
|
|
1737
|
+
}
|
|
1188
1738
|
return { ok: true, content: JSON.stringify({ tabId }) };
|
|
1189
1739
|
}
|
|
1190
1740
|
if (input.action === 'tabList') {
|
|
@@ -1195,13 +1745,30 @@ async function call(
|
|
|
1195
1745
|
runner: deps.run,
|
|
1196
1746
|
signal: preCtrl.signal,
|
|
1197
1747
|
});
|
|
1198
|
-
const claimed = new Set(
|
|
1199
|
-
|
|
1200
|
-
|
|
1748
|
+
const claimed = new Set(
|
|
1749
|
+
browserRunRegistry.claimedTabs(browserContext.conversationKey),
|
|
1750
|
+
);
|
|
1751
|
+
if (preTimer) {
|
|
1752
|
+
clearTimeout(preTimer);
|
|
1753
|
+
preTimer = null;
|
|
1754
|
+
}
|
|
1755
|
+
return {
|
|
1756
|
+
ok: true,
|
|
1757
|
+
content: JSON.stringify(tabs.filter((tab) => claimed.has(tab.id))),
|
|
1758
|
+
};
|
|
1201
1759
|
}
|
|
1202
1760
|
if (input.action === 'tabSwitch' && input.tabId) {
|
|
1203
|
-
|
|
1204
|
-
|
|
1761
|
+
claimedTabId = await claimConversationTabById(
|
|
1762
|
+
{
|
|
1763
|
+
context: browserContext,
|
|
1764
|
+
sessionName: BROWSER_SESSION,
|
|
1765
|
+
bin,
|
|
1766
|
+
globals,
|
|
1767
|
+
runner: deps.run,
|
|
1768
|
+
signal: preCtrl.signal,
|
|
1769
|
+
},
|
|
1770
|
+
input.tabId,
|
|
1771
|
+
);
|
|
1205
1772
|
} else if (input.action === 'tabClose' && input.tabId) {
|
|
1206
1773
|
browserRunRegistry.assertTab(browserContext, input.tabId);
|
|
1207
1774
|
claimedTabId = input.tabId;
|
|
@@ -1215,19 +1782,27 @@ async function call(
|
|
|
1215
1782
|
signal: preCtrl.signal,
|
|
1216
1783
|
});
|
|
1217
1784
|
}
|
|
1218
|
-
if (
|
|
1785
|
+
if (input.retention) {
|
|
1786
|
+
browserRunRegistry.markTab(browserContext, claimedTabId, {
|
|
1787
|
+
retention: input.retention,
|
|
1788
|
+
...(input.tabTitle ? { title: input.tabTitle } : {}),
|
|
1789
|
+
});
|
|
1790
|
+
}
|
|
1791
|
+
if (preTimer) {
|
|
1792
|
+
clearTimeout(preTimer);
|
|
1793
|
+
preTimer = null;
|
|
1794
|
+
}
|
|
1219
1795
|
|
|
1220
1796
|
// 同一 (动作, 选择器) 连续失败到上限 → 拒绝再试,强制换路径。
|
|
1221
1797
|
// 挡的是「原地换着选择器盲猜三五轮然后放弃」这个最常见的失败形状。
|
|
1222
|
-
if (
|
|
1223
|
-
|
|
1798
|
+
if (
|
|
1799
|
+
VERIFIED_BROWSER_ACTIONS.has(input.action) &&
|
|
1800
|
+
browserAttempts.failures(conversationKey, input.action, input.selector) >=
|
|
1801
|
+
BROWSER_FAILURE_LIMIT
|
|
1802
|
+
) {
|
|
1224
1803
|
return {
|
|
1225
1804
|
ok: false,
|
|
1226
|
-
error:
|
|
1227
|
-
`[${input.action}] 这个目标(${input.selector ?? '无选择器'})已经连续失败 `
|
|
1228
|
-
+ `${BROWSER_FAILURE_LIMIT} 次,引擎暂停了对它的重试。\n`
|
|
1229
|
-
+ '请先 snapshot 看清当前页面到底是什么状态(观测不受限制),换一个元素引用或换一条路径;'
|
|
1230
|
-
+ '如果判断不出来,就把现状如实告诉用户,不要继续盲试。',
|
|
1805
|
+
error: `[${input.action}] 这个目标(${input.selector ?? '无选择器'})已经连续失败 ${BROWSER_FAILURE_LIMIT} 次,引擎暂停了对它的重试。\n请先 snapshot 看清当前页面到底是什么状态(观测不受限制),换一个元素引用或换一条路径;如果判断不出来,就把现状如实告诉用户,不要继续盲试。`,
|
|
1231
1806
|
};
|
|
1232
1807
|
}
|
|
1233
1808
|
|
|
@@ -1239,27 +1814,40 @@ async function call(
|
|
|
1239
1814
|
// 一段控制流——谁加一条提前 return,拿到的就是运行时 undefined。
|
|
1240
1815
|
type SequenceOutcome =
|
|
1241
1816
|
| { blocked: string }
|
|
1242
|
-
| {
|
|
1817
|
+
| {
|
|
1818
|
+
blocked?: undefined;
|
|
1819
|
+
result: AgentBrowserRun;
|
|
1820
|
+
before?: string;
|
|
1821
|
+
probeSnapshot: string;
|
|
1822
|
+
noiseAfter?: string;
|
|
1823
|
+
};
|
|
1243
1824
|
const outcome = await runSequence<SequenceOutcome>(async (run) => {
|
|
1244
1825
|
// 站点白名单:判据必须是**当前真实 URL**,而且要在**本次动作真正会落到的那个
|
|
1245
1826
|
// 标签页**上取。缓存不行(navigate 只写「请求的」URL,非监督宿主恒 about:blank,
|
|
1246
1827
|
// 一个 302 就让白名单以为你还在 allowed 站点);锁外也不行(那时标签页还没切)。
|
|
1247
1828
|
if (DEVELOPER_MODE_ACTIONS.has(input.action)) {
|
|
1248
1829
|
const originError = requireAllowedDeveloperOrigin(
|
|
1249
|
-
await resolveLiveUrl(run) ?? stream.getCurrentUrl(),
|
|
1830
|
+
(await resolveLiveUrl(run)) ?? stream.getCurrentUrl(),
|
|
1250
1831
|
browserConfig.allowedCdpOrigins,
|
|
1251
1832
|
);
|
|
1252
1833
|
if (originError) return { blocked: originError };
|
|
1253
1834
|
}
|
|
1254
1835
|
|
|
1255
|
-
if (
|
|
1836
|
+
if (
|
|
1837
|
+
input.action === 'click' &&
|
|
1838
|
+
input.selector &&
|
|
1839
|
+
input.intent !== 'submit'
|
|
1840
|
+
) {
|
|
1256
1841
|
// The model is expected to mark final actions, but safety cannot depend
|
|
1257
1842
|
// solely on model compliance. Inspect the actual target's visible label
|
|
1258
1843
|
// and reject an obvious irreversible click until it is explicitly marked.
|
|
1259
1844
|
const target = await run(['get', 'text', input.selector]);
|
|
1260
1845
|
const targetLabel = `${input.selector}\n${target.code === 0 ? target.stdout : ''}`;
|
|
1261
1846
|
if (isLikelyIrreversibleBrowserTarget(targetLabel)) {
|
|
1262
|
-
return {
|
|
1847
|
+
return {
|
|
1848
|
+
blocked:
|
|
1849
|
+
'该控件看起来会执行发布、提交、购买、删除或发送操作。请仅在用户当前任务已明确授权时,使用 intent="submit" 重试。',
|
|
1850
|
+
};
|
|
1263
1851
|
}
|
|
1264
1852
|
}
|
|
1265
1853
|
|
|
@@ -1268,9 +1856,16 @@ async function call(
|
|
|
1268
1856
|
// get box 是本地 daemon 调用(非 LLM 往返);失败/找不到则不显示、动作照常执行(纯视觉增强,绝不阻断)。
|
|
1269
1857
|
// 仅受监督(Desktop)驱动:headless 无面板 → 不调用、零开销。
|
|
1270
1858
|
if (supervised && input.selector && POINTER_ACTIONS.has(input.action)) {
|
|
1271
|
-
const targetCenter = parsePointerCenter(
|
|
1859
|
+
const targetCenter = parsePointerCenter(
|
|
1860
|
+
await run(['get', 'box', input.selector, '--json']),
|
|
1861
|
+
);
|
|
1272
1862
|
if (targetCenter) {
|
|
1273
|
-
stream.setAgentPointer({
|
|
1863
|
+
stream.setAgentPointer({
|
|
1864
|
+
x: targetCenter.x,
|
|
1865
|
+
y: targetCenter.y,
|
|
1866
|
+
visible: true,
|
|
1867
|
+
action: pointerActionFor(input.action),
|
|
1868
|
+
});
|
|
1274
1869
|
flashTargetElement(stream, input.selector); // 点亮目标元素绿框,强化「AI 在点这里」
|
|
1275
1870
|
pointerShown = true;
|
|
1276
1871
|
}
|
|
@@ -1312,8 +1907,13 @@ async function call(
|
|
|
1312
1907
|
// 再采样纯属白付一次往返 + 一段等待。噪声是页面的持续属性,动作后测与
|
|
1313
1908
|
// 动作前测等价——而放在后面测,这段等待就不再推迟动作本身的发出。
|
|
1314
1909
|
let noiseAfter: string | undefined;
|
|
1315
|
-
if (
|
|
1316
|
-
&&
|
|
1910
|
+
if (
|
|
1911
|
+
verified &&
|
|
1912
|
+
result.code === 0 &&
|
|
1913
|
+
before !== undefined &&
|
|
1914
|
+
probeSnapshot &&
|
|
1915
|
+
differingLineIndexes(before, probeSnapshot)?.size
|
|
1916
|
+
) {
|
|
1317
1917
|
await sleep(NOISE_BASELINE_GAP_MS, signal);
|
|
1318
1918
|
const again = await run(['snapshot']);
|
|
1319
1919
|
if (again.code === 0) noiseAfter = again.stdout.trim();
|
|
@@ -1321,20 +1921,31 @@ async function call(
|
|
|
1321
1921
|
|
|
1322
1922
|
return { result, before, probeSnapshot, noiseAfter };
|
|
1323
1923
|
});
|
|
1324
|
-
if (outcome.blocked !== undefined)
|
|
1325
|
-
|
|
1924
|
+
if (outcome.blocked !== undefined)
|
|
1925
|
+
return { ok: false, error: outcome.blocked };
|
|
1926
|
+
const {
|
|
1927
|
+
result,
|
|
1928
|
+
before: beforeSnapshot,
|
|
1929
|
+
probeSnapshot,
|
|
1930
|
+
noiseAfter,
|
|
1931
|
+
} = outcome;
|
|
1326
1932
|
|
|
1327
1933
|
if (result.code === 0 && input.action === 'tabSwitch') {
|
|
1328
1934
|
browserContext.activeTabId = claimedTabId;
|
|
1329
1935
|
} else if (result.code === 0 && input.action === 'tabClose') {
|
|
1330
|
-
browserRunRegistry.releaseTab(
|
|
1936
|
+
browserRunRegistry.releaseTab(
|
|
1937
|
+
browserContext.conversationKey,
|
|
1938
|
+
claimedTabId,
|
|
1939
|
+
);
|
|
1331
1940
|
}
|
|
1332
1941
|
|
|
1333
1942
|
// 一次轻停(≤120ms)让点击落点/涟漪可见后再由 finally 淡出;不做更长的表演式停顿。
|
|
1334
|
-
if (supervised && pointerShown && !signal?.aborted)
|
|
1943
|
+
if (supervised && pointerShown && !signal?.aborted)
|
|
1944
|
+
await sleep(120, signal);
|
|
1335
1945
|
|
|
1336
1946
|
// 硬停止:整个 run 被父级 abort(底部 ■ / ESC / interrupt 队列)→ 立即返回,不进人工等待。
|
|
1337
|
-
if (signal?.aborted)
|
|
1947
|
+
if (signal?.aborted)
|
|
1948
|
+
return { ok: false, error: `[${input.action}] 浏览器任务已被中断` };
|
|
1338
1949
|
|
|
1339
1950
|
// URL 栏跟随 agent 导航(与 bridge 用户导航对齐,browserBridge.ts:228):用户手输网址会更新 URL 栏,
|
|
1340
1951
|
// agent 导航也应一致更新,否则面板「开着却网址栏空」。back/forward/reload 目标未知,靠媒体面连上后
|
|
@@ -1345,7 +1956,10 @@ async function call(
|
|
|
1345
1956
|
|
|
1346
1957
|
const challengeText = `${result.stdout}\n${result.stderr}\n${probeSnapshot}`;
|
|
1347
1958
|
|
|
1348
|
-
|
|
1959
|
+
// A failed click/navigation is a common way to discover a login, CAPTCHA,
|
|
1960
|
+
// or bot-check page. Do not gate detection on the CLI exit code or the
|
|
1961
|
+
// agent will retry the challenge as if it were an ordinary locator miss.
|
|
1962
|
+
const challenge = detectBrowserChallenge(challengeText);
|
|
1349
1963
|
if (challenge && !supervised) {
|
|
1350
1964
|
// 非受监督宿主(CLI -p / TUI / gateway)无可见浏览器、无人可接管:绝不阻塞等人。
|
|
1351
1965
|
// 拿一张现场快照,连同「需真人验证、当前环境无法越过」如实返回。用 ok:true(非 ok:false)
|
|
@@ -1354,25 +1968,31 @@ async function call(
|
|
|
1354
1968
|
const snapshot = probe.code === 0 ? probe.stdout.trim() : '';
|
|
1355
1969
|
return {
|
|
1356
1970
|
ok: true,
|
|
1357
|
-
content: `⚠️ ${challenge.summary}
|
|
1358
|
-
+ `请如实告知用户此页面需人工验证/登录后才能继续,不要反复重试或索要凭据。`
|
|
1359
|
-
+ (snapshot ? `\n\n当前页面快照:\n${snapshot}` : ''),
|
|
1971
|
+
content: `⚠️ ${challenge.summary}。当前为非交互环境(无可见浏览器供真人介入),无法自动越过该关卡。请如实告知用户此页面需人工验证/登录后才能继续,不要反复重试或索要凭据。${snapshot ? `\n\n当前页面快照:\n${snapshot}` : ''}`,
|
|
1360
1972
|
};
|
|
1361
1973
|
}
|
|
1362
1974
|
// challenge 自动暂停 与 用户点「接管」(requestHuman('manual'))都会把控制切到 waiting_human,
|
|
1363
1975
|
// 统一在此进入人工等待——接管已在 runAction 里即时掐断在跑动作,这里 result 可能是被掐断的部分结果。
|
|
1364
|
-
if (challenge)
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1976
|
+
if (challenge)
|
|
1977
|
+
stream.requestHuman(challenge.reason, {
|
|
1978
|
+
runId,
|
|
1979
|
+
conversationKey,
|
|
1980
|
+
ownerId: 'local-owner',
|
|
1981
|
+
summary: challenge.summary,
|
|
1982
|
+
});
|
|
1370
1983
|
if (supervised && stream.getControlState().status === 'waiting_human') {
|
|
1371
1984
|
// 交人处理前先淡出 AI 指针:控制权已交给用户,橙色指针不该再滞留(否则像 Agent 还在指)。
|
|
1372
|
-
if (pointerShown) {
|
|
1373
|
-
|
|
1985
|
+
if (pointerShown) {
|
|
1986
|
+
stream.setAgentPointer({ x: 0, y: 0, visible: false });
|
|
1987
|
+
}
|
|
1988
|
+
ctx.reportProgress?.(
|
|
1989
|
+
challenge
|
|
1990
|
+
? `${challenge.summary},等待你在浏览器中处理`
|
|
1991
|
+
: '你已接管浏览器,处理完成后点“完成并继续”',
|
|
1992
|
+
);
|
|
1374
1993
|
const resolution = await stream.waitForHuman(signal);
|
|
1375
|
-
if (resolution === 'stopped')
|
|
1994
|
+
if (resolution === 'stopped')
|
|
1995
|
+
return { ok: false, error: `[${input.action}] 浏览器任务已由用户停止` };
|
|
1376
1996
|
// resume 后控制权已回 running,runAction 取最新快照不会被误掐。
|
|
1377
1997
|
const refreshed = await runAction(['snapshot']);
|
|
1378
1998
|
const after = toToolResult('snapshot', refreshed);
|
|
@@ -1384,8 +2004,18 @@ async function call(
|
|
|
1384
2004
|
}
|
|
1385
2005
|
// ===== 成功率收口:失败要给现场 + 候选,成功要给「生效了没有」 =====
|
|
1386
2006
|
if (VERIFIED_BROWSER_ACTIONS.has(input.action)) {
|
|
1387
|
-
if (result.code === 0)
|
|
1388
|
-
|
|
2007
|
+
if (result.code === 0)
|
|
2008
|
+
browserAttempts.recordSuccess(
|
|
2009
|
+
conversationKey,
|
|
2010
|
+
input.action,
|
|
2011
|
+
input.selector,
|
|
2012
|
+
);
|
|
2013
|
+
else
|
|
2014
|
+
browserAttempts.recordFailure(
|
|
2015
|
+
conversationKey,
|
|
2016
|
+
input.action,
|
|
2017
|
+
input.selector,
|
|
2018
|
+
);
|
|
1389
2019
|
}
|
|
1390
2020
|
|
|
1391
2021
|
if (result.code !== 0) {
|
|
@@ -1405,23 +2035,84 @@ async function call(
|
|
|
1405
2035
|
);
|
|
1406
2036
|
}
|
|
1407
2037
|
const hint = candidates.length
|
|
1408
|
-
?
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
+ candidates.map((c) => ` ${c.ref} ← ${c.line}`).join('\n')
|
|
2038
|
+
? `\n\n【最接近的候选元素】直接用下面的 @eN 引用重试,不要再猜选择器:\n${candidates
|
|
2039
|
+
.map((c) => ` ${c.ref} ← ${c.line}`)
|
|
2040
|
+
.join('\n')}`
|
|
1412
2041
|
: '';
|
|
1413
2042
|
return {
|
|
1414
2043
|
...base,
|
|
1415
2044
|
// 失败回执也走 clampResult:截断要说出来。以前这里是裸 `.slice()`,
|
|
1416
2045
|
// 于是「后面还有内容」这件事只在成功路径告知,而失败恰恰是最需要知道的时候。
|
|
1417
|
-
error: clampResult(
|
|
2046
|
+
error: clampResult(
|
|
2047
|
+
`${base.error}${hint}\n\n【当前页面快照 @eN】\n${budgetSnapshot(probeSnapshot)}`,
|
|
2048
|
+
),
|
|
1418
2049
|
};
|
|
1419
2050
|
}
|
|
1420
2051
|
return base;
|
|
1421
2052
|
}
|
|
1422
2053
|
|
|
2054
|
+
// WebBrowser's legacy page-assets action used to be only an eval alias.
|
|
2055
|
+
// Materialize the requested bundle at this boundary so the advertised
|
|
2056
|
+
// action actually creates `manifest.json` and asset files.
|
|
2057
|
+
if (input.action === 'bundlePageAssets') {
|
|
2058
|
+
const parsed = parseEvalJson(result.stdout);
|
|
2059
|
+
if (!Array.isArray(parsed)) {
|
|
2060
|
+
return {
|
|
2061
|
+
ok: false,
|
|
2062
|
+
error:
|
|
2063
|
+
'页面资产返回格式无效,无法打包。请先使用 pageAssets 查看页面资源。',
|
|
2064
|
+
};
|
|
2065
|
+
}
|
|
2066
|
+
try {
|
|
2067
|
+
const bundle = await bundlePageAssets(
|
|
2068
|
+
parsed as BrowserPageAsset[],
|
|
2069
|
+
input.bundleDir!,
|
|
2070
|
+
async (url) => {
|
|
2071
|
+
const fetched = parseEvalJson(
|
|
2072
|
+
(await runAction(['eval', browserAssetFetchExpression(url)]))
|
|
2073
|
+
.stdout,
|
|
2074
|
+
);
|
|
2075
|
+
if (
|
|
2076
|
+
!fetched ||
|
|
2077
|
+
typeof fetched !== 'object' ||
|
|
2078
|
+
(fetched as { ok?: unknown }).ok !== true ||
|
|
2079
|
+
typeof (fetched as { base64?: unknown }).base64 !== 'string'
|
|
2080
|
+
) {
|
|
2081
|
+
const status =
|
|
2082
|
+
fetched && typeof fetched === 'object'
|
|
2083
|
+
? String(
|
|
2084
|
+
(fetched as { status?: unknown }).status ?? 'unknown',
|
|
2085
|
+
)
|
|
2086
|
+
: 'unknown';
|
|
2087
|
+
throw new Error(`浏览器上下文抓取资源失败 HTTP ${status}`);
|
|
2088
|
+
}
|
|
2089
|
+
const value = fetched as { base64: string; type?: string };
|
|
2090
|
+
return new Response(Buffer.from(value.base64, 'base64'), {
|
|
2091
|
+
status: 200,
|
|
2092
|
+
headers: value.type ? { 'content-type': value.type } : undefined,
|
|
2093
|
+
});
|
|
2094
|
+
},
|
|
2095
|
+
);
|
|
2096
|
+
return {
|
|
2097
|
+
ok: true,
|
|
2098
|
+
content: clampResult(
|
|
2099
|
+
JSON.stringify({ assets: parsed, bundle }, null, 2),
|
|
2100
|
+
),
|
|
2101
|
+
};
|
|
2102
|
+
} catch (error) {
|
|
2103
|
+
return {
|
|
2104
|
+
ok: false,
|
|
2105
|
+
error: `页面资产打包失败:${error instanceof Error ? error.message : String(error)}`,
|
|
2106
|
+
};
|
|
2107
|
+
}
|
|
2108
|
+
}
|
|
2109
|
+
|
|
1423
2110
|
const verdict = VERIFIED_BROWSER_ACTIONS.has(input.action)
|
|
1424
|
-
? classifyPageEffect(
|
|
2111
|
+
? classifyPageEffect(
|
|
2112
|
+
beforeSnapshot,
|
|
2113
|
+
probeSnapshot || undefined,
|
|
2114
|
+
noiseAfter,
|
|
2115
|
+
)
|
|
1425
2116
|
: null;
|
|
1426
2117
|
// 面板也要看得见:验证结论只进对话的话,盯着浏览器的人根本不知道刚那下没生效。
|
|
1427
2118
|
// 走**自己的**帧而不是借 agent_acting——后者在 finally 里被无条件置回 false,
|
|
@@ -1445,38 +2136,62 @@ async function call(
|
|
|
1445
2136
|
// 关键节点(navigate 加载完成 / snapshot 读取页面)确定性截当前可见画面,作为「地面真相」附到结果上,
|
|
1446
2137
|
// 由主循环回填成一条带图 user 消息喂视觉模型自我核验。仅基座支持识图时截(否则跳过,不浪费 I/O、不假装核验);
|
|
1447
2138
|
// 截图失败绝不影响原动作结果(纯增强)。
|
|
1448
|
-
if (
|
|
1449
|
-
|
|
2139
|
+
if (
|
|
2140
|
+
finalResult.ok &&
|
|
2141
|
+
KEY_NODE_SHOT_ACTIONS.has(input.action) &&
|
|
2142
|
+
ctx.provider?.supportsVision === true &&
|
|
2143
|
+
!signal?.aborted
|
|
2144
|
+
) {
|
|
1450
2145
|
try {
|
|
1451
2146
|
const shotPath = await createArtifactPath('browser-verify', 'png');
|
|
1452
2147
|
const shot = await runAction(['screenshot', shotPath]);
|
|
1453
2148
|
if (shot.code === 0) {
|
|
1454
|
-
const shots: ImageInputSpec[] = [
|
|
2149
|
+
const shots: ImageInputSpec[] = [
|
|
2150
|
+
{ source: shotPath, detail: 'low', maxLongSidePixel: 1400 },
|
|
2151
|
+
];
|
|
1455
2152
|
return {
|
|
1456
2153
|
...finalResult,
|
|
1457
|
-
content: finalResult.content
|
|
1458
|
-
+ '\n\n[已在关键节点截取当前可见页面,见随后的实况画面——请对照核对是否真的到达/呈现了预期状态,不要假设成功。]',
|
|
2154
|
+
content: `${finalResult.content}\n\n[已在关键节点截取当前可见页面,见随后的实况画面——请对照核对是否真的到达/呈现了预期状态,不要假设成功。]`,
|
|
1459
2155
|
visionImages: shots,
|
|
1460
2156
|
};
|
|
1461
2157
|
}
|
|
1462
|
-
} catch {
|
|
2158
|
+
} catch {
|
|
2159
|
+
/* 截图纯增强,失败不影响动作结果 */
|
|
2160
|
+
}
|
|
1463
2161
|
}
|
|
1464
2162
|
return finalResult;
|
|
1465
2163
|
} catch (error) {
|
|
1466
|
-
if (signal?.aborted)
|
|
2164
|
+
if (signal?.aborted)
|
|
2165
|
+
return { ok: false, error: `[${input.action}] 浏览器任务已被中断` };
|
|
1467
2166
|
if (error instanceof BrowserTabError) {
|
|
1468
|
-
return {
|
|
2167
|
+
return {
|
|
2168
|
+
ok: false,
|
|
2169
|
+
error: `[${input.action}] ${error.code}: ${error.message}`,
|
|
2170
|
+
};
|
|
1469
2171
|
}
|
|
1470
2172
|
return {
|
|
1471
2173
|
ok: false,
|
|
1472
2174
|
error: `[${input.action}] ${error instanceof Error ? error.message : String(error)}`,
|
|
1473
2175
|
};
|
|
1474
2176
|
} finally {
|
|
1475
|
-
|
|
1476
|
-
|
|
2177
|
+
// ⚠️ 释放排在**第一位**,且下面两个装饰动作各自吞掉异常。
|
|
2178
|
+
// 它们都会 emit 到宿主注入的 listener(Desktop 面板 / server 推流),listener 一抛错
|
|
2179
|
+
// 就会吃掉它后面的所有语句 —— 排在后面的释放因此永远不执行,锁泄漏。
|
|
2180
|
+
// 一把进程级单例锁的命运不该由 UI 增强的成败决定;反过来,两个装饰之间也不该互相牵连。
|
|
1477
2181
|
stream.finishAgentAction(runId);
|
|
2182
|
+
try {
|
|
2183
|
+
if (visibleAgentAction) stream.setAgentActing(false);
|
|
2184
|
+
} catch {
|
|
2185
|
+
/* 面板装饰,失败不影响动作结果,更不该影响锁 */
|
|
2186
|
+
}
|
|
2187
|
+
try {
|
|
2188
|
+
if (pointerShown) stream.setAgentPointer({ x: 0, y: 0, visible: false }); // C4:动作结束淡出指针
|
|
2189
|
+
} catch {
|
|
2190
|
+
/* 同上 */
|
|
2191
|
+
}
|
|
1478
2192
|
if (preTimer) clearTimeout(preTimer);
|
|
1479
2193
|
signal?.removeEventListener('abort', abortPreFromParent);
|
|
2194
|
+
browserRunRegistry.finish(conversationKey, runId);
|
|
1480
2195
|
}
|
|
1481
2196
|
}
|
|
1482
2197
|
|
|
@@ -1489,11 +2204,31 @@ function appendEffectNote(base: ToolResult, note: string | null): ToolResult {
|
|
|
1489
2204
|
/** 把工具输出收进结果预算,超出时如实说明被截断(而不是悄悄截)。 */
|
|
1490
2205
|
function clampResult(text: string): string {
|
|
1491
2206
|
if (text.length <= DEFAULT_MAX_RESULT_SIZE_CHARS) return text;
|
|
1492
|
-
return (
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
2207
|
+
return `${text.slice(0, DEFAULT_MAX_RESULT_SIZE_CHARS)}\n\n…(输出超过 ${DEFAULT_MAX_RESULT_SIZE_CHARS} 字符已截断;需要完整内容请缩小选择器范围或分段读取)`;
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
/**
|
|
2211
|
+
* agent-browser returns the value of `eval` as JSON in current releases, but
|
|
2212
|
+
* older versions wrapped JSON strings in a quoted envelope. Keep this
|
|
2213
|
+
* compatibility parsing at the adapter boundary so page-assets remains a
|
|
2214
|
+
* real capability in WebBrowser rather than an alias that only lists assets.
|
|
2215
|
+
*/
|
|
2216
|
+
function parseEvalJson(stdout: string): unknown {
|
|
2217
|
+
const trimmed = stdout.trim();
|
|
2218
|
+
if (!trimmed) return undefined;
|
|
2219
|
+
try {
|
|
2220
|
+
const value = JSON.parse(trimmed) as unknown;
|
|
2221
|
+
if (typeof value === 'string') {
|
|
2222
|
+
try {
|
|
2223
|
+
return JSON.parse(value) as unknown;
|
|
2224
|
+
} catch {
|
|
2225
|
+
return value;
|
|
2226
|
+
}
|
|
2227
|
+
}
|
|
2228
|
+
return value;
|
|
2229
|
+
} catch {
|
|
2230
|
+
return undefined;
|
|
2231
|
+
}
|
|
1497
2232
|
}
|
|
1498
2233
|
|
|
1499
2234
|
function toToolResult(
|
|
@@ -1508,7 +2243,10 @@ function toToolResult(
|
|
|
1508
2243
|
// 错误路径截了、withPostActionSnapshot 截了,唯独最常走的这条没截。
|
|
1509
2244
|
const body = clampResult(r.stdout.trim());
|
|
1510
2245
|
if (outPath) {
|
|
1511
|
-
return {
|
|
2246
|
+
return {
|
|
2247
|
+
ok: true,
|
|
2248
|
+
content: body ? `${body}\nSaved: ${outPath}` : `Saved: ${outPath}`,
|
|
2249
|
+
};
|
|
1512
2250
|
}
|
|
1513
2251
|
if (action === 'cookiesGet') {
|
|
1514
2252
|
return { ok: true, content: body || '[]' };
|
|
@@ -1534,7 +2272,8 @@ export const webbrowserTool: Tool<WebBrowserInput> = {
|
|
|
1534
2272
|
isConcurrencySafe: false,
|
|
1535
2273
|
maxResultSizeChars: DEFAULT_MAX_RESULT_SIZE_CHARS,
|
|
1536
2274
|
call,
|
|
1537
|
-
callWithContext: (input, signal, context) =>
|
|
2275
|
+
callWithContext: (input, signal, context) =>
|
|
2276
|
+
call(input, signal, defaultDeps, context),
|
|
1538
2277
|
};
|
|
1539
2278
|
|
|
1540
2279
|
export const _webbrowserCall = call;
|