minimal-agent 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/package.json +60 -4
- package/plugins/computer_use/.claude-plugin/plugin.json +1 -1
- package/plugins/cron-runner/src/commands.js +4 -4
- package/plugins/cron-runner/src/commands.ts +4 -4
- package/plugins/workflow-runner/.claude-plugin/plugin.json +2 -2
- package/plugins/workflow-runner/commands/workflow.md +57 -90
- package/plugins/workflow-runner/commands/workflows.md +4 -4
- package/plugins/workflow-runner/plugin.js +24 -13
- package/plugins/workflow-runner/plugin.ts +28 -13
- package/plugins/workflow-runner/src/discovery.js +160 -0
- package/plugins/workflow-runner/src/discovery.ts +194 -0
- package/plugins/workflow-runner/src/goalAnchor.js +53 -0
- package/plugins/workflow-runner/src/goalAnchor.ts +59 -0
- package/plugins/workflow-runner/src/index.js +197 -206
- package/plugins/workflow-runner/src/index.ts +248 -249
- package/plugins/workflow-runner/src/inputCoercion.js +40 -0
- package/plugins/workflow-runner/src/inputCoercion.ts +43 -0
- package/plugins/workflow-runner/src/journal.js +224 -0
- package/plugins/workflow-runner/src/journal.ts +274 -0
- package/plugins/workflow-runner/src/jsonSchema.js +132 -0
- package/plugins/workflow-runner/src/jsonSchema.ts +162 -0
- package/plugins/workflow-runner/src/moduleResolution.js +64 -0
- package/plugins/workflow-runner/src/moduleResolution.ts +71 -0
- package/plugins/workflow-runner/src/resourceInputs.js +220 -0
- package/plugins/workflow-runner/src/resourceInputs.ts +300 -0
- package/plugins/workflow-runner/src/runner.js +400 -489
- package/plugins/workflow-runner/src/runner.ts +488 -560
- package/plugins/workflow-runner/src/runtime/config.js +46 -0
- package/plugins/workflow-runner/src/runtime/config.ts +69 -0
- package/plugins/workflow-runner/src/runtime/context.js +1304 -0
- package/plugins/workflow-runner/src/runtime/context.ts +1786 -0
- package/plugins/workflow-runner/src/runtime/exec.js +34 -0
- package/plugins/workflow-runner/src/runtime/exec.ts +53 -0
- package/plugins/workflow-runner/src/runtime/flow.js +8 -0
- package/plugins/workflow-runner/src/runtime/flow.ts +12 -0
- package/plugins/workflow-runner/src/runtime/fs.js +27 -0
- package/plugins/workflow-runner/src/runtime/fs.ts +41 -0
- package/plugins/workflow-runner/src/runtime/model.js +6 -0
- package/plugins/workflow-runner/src/runtime/model.ts +8 -0
- package/plugins/workflow-runner/src/runtime/output.js +7 -0
- package/plugins/workflow-runner/src/runtime/output.ts +21 -0
- package/plugins/workflow-runner/src/runtime/scheduler.js +141 -0
- package/plugins/workflow-runner/src/runtime/scheduler.ts +166 -0
- package/plugins/workflow-runner/src/runtime/state.js +16 -0
- package/plugins/workflow-runner/src/runtime/state.ts +19 -0
- package/plugins/workflow-runner/src/runtime/tools.js +3 -0
- package/plugins/workflow-runner/src/runtime/tools.ts +6 -0
- package/plugins/workflow-runner/src/sandbox.js +141 -0
- package/plugins/workflow-runner/src/sandbox.ts +182 -0
- package/plugins/workflow-runner/src/scriptGuard.d.mts +38 -0
- package/plugins/workflow-runner/src/scriptGuard.mjs +449 -0
- package/plugins/workflow-runner/src/taskIdentity.js +151 -307
- package/plugins/workflow-runner/src/taskIdentity.ts +207 -355
- package/plugins/workflow-runner/src/toolRuntime.js +29 -8
- package/plugins/workflow-runner/src/toolRuntime.ts +36 -11
- package/plugins/workflow-runner/src/types.js +7 -211
- package/plugins/workflow-runner/src/types.ts +154 -623
- package/plugins/workflow-runner/src/util.js +47 -0
- package/plugins/workflow-runner/src/util.ts +51 -0
- package/plugins/workflow-runner/src/workflowContext.js +20 -0
- package/plugins/workflow-runner/src/workflowContext.ts +58 -0
- package/plugins/workflow-runner/src/workflowDependencies.js +36 -0
- package/plugins/workflow-runner/src/workflowDependencies.ts +59 -0
- package/plugins/workflow-runner/src/workflowFingerprint.js +118 -0
- package/plugins/workflow-runner/src/workflowFingerprint.ts +122 -0
- package/plugins/workflow-runner/src/workflowState.js +474 -802
- package/plugins/workflow-runner/src/workflowState.ts +575 -970
- package/plugins/workflow-runner/test/api.test.ts +223 -0
- package/plugins/workflow-runner/test/apiCapabilities.test.ts +614 -0
- package/plugins/workflow-runner/test/apiConcurrencyGate.test.ts +104 -0
- package/plugins/workflow-runner/test/apiEcho.test.ts +210 -0
- package/plugins/workflow-runner/test/apiReadText.test.ts +178 -0
- package/plugins/workflow-runner/test/apiResourceLedger.test.ts +184 -0
- package/plugins/workflow-runner/test/childWorkflow.test.ts +304 -0
- package/plugins/workflow-runner/test/discovery.test.ts +234 -0
- package/plugins/workflow-runner/test/goalAnchor.test.ts +106 -0
- package/plugins/workflow-runner/test/goalController.test.ts +317 -0
- package/plugins/workflow-runner/test/journalLanes.test.ts +198 -0
- package/plugins/workflow-runner/test/listResources.test.ts +106 -0
- package/plugins/workflow-runner/test/resourceInputs.test.ts +294 -0
- package/plugins/workflow-runner/test/sandbox.test.ts +126 -0
- package/plugins/workflow-runner/test/scheduler.test.ts +70 -0
- package/plugins/workflow-runner/test/scriptGuard.test.ts +276 -0
- package/plugins/workflow-runner/test/skillDocSync.test.ts +78 -0
- package/plugins/workflow-runner/test/taskIdentity.test.ts +288 -179
- package/plugins/workflow-runner/test/workflowFingerprint.test.ts +106 -0
- package/plugins/workflow-runner/test/workflowState.test.ts +429 -0
- package/skills/pptx/examples/assets/cover-bg.svg +23 -0
- package/skills/pptx/examples/assets/icon-spark.svg +5 -0
- package/skills/ui-design/SKILL.md +5 -2
- package/skills/ui-design/references/layout-composition.md +1 -1
- package/skills/workflow-creator/SKILL.md +344 -0
- package/skills/workflow-creator/assets/template/workflow.mjs +48 -0
- package/skills/workflow-creator/references/api.md +216 -0
- package/skills/workflow-creator/references/migration.md +103 -0
- package/skills/workflow-creator/references/patterns.md +779 -0
- package/skills/workflow-creator/scripts/list-resources.mjs +205 -0
- package/skills/workflow-creator/scripts/scaffold.mjs +148 -0
- package/skills/workflow-creator/scripts/validate.mjs +275 -0
- package/skills/workflow-creator/scripts/workflow-creator.test.mjs +82 -0
- package/src/advisor/advisorCoordinator.js +536 -0
- package/src/advisor/advisorCoordinator.ts +693 -0
- package/src/cli/configMenu.tsx +0 -1
- package/src/cli/print.js +27 -6
- package/src/cli/print.ts +48 -4
- package/src/cli/quickModelSetup.js +12 -1
- package/src/cli/quickModelSetup.tsx +15 -3
- package/src/cli/streamJson.js +5 -6
- package/src/cli/streamJson.ts +8 -4
- package/src/cli/wizards/advisorWizard.js +4 -2
- package/src/cli/wizards/advisorWizard.tsx +8 -2
- package/src/cli/wizards/contextAdvisorWizard.js +5 -4
- package/src/cli/wizards/contextAdvisorWizard.tsx +7 -6
- package/src/cli/wizards/providerPresets.js +18 -8
- package/src/cli/wizards/providerPresets.ts +18 -8
- package/src/collaboration/taskFlow.js +115 -0
- package/src/collaboration/taskFlow.ts +169 -0
- package/src/config/configDefaults.js +41 -1
- package/src/config/configDefaults.ts +43 -1
- package/src/config/configDiagnostics.ts +2 -2
- package/src/config/configFile.js +40 -9
- package/src/config/configFile.ts +60 -11
- package/src/config/configService.js +419 -74
- package/src/config/configService.ts +703 -145
- package/src/config/numericLimits.js +22 -0
- package/src/config/numericLimits.ts +24 -0
- package/src/config.js +1 -0
- package/src/config.ts +3 -2
- package/src/context/budgetPlan.js +1 -1
- package/src/context/budgetPlan.ts +1 -1
- package/src/context/compact.js +7 -2
- package/src/context/compact.ts +9 -2
- package/src/context/microCompactLite.js +212 -13
- package/src/context/microCompactLite.ts +261 -12
- package/src/context/sessionRegistry.js +8 -1
- package/src/context/sessionRegistry.ts +18 -5
- package/src/context/toolResultArchive.js +72 -0
- package/src/context/toolResultArchive.ts +95 -0
- package/src/goal/errors.js +8 -0
- package/src/goal/errors.ts +18 -0
- package/src/goal/goalRuntime.js +344 -0
- package/src/goal/goalRuntime.ts +466 -0
- package/src/goal/goalStore.js +200 -0
- package/src/goal/goalStore.ts +219 -0
- package/src/goal/index.js +4 -0
- package/src/goal/index.ts +4 -0
- package/src/goal/statusMapping.js +71 -0
- package/src/goal/statusMapping.ts +95 -0
- package/src/goal/types.js +7 -0
- package/src/goal/types.ts +179 -0
- package/src/goal/workflowController.js +306 -0
- package/src/goal/workflowController.ts +360 -0
- package/src/harness/evidenceLedger.js +2 -0
- package/src/harness/evidenceLedger.ts +1 -0
- package/src/harness/stallGuard.js +9 -3
- package/src/harness/stallGuard.ts +7 -3
- package/src/images/imageResolver.js +161 -0
- package/src/images/imageResolver.ts +189 -0
- package/src/infra/verification.js +34 -5
- package/src/infra/verification.ts +39 -4
- package/src/llm/client.js +79 -21
- package/src/llm/client.ts +182 -35
- package/src/llm/gatewayClient.js +201 -0
- package/src/llm/gatewayClient.ts +299 -0
- package/src/llm/messageProvenance.js +46 -0
- package/src/llm/messageProvenance.ts +58 -0
- package/src/llm/modelProvider.js +1 -0
- package/src/llm/modelProvider.ts +22 -0
- package/src/llm/providers/anthropicFormat.js +108 -1
- package/src/llm/providers/anthropicFormat.ts +115 -4
- package/src/llm/providers/openaiFormat.js +9 -0
- package/src/llm/providers/openaiFormat.ts +12 -3
- package/src/loop.js +328 -55
- package/src/loop.ts +382 -58
- package/src/main.js +4 -1
- package/src/main.tsx +4 -1
- package/src/managedExecution/index.js +2 -0
- package/src/managedExecution/index.ts +2 -0
- package/src/managedExecution/registry.js +15 -0
- package/src/managedExecution/registry.ts +20 -0
- package/src/managedExecution/types.js +1 -0
- package/src/managedExecution/types.ts +86 -0
- package/src/modelRef.js +9 -0
- package/src/modelRef.ts +31 -0
- package/src/multimodal.js +14 -28
- package/src/multimodal.ts +17 -33
- package/src/net/lanDiscovery.js +65 -0
- package/src/net/lanDiscovery.ts +82 -0
- package/src/plugin-sdk.js +28 -3
- package/src/plugin-sdk.ts +39 -6
- package/src/plugins/pluginApi.ts +16 -1
- package/src/plugins/pluginLoader.js +4 -0
- package/src/plugins/pluginLoader.ts +6 -0
- package/src/plugins/pluginRunner.js +139 -7
- package/src/plugins/pluginRunner.ts +173 -9
- package/src/plugins/slashFallback.js +113 -0
- package/src/plugins/slashFallback.ts +137 -0
- package/src/prompts/policyRegistry.js +12 -32
- package/src/prompts/policyRegistry.ts +15 -32
- package/src/prompts/promptAssembly.js +103 -12
- package/src/prompts/promptAssembly.ts +114 -12
- package/src/prompts/skillList.js +21 -3
- package/src/prompts/skillList.ts +27 -2
- package/src/prompts/stableSystemPrompt.js +18 -0
- package/src/prompts/stableSystemPrompt.ts +33 -0
- package/src/runtime/agentRuntimeContext.js +8 -2
- package/src/runtime/agentRuntimeContext.ts +17 -4
- package/src/session/runAgentTurn.js +78 -12
- package/src/session/runAgentTurn.ts +105 -23
- package/src/session/runTurn.js +10 -9
- package/src/session/runTurn.ts +17 -8
- package/src/session/sessionState.ts +2 -0
- package/src/team/auth.js +186 -0
- package/src/team/auth.ts +298 -0
- package/src/team/cli.js +105 -0
- package/src/team/cli.ts +122 -0
- package/src/team/onlineToolContract.js +64 -0
- package/src/team/onlineToolContract.ts +109 -0
- package/src/team/toolPolicy.js +36 -0
- package/src/team/toolPolicy.ts +75 -0
- package/src/tools/advisor/advisor.js +115 -50
- package/src/tools/advisor/advisor.ts +124 -51
- package/src/tools/advisor/advisorDecision.js +71 -0
- package/src/tools/advisor/advisorDecision.ts +85 -0
- package/src/tools/agent/agent.js +18 -5
- package/src/tools/agent/agent.ts +21 -7
- package/src/tools/agent/agentControl.js +246 -0
- package/src/tools/agent/agentControl.ts +284 -0
- package/src/tools/agent/agentTaskService.js +40 -1
- package/src/tools/agent/agentTaskService.ts +44 -1
- package/src/tools/agent/agentTypes.js +14 -2
- package/src/tools/agent/agentTypes.ts +15 -2
- package/src/tools/applypatch/applyPatch.js +362 -0
- package/src/tools/applypatch/applyPatch.ts +428 -0
- package/src/tools/bash/bash.js +22 -15
- package/src/tools/bash/bash.ts +23 -17
- package/src/tools/capabilities.js +10 -5
- package/src/tools/capabilities.ts +10 -5
- package/src/tools/computeruse/computerIntent.js +32 -0
- package/src/tools/computeruse/computerIntent.ts +46 -0
- package/src/tools/computeruse/telemetryLog.js +12 -0
- package/src/tools/computeruse/telemetryLog.ts +16 -0
- package/src/tools/computeruse/vision/telemetry.js +1 -0
- package/src/tools/computeruse/vision/telemetry.ts +1 -0
- package/src/tools/effectiveToolSet.js +4 -0
- package/src/tools/effectiveToolSet.ts +4 -0
- package/src/tools/index.js +21 -2
- package/src/tools/index.ts +33 -2
- package/src/tools/presentresult/presentresult.js +6 -15
- package/src/tools/presentresult/presentresult.ts +6 -15
- package/src/tools/read/read.js +62 -9
- package/src/tools/read/read.ts +63 -9
- package/src/tools/shared/fileUtils.js +1 -2
- package/src/tools/shared/fileUtils.ts +1 -2
- package/src/tools/shared/toolOutputStore.js +23 -1
- package/src/tools/shared/toolOutputStore.ts +22 -1
- package/src/tools/skill/skill.js +9 -3
- package/src/tools/skill/skill.ts +8 -4
- package/src/tools/toolPromptGuides.js +125 -0
- package/src/tools/toolPromptGuides.ts +146 -0
- package/src/tools/webbrowser/agentBrowserPath.js +19 -9
- package/src/tools/webbrowser/agentBrowserPath.ts +70 -28
- package/src/tools/webbrowser/browser.js +27 -11
- package/src/tools/webbrowser/browser.ts +68 -24
- package/src/tools/webbrowser/browserAutomation.js +329 -0
- package/src/tools/webbrowser/browserAutomation.ts +467 -0
- package/src/tools/webbrowser/browserAxState.js +97 -0
- package/src/tools/webbrowser/browserAxState.ts +142 -0
- package/src/tools/webbrowser/browserBatch.js +1578 -0
- package/src/tools/webbrowser/browserBatch.ts +2382 -0
- package/src/tools/webbrowser/browserCdp.js +208 -0
- package/src/tools/webbrowser/browserCdp.ts +255 -0
- package/src/tools/webbrowser/browserConditions.js +24 -0
- package/src/tools/webbrowser/browserConditions.ts +45 -0
- package/src/tools/webbrowser/browserContent.js +281 -0
- package/src/tools/webbrowser/browserContent.ts +438 -0
- package/src/tools/webbrowser/browserControl.js +131 -34
- package/src/tools/webbrowser/browserControl.ts +210 -56
- package/src/tools/webbrowser/browserDaemon.js +22 -14
- package/src/tools/webbrowser/browserDaemon.ts +54 -21
- package/src/tools/webbrowser/browserFrames.js +47 -0
- package/src/tools/webbrowser/browserFrames.ts +54 -0
- package/src/tools/webbrowser/browserInteractionResolver.js +17 -0
- package/src/tools/webbrowser/browserInteractionResolver.ts +40 -0
- package/src/tools/webbrowser/browserLocator.js +140 -0
- package/src/tools/webbrowser/browserLocator.ts +194 -0
- package/src/tools/webbrowser/browserOutcome.js +10 -17
- package/src/tools/webbrowser/browserOutcome.ts +48 -31
- package/src/tools/webbrowser/browserRecovery.js +58 -0
- package/src/tools/webbrowser/browserRecovery.ts +83 -0
- package/src/tools/webbrowser/browserRunRegistry.js +35 -3
- package/src/tools/webbrowser/browserRunRegistry.ts +91 -15
- package/src/tools/webbrowser/browserService.js +28 -2
- package/src/tools/webbrowser/browserService.ts +49 -9
- package/src/tools/webbrowser/browserStream.js +132 -33
- package/src/tools/webbrowser/browserStream.ts +224 -50
- package/src/tools/webbrowser/browserTabs.js +48 -15
- package/src/tools/webbrowser/browserTabs.ts +178 -96
- package/src/tools/webbrowser/webbrowser.js +679 -144
- package/src/tools/webbrowser/webbrowser.ts +975 -236
- package/src/tools/webfetch/preapproved.js +0 -2
- package/src/tools/webfetch/preapproved.ts +0 -2
- package/src/tools/websearch/websearch.js +129 -8
- package/src/tools/websearch/websearch.ts +138 -6
- package/src/types.ts +459 -40
- package/src/ui/App.js +377 -58
- package/src/ui/App.tsx +576 -111
- package/src/ui/LiveArea.js +14 -8
- package/src/ui/LiveArea.tsx +14 -10
- package/src/ui/MessageList.js +63 -20
- package/src/ui/MessageList.tsx +101 -16
- package/src/ui/ModelPicker.js +7 -4
- package/src/ui/ModelPicker.tsx +55 -8
- package/src/ui/hooks/useChat.js +152 -163
- package/src/ui/hooks/useChat.ts +396 -334
- package/src/ui/loginCommand.js +56 -0
- package/src/ui/loginCommand.ts +63 -0
- package/src/ui/toolProgress.js +41 -0
- package/src/ui/toolProgress.ts +44 -0
- package/src/web/browserBridge.js +20 -2
- package/src/web/browserBridge.ts +20 -2
- package/src/web/protocol.ts +1897 -194
- package/workflows/adaptive-research/workflow.mjs +86 -0
- package/workflows/e2e-capabilities/workflow.mjs +102 -0
- package/workflows/e2e-llm/workflow.mjs +52 -0
- package/workflows/e2e-smoke/workflow.mjs +59 -0
- package/workflows/e2e-vision-schema/README.md +48 -0
- package/workflows/e2e-vision-schema/workflow.mjs +149 -0
- package/workflows/e2e-write-greet/workflow.mjs +37 -0
- package/workflows/file-census/README.md +14 -0
- package/workflows/file-census/scripts/.gitkeep +2 -0
- package/workflows/file-census/workflow.mjs +110 -0
- package/workflows/goal/README.md +166 -0
- package/workflows/goal/prompts/audit.md +57 -0
- package/workflows/goal/prompts/contract.md +56 -0
- package/workflows/goal/prompts/iterate.md +37 -0
- package/workflows/goal/workflow.mjs +501 -0
- package/workflows/illustrate/README.md +90 -0
- package/workflows/illustrate/assets/engines.json +20 -0
- package/workflows/illustrate/assets/extract-baidu.js +21 -0
- package/workflows/illustrate/assets/extract-bing.js +33 -0
- package/workflows/illustrate/assets/extract-google.js +31 -0
- package/workflows/illustrate/workflow.mjs +301 -0
- package/workflows/plan/workflow.mjs +389 -0
- package/workflows/question-crops/README.md +35 -0
- package/workflows/question-crops/prompts/inventory.md +23 -0
- package/workflows/question-crops/scripts/image-ops.mjs +305 -0
- package/workflows/question-crops/scripts/image-ops.test.mjs +95 -0
- package/workflows/question-crops/workflow.mjs +172 -0
- package/workflows/ralph-loop/workflow.mjs +162 -0
- package/workflows/research/README.md +74 -0
- package/workflows/research/prompts/s0-goal.md +43 -0
- package/workflows/research/prompts/s1-early-exit.md +22 -0
- package/workflows/research/prompts/s1-map.md +58 -0
- package/workflows/research/prompts/s2-options.md +53 -0
- package/workflows/research/prompts/s3-deep-dive.md +56 -0
- package/workflows/research/prompts/s4-verify.md +74 -0
- package/workflows/research/prompts/s5-synthesis.md +81 -0
- package/workflows/research/prompts/s5b-fill-gaps.md +32 -0
- package/workflows/research/prompts/s6-package.md +32 -0
- package/workflows/research/scripts/build-report.mjs +98 -0
- package/workflows/research/workflow.mjs +288 -0
- package/workflows/wechat-reading-shots/prompts/collect-params.md +16 -0
- package/workflows/wechat-reading-shots/prompts/evaluate.md +20 -0
- package/workflows/wechat-reading-shots/prompts/open-book.md +28 -0
- package/workflows/wechat-reading-shots/prompts/read-page.md +13 -0
- package/workflows/wechat-reading-shots/prompts/turn-batch.md +24 -0
- package/workflows/wechat-reading-shots/test/control-flow.test.ts +329 -0
- package/workflows/wechat-reading-shots/workflow.mjs +266 -0
- package/workflows/wrong-question-docx/README.md +55 -0
- package/workflows/wrong-question-docx/prompts/figure-qa.md +8 -0
- package/workflows/wrong-question-docx/prompts/transcribe.md +25 -0
- package/workflows/wrong-question-docx/scripts/workbook.mjs +857 -0
- package/workflows/wrong-question-docx/test/control-flow.test.ts +165 -0
- package/workflows/wrong-question-docx/test/workbook.test.mjs +303 -0
- package/workflows/wrong-question-docx/workflow.mjs +485 -0
- package/plugins/workflow-runner/src/agentCommon.js +0 -354
- package/plugins/workflow-runner/src/agentCommon.ts +0 -435
- package/plugins/workflow-runner/src/controlFlow/branch.js +0 -40
- package/plugins/workflow-runner/src/controlFlow/branch.ts +0 -46
- package/plugins/workflow-runner/src/controlFlow/loop.js +0 -211
- package/plugins/workflow-runner/src/controlFlow/loop.ts +0 -248
- package/plugins/workflow-runner/src/controlFlow/pause.js +0 -129
- package/plugins/workflow-runner/src/controlFlow/pause.ts +0 -141
- package/plugins/workflow-runner/src/controlFlow/signal.js +0 -21
- package/plugins/workflow-runner/src/controlFlow/signal.ts +0 -29
- package/plugins/workflow-runner/src/controlFlow/types.js +0 -18
- package/plugins/workflow-runner/src/controlFlow/types.ts +0 -63
- package/plugins/workflow-runner/src/dslManifest.js +0 -72
- package/plugins/workflow-runner/src/dslManifest.ts +0 -127
- package/plugins/workflow-runner/src/expressions.js +0 -639
- package/plugins/workflow-runner/src/expressions.ts +0 -640
- package/plugins/workflow-runner/src/loader.js +0 -573
- package/plugins/workflow-runner/src/loader.ts +0 -721
- package/plugins/workflow-runner/src/multimodal.js +0 -38
- package/plugins/workflow-runner/src/multimodal.ts +0 -64
- package/plugins/workflow-runner/src/recovery.js +0 -126
- package/plugins/workflow-runner/src/recovery.ts +0 -156
- package/plugins/workflow-runner/src/stepExecutors/agent.js +0 -173
- package/plugins/workflow-runner/src/stepExecutors/agent.ts +0 -234
- package/plugins/workflow-runner/src/stepExecutors/assert.js +0 -42
- package/plugins/workflow-runner/src/stepExecutors/assert.ts +0 -52
- package/plugins/workflow-runner/src/stepExecutors/echo.js +0 -31
- package/plugins/workflow-runner/src/stepExecutors/echo.ts +0 -38
- package/plugins/workflow-runner/src/stepExecutors/llm.js +0 -99
- package/plugins/workflow-runner/src/stepExecutors/llm.ts +0 -133
- package/plugins/workflow-runner/src/stepExecutors/set.js +0 -49
- package/plugins/workflow-runner/src/stepExecutors/set.ts +0 -56
- package/plugins/workflow-runner/src/stepExecutors/skill.js +0 -111
- package/plugins/workflow-runner/src/stepExecutors/skill.ts +0 -140
- package/plugins/workflow-runner/src/stepExecutors/tool.js +0 -227
- package/plugins/workflow-runner/src/stepExecutors/tool.ts +0 -274
- package/plugins/workflow-runner/src/stepExecutors/verify.js +0 -61
- package/plugins/workflow-runner/src/stepExecutors/verify.ts +0 -72
- package/plugins/workflow-runner/src/stepPipeline.js +0 -277
- package/plugins/workflow-runner/src/stepPipeline.ts +0 -366
- package/plugins/workflow-runner/test/agent.test.ts +0 -587
- package/plugins/workflow-runner/test/agentLlmError.test.ts +0 -373
- package/plugins/workflow-runner/test/bashBlacklist.sweep.test.ts +0 -65
- package/plugins/workflow-runner/test/captureMisbind.test.ts +0 -99
- package/plugins/workflow-runner/test/cli.e2e.test.ts +0 -167
- package/plugins/workflow-runner/test/context-files.test.ts +0 -320
- package/plugins/workflow-runner/test/e2e.test.ts +0 -370
- package/plugins/workflow-runner/test/e2eNewLayout.test.ts +0 -314
- package/plugins/workflow-runner/test/engineRobustness.test.ts +0 -234
- package/plugins/workflow-runner/test/expressions.test.ts +0 -439
- package/plugins/workflow-runner/test/foundation.test.ts +0 -126
- package/plugins/workflow-runner/test/graceful.test.ts +0 -139
- package/plugins/workflow-runner/test/greedyInputs.load.test.ts +0 -47
- package/plugins/workflow-runner/test/hostCapability.test.ts +0 -108
- package/plugins/workflow-runner/test/illustrate.load.test.ts +0 -183
- package/plugins/workflow-runner/test/jsonPayload.test.ts +0 -83
- package/plugins/workflow-runner/test/loader.test.ts +0 -617
- package/plugins/workflow-runner/test/maxTurnsRetryDelay.test.ts +0 -33
- package/plugins/workflow-runner/test/multimodal.test.ts +0 -342
- package/plugins/workflow-runner/test/output-schema.test.ts +0 -213
- package/plugins/workflow-runner/test/parseWorkflowArgs.test.ts +0 -106
- package/plugins/workflow-runner/test/pluginRunner.isolation.test.ts +0 -269
- package/plugins/workflow-runner/test/ppt.load.test.ts +0 -1220
- package/plugins/workflow-runner/test/ralphLoop.e2e.test.ts +0 -221
- package/plugins/workflow-runner/test/recovery.test.ts +0 -265
- package/plugins/workflow-runner/test/research.load.test.ts +0 -328
- package/plugins/workflow-runner/test/runner.test.ts +0 -911
- package/plugins/workflow-runner/test/runnerPreflight.test.ts +0 -436
- package/plugins/workflow-runner/test/skillTurnDone.test.ts +0 -71
- package/plugins/workflow-runner/test/stateLifecycle.test.ts +0 -198
- package/plugins/workflow-runner/test/stepPipeline.test.ts +0 -114
- package/plugins/workflow-runner/test/taskArbitration.test.ts +0 -354
- package/plugins/workflow-runner/test/taskLifecycle.test.ts +0 -324
- package/plugins/workflow-runner/test/toolArtifactOverwrite.test.ts +0 -221
- package/plugins/workflow-runner/test/toolCapabilityBoundary.test.ts +0 -29
- package/plugins/workflow-runner/test/toolRuntimeIntegration.test.ts +0 -168
- package/plugins/workflow-runner/test/webppt.load.test.ts +0 -250
- package/plugins/workflow-runner/test/word.load.test.ts +0 -208
- package/plugins/workflow-runner/test/workflows.smoke.test.ts +0 -76
- package/plugins/workflow-runner/test/yamlGuardConditions.test.ts +0 -207
- package/src/advisorState.js +0 -22
- package/src/cli/configWizard.js +0 -293
- package/src/tools/toolProgressBridge.js +0 -48
- package/workflows/adaptive-research.yaml +0 -50
- package/workflows/book-review-short.yaml +0 -125
- package/workflows/e2e-write-greet.yaml +0 -31
- package/workflows/illustrate.yaml +0 -227
- package/workflows/plan.yaml +0 -470
- package/workflows/ppt.yaml +0 -1670
- package/workflows/ralph-loop.yaml +0 -201
- package/workflows/research.yaml +0 -873
- package/workflows/schema.json +0 -176
- package/workflows/webppt.yaml +0 -496
- package/workflows/wechat-reading-shots.yaml +0 -315
- package/workflows/word.yaml +0 -720
- package/workflows/youtube-shorts.yaml +0 -202
- /package/workflows/{research.md → research/DESIGN.md} +0 -0
|
@@ -23,21 +23,25 @@
|
|
|
23
23
|
*/
|
|
24
24
|
import { z } from 'zod';
|
|
25
25
|
import { randomUUID } from 'node:crypto';
|
|
26
|
-
import { realpath } from 'node:fs/promises';
|
|
27
|
-
import { isAbsolute, relative, resolve } from 'node:path';
|
|
26
|
+
import { mkdir, realpath } from 'node:fs/promises';
|
|
27
|
+
import { isAbsolute, join, relative, resolve } from 'node:path';
|
|
28
|
+
import { getWorkingDir } from '../../bootstrap/workingDir.js';
|
|
28
29
|
import { configService } from '../../config/configService.js';
|
|
29
|
-
import { DEFAULT_MAX_RESULT_SIZE_CHARS } from '../types.js';
|
|
30
30
|
import { toToolParameters } from '../../utils/zodToJson.js';
|
|
31
|
+
import { DEFAULT_MAX_RESULT_SIZE_CHARS } from '../types.js';
|
|
31
32
|
import { resolveAgentBrowserPath } from './agentBrowserPath.js';
|
|
32
|
-
import { browserArtifactDir, createArtifactPath, isAllowedBrowserScheme, normalizeUrl, sessionArgs, spawnRunner, validateArtifactOutputPath,
|
|
33
|
-
import {
|
|
34
|
-
import {
|
|
35
|
-
import { getBrowserStream } from './browserStream.js';
|
|
36
|
-
import { browserService } from './browserService.js';
|
|
33
|
+
import { BROWSER_SESSION, browserArtifactDir, createArtifactPath, isAllowedBrowserScheme, normalizeUrl, sessionArgs, spawnRunner, validateArtifactOutputPath, } from './browser.js';
|
|
34
|
+
import { browserDownloadDir } from './browserAutomation.js';
|
|
35
|
+
import { CLIPBOARD_READ_EXPRESSION, CLIPBOARD_READ_TEXT_EXPRESSION, PAGE_ASSETS_EXPRESSION, browserAssetFetchExpression, bundlePageAssets, clipboardWriteExpression, clipboardWriteTextExpression, pageContentExpression, } from './browserContent.js';
|
|
37
36
|
import { detectBrowserChallenge } from './browserControl.js';
|
|
37
|
+
import { ensureDaemon, getStreamPort, stopDaemon } from './browserDaemon.js';
|
|
38
|
+
import { FRAME_INFO_EXPRESSION } from './browserFrames.js';
|
|
39
|
+
import { BROWSER_FAILURE_LIMIT, VERIFIED_BROWSER_ACTIONS, browserAttempts, budgetSnapshot, classifyPageEffect, describeEffect, differingLineIndexes, suggestSelectorCandidates, } from './browserOutcome.js';
|
|
40
|
+
import { recoverableAgentBrowserRunFailure, withBrowserRecovery, } from './browserRecovery.js';
|
|
38
41
|
import { browserRunRegistry, isLikelyIrreversibleBrowserTarget, } from './browserRunRegistry.js';
|
|
39
|
-
import {
|
|
40
|
-
import {
|
|
42
|
+
import { browserService } from './browserService.js';
|
|
43
|
+
import { getBrowserStream } from './browserStream.js';
|
|
44
|
+
import { BrowserTabError, claimConversationTabById, createConversationTab, ensureConversationTab, listBrowserTabs, } from './browserTabs.js';
|
|
41
45
|
const CHALLENGE_PROBE_ACTIONS = new Set([
|
|
42
46
|
'navigate',
|
|
43
47
|
'click',
|
|
@@ -51,7 +55,14 @@ const CHALLENGE_PROBE_ACTIONS = new Set([
|
|
|
51
55
|
'dialogAccept',
|
|
52
56
|
]);
|
|
53
57
|
// C4:驱动可见 AI 指针的交互动作(有明确元素目标)。navigate/press/dialogAccept 无元素目标不列入。
|
|
54
|
-
const POINTER_ACTIONS = new Set([
|
|
58
|
+
const POINTER_ACTIONS = new Set([
|
|
59
|
+
'click',
|
|
60
|
+
'fill',
|
|
61
|
+
'type',
|
|
62
|
+
'hover',
|
|
63
|
+
'select',
|
|
64
|
+
'drag',
|
|
65
|
+
]);
|
|
55
66
|
// 关键节点截图核验:这些是「页面到达/读取评估」的确定性检查点,成功后自动截当前可见画面喂视觉模型自我核验,
|
|
56
67
|
// 根治「AI 以为成功了、实际页面不是那回事」。不是每个动作都截(点击/输入等中间步不截),故不会「一直截图」。
|
|
57
68
|
const KEY_NODE_SHOT_ACTIONS = new Set(['navigate', 'snapshot']);
|
|
@@ -123,7 +134,10 @@ async function viewportPoint(run, x, y) {
|
|
|
123
134
|
catch {
|
|
124
135
|
return null;
|
|
125
136
|
}
|
|
126
|
-
if (!Number.isFinite(width) ||
|
|
137
|
+
if (!Number.isFinite(width) ||
|
|
138
|
+
!Number.isFinite(height) ||
|
|
139
|
+
width <= 0 ||
|
|
140
|
+
height <= 0) {
|
|
127
141
|
return null;
|
|
128
142
|
}
|
|
129
143
|
// 夹在视口内:越界坐标在 CDP 侧不报错,只会点到边缘,症状是"点了但没反应"。
|
|
@@ -147,11 +161,16 @@ async function runPointerAt(run, input, kind) {
|
|
|
147
161
|
return {
|
|
148
162
|
code: 1,
|
|
149
163
|
stdout: '',
|
|
150
|
-
stderr: '无法取得视口尺寸,坐标点击已放弃 —— 与其按一个猜出来的尺寸点在错误位置,'
|
|
151
|
-
|
|
164
|
+
stderr: '无法取得视口尺寸,坐标点击已放弃 —— 与其按一个猜出来的尺寸点在错误位置,' +
|
|
165
|
+
'不如不点。请确认页面已加载(先 navigate 或 snapshot),或改用选择器定位。',
|
|
152
166
|
};
|
|
153
167
|
}
|
|
154
|
-
const moved = await run([
|
|
168
|
+
const moved = await run([
|
|
169
|
+
'mouse',
|
|
170
|
+
'move',
|
|
171
|
+
String(point.px),
|
|
172
|
+
String(point.py),
|
|
173
|
+
]);
|
|
155
174
|
if (moved.code !== 0)
|
|
156
175
|
return moved;
|
|
157
176
|
if (kind === 'hover') {
|
|
@@ -163,7 +182,10 @@ async function runPointerAt(run, input, kind) {
|
|
|
163
182
|
const up = await run(['mouse', 'up']);
|
|
164
183
|
if (up.code !== 0)
|
|
165
184
|
return up;
|
|
166
|
-
return {
|
|
185
|
+
return {
|
|
186
|
+
...up,
|
|
187
|
+
stdout: `clicked at (${point.px}, ${point.py}) [视口比例 ${input.x}, ${input.y}]`,
|
|
188
|
+
};
|
|
167
189
|
}
|
|
168
190
|
/**
|
|
169
191
|
* 取**当前真实**页面 URL,供站点白名单判定。
|
|
@@ -212,7 +234,16 @@ const inputSchema = z.object({
|
|
|
212
234
|
'forward',
|
|
213
235
|
'reload',
|
|
214
236
|
'snapshot',
|
|
237
|
+
'frameInfo',
|
|
215
238
|
'getContent',
|
|
239
|
+
'pageAssets',
|
|
240
|
+
'listPageAssets',
|
|
241
|
+
'bundlePageAssets',
|
|
242
|
+
'exportContent',
|
|
243
|
+
'clipboardReadText',
|
|
244
|
+
'clipboardWriteText',
|
|
245
|
+
'clipboardRead',
|
|
246
|
+
'clipboardWrite',
|
|
216
247
|
'screenshot',
|
|
217
248
|
'click',
|
|
218
249
|
'fill',
|
|
@@ -256,9 +287,21 @@ const inputSchema = z.object({
|
|
|
256
287
|
'traceStop',
|
|
257
288
|
'reactTree',
|
|
258
289
|
'reactInspect',
|
|
290
|
+
'download',
|
|
291
|
+
'doubleClick',
|
|
292
|
+
'check',
|
|
293
|
+
'uncheck',
|
|
294
|
+
'setChecked',
|
|
295
|
+
'getState',
|
|
296
|
+
'waitForURL',
|
|
297
|
+
'waitForDownload',
|
|
298
|
+
'waitForFileChooser',
|
|
259
299
|
])
|
|
260
300
|
.describe('Browser action to perform'),
|
|
261
|
-
url: z
|
|
301
|
+
url: z
|
|
302
|
+
.string()
|
|
303
|
+
.optional()
|
|
304
|
+
.describe('URL to navigate to (for navigate/tabNew)'),
|
|
262
305
|
selector: z
|
|
263
306
|
.string()
|
|
264
307
|
.optional()
|
|
@@ -274,19 +317,64 @@ const inputSchema = z.object({
|
|
|
274
317
|
*
|
|
275
318
|
* 与 ComputerUse 的坐标系**刻意一致**(同为 0–1000),模型不必为两个工具切换心智。
|
|
276
319
|
*/
|
|
277
|
-
x: z
|
|
320
|
+
x: z
|
|
321
|
+
.number()
|
|
322
|
+
.min(0)
|
|
323
|
+
.max(1000)
|
|
324
|
+
.optional()
|
|
278
325
|
.describe('视口比例横坐标 0-1000(左=0,右=1000)。仅在没有可用选择器时用;必须与 y 同时提供'),
|
|
279
|
-
y: z
|
|
326
|
+
y: z
|
|
327
|
+
.number()
|
|
328
|
+
.min(0)
|
|
329
|
+
.max(1000)
|
|
330
|
+
.optional()
|
|
280
331
|
.describe('视口比例纵坐标 0-1000(上=0,下=1000)。仅在没有可用选择器时用;必须与 x 同时提供'),
|
|
281
332
|
value: z.string().optional().describe('Text value for fill/type/select'),
|
|
333
|
+
clipboardText: z
|
|
334
|
+
.string()
|
|
335
|
+
.optional()
|
|
336
|
+
.describe('Text to write to the browser profile clipboard'),
|
|
337
|
+
clipboardItems: z
|
|
338
|
+
.array(z.object({ type: z.string().min(1), base64: z.string() }))
|
|
339
|
+
.optional()
|
|
340
|
+
.describe('Binary clipboard items for clipboardWrite, encoded as base64'),
|
|
341
|
+
exportFormat: z
|
|
342
|
+
.enum([
|
|
343
|
+
'text',
|
|
344
|
+
'txt',
|
|
345
|
+
'markdown',
|
|
346
|
+
'md',
|
|
347
|
+
'html',
|
|
348
|
+
'json',
|
|
349
|
+
'page',
|
|
350
|
+
'google-workspace',
|
|
351
|
+
'youtube-transcript',
|
|
352
|
+
'pdf',
|
|
353
|
+
'docx',
|
|
354
|
+
'xlsx',
|
|
355
|
+
'csv',
|
|
356
|
+
'pptx',
|
|
357
|
+
])
|
|
358
|
+
.optional()
|
|
359
|
+
.describe('Format for exportContent'),
|
|
360
|
+
bundleDir: z
|
|
361
|
+
.string()
|
|
362
|
+
.optional()
|
|
363
|
+
.describe('Directory under the workspace or browser artifact root for bundlePageAssets'),
|
|
282
364
|
values: z.array(z.string()).optional().describe('Values for multi-select'),
|
|
283
|
-
key: z
|
|
365
|
+
key: z
|
|
366
|
+
.string()
|
|
367
|
+
.optional()
|
|
368
|
+
.describe('Key to press (for press): Enter, Tab, Escape, ArrowUp/Down/Left/Right, etc.'),
|
|
284
369
|
js: z.string().optional().describe('JavaScript to run (for eval)'),
|
|
285
370
|
path: z
|
|
286
371
|
.string()
|
|
287
372
|
.optional()
|
|
288
373
|
.describe('Output file path for screenshot/pdf/recording (optional; defaults to temp file)'),
|
|
289
|
-
files: z
|
|
374
|
+
files: z
|
|
375
|
+
.array(z.string())
|
|
376
|
+
.optional()
|
|
377
|
+
.describe('File paths to upload (for upload)'),
|
|
290
378
|
annotate: z
|
|
291
379
|
.boolean()
|
|
292
380
|
.optional()
|
|
@@ -299,16 +387,46 @@ const inputSchema = z.object({
|
|
|
299
387
|
.enum(['up', 'down', 'top', 'bottom'])
|
|
300
388
|
.optional()
|
|
301
389
|
.describe('scroll direction'),
|
|
302
|
-
scrollX: z
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
390
|
+
scrollX: z
|
|
391
|
+
.number()
|
|
392
|
+
.optional()
|
|
393
|
+
.describe('scroll pixels horizontally (positive=right, negative=left)'),
|
|
394
|
+
scrollY: z
|
|
395
|
+
.number()
|
|
396
|
+
.optional()
|
|
397
|
+
.describe('scroll pixels vertically (positive=down, negative=up)'),
|
|
398
|
+
waitMs: z
|
|
399
|
+
.number()
|
|
400
|
+
.int()
|
|
401
|
+
.positive()
|
|
402
|
+
.optional()
|
|
403
|
+
.describe('Wait time in milliseconds'),
|
|
404
|
+
waitSelector: z
|
|
405
|
+
.string()
|
|
406
|
+
.optional()
|
|
407
|
+
.describe('Wait until this selector appears'),
|
|
408
|
+
waitText: z
|
|
409
|
+
.string()
|
|
410
|
+
.optional()
|
|
411
|
+
.describe('Wait until this text appears on page'),
|
|
307
412
|
waitLoad: z
|
|
308
413
|
.enum(['load', 'domcontentloaded', 'networkidle'])
|
|
309
414
|
.optional()
|
|
310
415
|
.describe("Wait until the page reaches a load state after navigation. 'load' / 'domcontentloaded' settle quickly; 'networkidle' can wait up to ~25s on pages with long-lived connections."),
|
|
416
|
+
waitUrl: z
|
|
417
|
+
.string()
|
|
418
|
+
.optional()
|
|
419
|
+
.describe('Wait until the current URL matches this pattern'),
|
|
311
420
|
tabId: z.string().optional().describe('Tab ID for tab operations'),
|
|
421
|
+
retention: z
|
|
422
|
+
.enum(['ephemeral', 'conversation', 'handoff', 'deliverable'])
|
|
423
|
+
.optional()
|
|
424
|
+
.describe('Retention policy for the claimed tab'),
|
|
425
|
+
tabTitle: z
|
|
426
|
+
.string()
|
|
427
|
+
.max(200)
|
|
428
|
+
.optional()
|
|
429
|
+
.describe('Human-readable title for a retained tab'),
|
|
312
430
|
cookies: z
|
|
313
431
|
.array(z.object({
|
|
314
432
|
name: z.string(),
|
|
@@ -318,19 +436,51 @@ const inputSchema = z.object({
|
|
|
318
436
|
}))
|
|
319
437
|
.optional()
|
|
320
438
|
.describe('Cookies to set (for cookiesSet)'),
|
|
321
|
-
viewportWidth: z
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
439
|
+
viewportWidth: z
|
|
440
|
+
.number()
|
|
441
|
+
.int()
|
|
442
|
+
.positive()
|
|
443
|
+
.optional()
|
|
444
|
+
.describe('Viewport width'),
|
|
445
|
+
viewportHeight: z
|
|
446
|
+
.number()
|
|
447
|
+
.int()
|
|
448
|
+
.positive()
|
|
449
|
+
.optional()
|
|
450
|
+
.describe('Viewport height'),
|
|
451
|
+
devicePreset: z
|
|
452
|
+
.string()
|
|
453
|
+
.optional()
|
|
454
|
+
.describe('Device preset name (iPhone 15, iPad, Pixel 9, etc.)'),
|
|
455
|
+
colorScheme: z
|
|
456
|
+
.enum(['light', 'dark', 'no-preference'])
|
|
457
|
+
.optional()
|
|
458
|
+
.describe('Color scheme preference'),
|
|
325
459
|
offline: z.boolean().optional().describe('Enable offline mode'),
|
|
326
|
-
mode: z
|
|
327
|
-
|
|
460
|
+
mode: z
|
|
461
|
+
.enum(['browse', 'assist', 'auto'])
|
|
462
|
+
.optional()
|
|
463
|
+
.describe('Deprecated compatibility field; supervised Desktop control is automatic'),
|
|
464
|
+
dialogText: z
|
|
465
|
+
.string()
|
|
466
|
+
.optional()
|
|
467
|
+
.describe('Text to enter in prompt dialog (for dialogAccept)'),
|
|
328
468
|
pickX: z.number().optional().describe('X coordinate for pickElement'),
|
|
329
469
|
pickY: z.number().optional().describe('Y coordinate for pickElement'),
|
|
330
470
|
markLabel: z.string().optional().describe('Label for marked element'),
|
|
331
|
-
markColor: z
|
|
471
|
+
markColor: z
|
|
472
|
+
.string()
|
|
473
|
+
.optional()
|
|
474
|
+
.describe('Color for marked element highlight'),
|
|
332
475
|
dragTo: z.string().optional().describe('Target selector for drag action'),
|
|
333
|
-
|
|
476
|
+
checked: z
|
|
477
|
+
.boolean()
|
|
478
|
+
.optional()
|
|
479
|
+
.describe('Desired checkbox state for setChecked'),
|
|
480
|
+
reactId: z
|
|
481
|
+
.string()
|
|
482
|
+
.optional()
|
|
483
|
+
.describe('React fiber id returned by reactTree'),
|
|
334
484
|
timeout: z
|
|
335
485
|
.number()
|
|
336
486
|
.int()
|
|
@@ -343,17 +493,47 @@ const inputSchema = z.object({
|
|
|
343
493
|
// ⚠️ 语义是**整个序列**的总时长,不是单条命令。一次 click 在锁内会跑「目标文案
|
|
344
494
|
// 探测 → 元素盒子 → 前置快照 → 动作 → 稳定重探 → 可选噪声基线」,这个预算盖住
|
|
345
495
|
// 全部。描述必须说清楚,否则调用方按「单个动作」估的值会把前后探测一起掐掉。
|
|
346
|
-
.describe('Timeout in milliseconds for the whole action sequence — including the pre-action probe, '
|
|
347
|
-
|
|
496
|
+
.describe('Timeout in milliseconds for the whole action sequence — including the pre-action probe, ' +
|
|
497
|
+
'the action itself, and the post-action settle/verify probes (default: 120000, max: 300000)'),
|
|
348
498
|
intent: z
|
|
349
499
|
.enum(['normal', 'submit'])
|
|
350
500
|
.optional()
|
|
351
501
|
.describe('Use submit for the final irreversible action, such as publish, purchase, delete, or form submission'),
|
|
352
502
|
});
|
|
353
503
|
const OBSERVE_ACTIONS = new Set([
|
|
354
|
-
'snapshot',
|
|
355
|
-
'
|
|
356
|
-
'
|
|
504
|
+
'snapshot',
|
|
505
|
+
'frameInfo',
|
|
506
|
+
'getContent',
|
|
507
|
+
'pageAssets',
|
|
508
|
+
'listPageAssets',
|
|
509
|
+
'bundlePageAssets',
|
|
510
|
+
'exportContent',
|
|
511
|
+
'clipboardReadText',
|
|
512
|
+
'clipboardRead',
|
|
513
|
+
'screenshot',
|
|
514
|
+
'tabList',
|
|
515
|
+
'streamInfo',
|
|
516
|
+
'console',
|
|
517
|
+
'errors',
|
|
518
|
+
'networkRequests',
|
|
519
|
+
'getStyles',
|
|
520
|
+
'vitals',
|
|
521
|
+
'reactTree',
|
|
522
|
+
'reactInspect',
|
|
523
|
+
]);
|
|
524
|
+
// These capabilities use fixed internal expressions. They are not an escape
|
|
525
|
+
// hatch for arbitrary JavaScript; caller supplied code remains the `eval`
|
|
526
|
+
// action and keeps its Developer Mode gate.
|
|
527
|
+
const INTERNAL_SCRIPT_ACTIONS = new Set([
|
|
528
|
+
'pageAssets',
|
|
529
|
+
'exportContent',
|
|
530
|
+
'clipboardReadText',
|
|
531
|
+
'clipboardWriteText',
|
|
532
|
+
'clipboardRead',
|
|
533
|
+
'clipboardWrite',
|
|
534
|
+
'frameInfo',
|
|
535
|
+
'listPageAssets',
|
|
536
|
+
'bundlePageAssets',
|
|
357
537
|
]);
|
|
358
538
|
const parameters = toToolParameters(inputSchema);
|
|
359
539
|
const description = `Controls KakadeAI Desktop's visible, human-supervised browser for browser automation.
|
|
@@ -394,7 +574,12 @@ Navigation:
|
|
|
394
574
|
|
|
395
575
|
Page Reading:
|
|
396
576
|
snapshot - Accessibility tree with @eN refs (LLM gold standard for seeing page).
|
|
577
|
+
frameInfo - List the current page's iframe/frame handles and URLs.
|
|
397
578
|
getContent [selector] - Plain text of page or element (default body).
|
|
579
|
+
pageAssets - List images, media, styles, scripts, fonts and icons referenced by the page.
|
|
580
|
+
exportContent - Export page, Google Workspace or YouTube transcript content as structured data.
|
|
581
|
+
clipboardReadText - Read text from the browser profile clipboard.
|
|
582
|
+
clipboardWriteText - Write clipboardText to the browser profile clipboard.
|
|
398
583
|
screenshot [--annotate] [--fullPage] [--path] - Save PNG; --annotate numbers interactive elements.
|
|
399
584
|
pdf [path] - Save page as PDF.
|
|
400
585
|
eval <js> - Run JavaScript, return result.
|
|
@@ -470,6 +655,7 @@ export const ARTIFACT_ACTIONS = {
|
|
|
470
655
|
pdf: { prefix: 'browser-page', ext: 'pdf' },
|
|
471
656
|
recordStart: { prefix: 'recording', ext: 'webm' },
|
|
472
657
|
traceStop: { prefix: 'browser-trace', ext: 'json' },
|
|
658
|
+
download: { prefix: 'browser-download', ext: 'download' },
|
|
473
659
|
};
|
|
474
660
|
/**
|
|
475
661
|
* @param resolvedOutPath 已经过 `validateArtifactOutputPath` 校验的落盘路径。
|
|
@@ -506,12 +692,44 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
506
692
|
return { args: ['reload'] };
|
|
507
693
|
case 'snapshot':
|
|
508
694
|
return { args: ['snapshot'] };
|
|
695
|
+
case 'frameInfo':
|
|
696
|
+
return { args: ['eval', FRAME_INFO_EXPRESSION] };
|
|
509
697
|
case 'getContent':
|
|
510
698
|
return { args: ['get', 'text', input.selector ?? 'body'] };
|
|
699
|
+
case 'pageAssets':
|
|
700
|
+
case 'listPageAssets':
|
|
701
|
+
return { args: ['eval', PAGE_ASSETS_EXPRESSION] };
|
|
702
|
+
case 'bundlePageAssets':
|
|
703
|
+
return { args: ['eval', PAGE_ASSETS_EXPRESSION] };
|
|
704
|
+
case 'exportContent':
|
|
705
|
+
return {
|
|
706
|
+
args: [
|
|
707
|
+
'eval',
|
|
708
|
+
pageContentExpression(input.exportFormat ?? 'page', input.selector),
|
|
709
|
+
],
|
|
710
|
+
};
|
|
711
|
+
case 'clipboardReadText':
|
|
712
|
+
return { args: ['eval', CLIPBOARD_READ_TEXT_EXPRESSION] };
|
|
713
|
+
case 'clipboardWriteText':
|
|
714
|
+
if (input.clipboardText === undefined)
|
|
715
|
+
return { error: 'clipboardWriteText requires clipboardText' };
|
|
716
|
+
return {
|
|
717
|
+
args: ['eval', clipboardWriteTextExpression(input.clipboardText)],
|
|
718
|
+
};
|
|
719
|
+
case 'clipboardRead':
|
|
720
|
+
return { args: ['eval', CLIPBOARD_READ_EXPRESSION] };
|
|
721
|
+
case 'clipboardWrite':
|
|
722
|
+
if (!input.clipboardItems?.length)
|
|
723
|
+
return { error: 'clipboardWrite requires clipboardItems' };
|
|
724
|
+
return {
|
|
725
|
+
args: ['eval', clipboardWriteExpression(input.clipboardItems)],
|
|
726
|
+
};
|
|
511
727
|
case 'screenshot': {
|
|
512
728
|
const out = resolvedOutPath ?? input.path;
|
|
513
729
|
if (!out)
|
|
514
|
-
return {
|
|
730
|
+
return {
|
|
731
|
+
error: `${action} 需要已解析的输出路径(由 call() 校验后传入)`,
|
|
732
|
+
};
|
|
515
733
|
const args = ['screenshot', out];
|
|
516
734
|
if (input.annotate)
|
|
517
735
|
args.push('--annotate');
|
|
@@ -525,9 +743,31 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
525
743
|
if (!input.selector) {
|
|
526
744
|
if (hasPointCoords(input))
|
|
527
745
|
return { args: [], pointerAction: 'click' };
|
|
528
|
-
return {
|
|
746
|
+
return {
|
|
747
|
+
error: 'click 需要 selector;没有可用选择器时可改用视口比例坐标 x/y(0-1000)',
|
|
748
|
+
};
|
|
529
749
|
}
|
|
530
750
|
return { args: ['click', input.selector] };
|
|
751
|
+
case 'doubleClick':
|
|
752
|
+
if (!input.selector)
|
|
753
|
+
return { error: 'doubleClick requires selector' };
|
|
754
|
+
return { args: ['dblclick', input.selector] };
|
|
755
|
+
case 'check':
|
|
756
|
+
if (!input.selector)
|
|
757
|
+
return { error: 'check requires selector' };
|
|
758
|
+
return { args: ['check', input.selector] };
|
|
759
|
+
case 'uncheck':
|
|
760
|
+
if (!input.selector)
|
|
761
|
+
return { error: 'uncheck requires selector' };
|
|
762
|
+
return { args: ['uncheck', input.selector] };
|
|
763
|
+
case 'setChecked':
|
|
764
|
+
if (!input.selector || input.checked === undefined)
|
|
765
|
+
return { error: 'setChecked requires selector and checked' };
|
|
766
|
+
return { args: [input.checked ? 'check' : 'uncheck', input.selector] };
|
|
767
|
+
case 'getState':
|
|
768
|
+
if (!input.selector)
|
|
769
|
+
return { error: 'getState requires selector' };
|
|
770
|
+
return { args: ['is', 'visible', input.selector] };
|
|
531
771
|
case 'fill':
|
|
532
772
|
if (!input.selector || input.value === undefined)
|
|
533
773
|
return { error: 'fill requires selector and value parameters' };
|
|
@@ -540,7 +780,9 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
540
780
|
if (!input.selector) {
|
|
541
781
|
if (hasPointCoords(input))
|
|
542
782
|
return { args: [], pointerAction: 'hover' };
|
|
543
|
-
return {
|
|
783
|
+
return {
|
|
784
|
+
error: 'hover 需要 selector;没有可用选择器时可改用视口比例坐标 x/y(0-1000)',
|
|
785
|
+
};
|
|
544
786
|
}
|
|
545
787
|
return { args: ['hover', input.selector] };
|
|
546
788
|
case 'scroll': {
|
|
@@ -548,7 +790,7 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
548
790
|
if (input.selector)
|
|
549
791
|
args.push(input.selector);
|
|
550
792
|
if (input.direction)
|
|
551
|
-
args.push(
|
|
793
|
+
args.push(`--${input.direction}`);
|
|
552
794
|
if (input.scrollY)
|
|
553
795
|
args.push('--by-y', String(input.scrollY));
|
|
554
796
|
if (input.scrollX)
|
|
@@ -557,14 +799,18 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
557
799
|
}
|
|
558
800
|
case 'select': {
|
|
559
801
|
if (!input.selector || (input.value === undefined && !input.values))
|
|
560
|
-
return {
|
|
802
|
+
return {
|
|
803
|
+
error: 'select requires selector and value/values parameters',
|
|
804
|
+
};
|
|
561
805
|
const vals = input.values ?? [input.value];
|
|
562
806
|
return { args: ['select', input.selector, ...vals] };
|
|
563
807
|
}
|
|
564
808
|
case 'press':
|
|
565
809
|
if (!input.key)
|
|
566
810
|
return { error: 'press requires key parameter' };
|
|
567
|
-
return {
|
|
811
|
+
return {
|
|
812
|
+
args: ['press', input.key, ...(input.selector ? [input.selector] : [])],
|
|
813
|
+
};
|
|
568
814
|
case 'wait': {
|
|
569
815
|
// agent-browser `wait` grammar: bare positional `wait <ms>` = plain sleep,
|
|
570
816
|
// `wait <selector>` = wait-for-element; only --text/--load/--url/--fn are flags.
|
|
@@ -578,8 +824,20 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
578
824
|
return { args: ['wait', '--load', input.waitLoad] };
|
|
579
825
|
if (input.waitMs)
|
|
580
826
|
return { args: ['wait', String(input.waitMs)] };
|
|
581
|
-
return {
|
|
827
|
+
return {
|
|
828
|
+
error: 'wait requires waitMs / waitSelector / waitText / waitLoad',
|
|
829
|
+
};
|
|
582
830
|
}
|
|
831
|
+
case 'waitForURL':
|
|
832
|
+
if (!input.waitUrl && !input.url)
|
|
833
|
+
return { error: 'waitForURL requires waitUrl or url' };
|
|
834
|
+
return { args: ['wait', '--url', input.waitUrl ?? input.url] };
|
|
835
|
+
case 'waitForDownload':
|
|
836
|
+
return { args: ['wait', '--download'] };
|
|
837
|
+
case 'waitForFileChooser':
|
|
838
|
+
return {
|
|
839
|
+
error: 'waitForFileChooser is available in BrowserBatch, where it can be atomically paired with its triggering action',
|
|
840
|
+
};
|
|
583
841
|
case 'eval':
|
|
584
842
|
if (!input.js)
|
|
585
843
|
return { error: 'eval requires js parameter' };
|
|
@@ -587,15 +845,23 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
587
845
|
case 'pdf': {
|
|
588
846
|
const out = resolvedOutPath ?? input.path;
|
|
589
847
|
if (!out)
|
|
590
|
-
return {
|
|
848
|
+
return {
|
|
849
|
+
error: `${action} 需要已解析的输出路径(由 call() 校验后传入)`,
|
|
850
|
+
};
|
|
591
851
|
return { args: ['pdf', out], outPath: out };
|
|
592
852
|
}
|
|
593
853
|
case 'close':
|
|
594
854
|
return { args: ['close'] };
|
|
595
855
|
case 'tabNew':
|
|
596
|
-
return {
|
|
856
|
+
return {
|
|
857
|
+
args: input.url
|
|
858
|
+
? ['tab', 'new', normalizeUrl(input.url)]
|
|
859
|
+
: ['tab', 'new'],
|
|
860
|
+
};
|
|
597
861
|
case 'tabClose':
|
|
598
|
-
return {
|
|
862
|
+
return {
|
|
863
|
+
args: input.tabId ? ['tab', 'close', input.tabId] : ['tab', 'close'],
|
|
864
|
+
};
|
|
599
865
|
case 'tabSwitch':
|
|
600
866
|
if (!input.tabId)
|
|
601
867
|
return { error: 'tabSwitch requires tabId parameter' };
|
|
@@ -607,14 +873,25 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
607
873
|
case 'cookiesSet': {
|
|
608
874
|
if (!input.cookies || input.cookies.length === 0)
|
|
609
875
|
return { error: 'cookiesSet requires cookies array' };
|
|
610
|
-
return {
|
|
876
|
+
return {
|
|
877
|
+
args: ['cookies', 'set', '--json', JSON.stringify(input.cookies)],
|
|
878
|
+
};
|
|
611
879
|
}
|
|
612
880
|
case 'cookiesClear':
|
|
613
881
|
return { args: ['cookies', 'clear'] };
|
|
614
882
|
case 'setViewport': {
|
|
615
883
|
if (!input.viewportWidth || !input.viewportHeight)
|
|
616
|
-
return {
|
|
617
|
-
|
|
884
|
+
return {
|
|
885
|
+
error: 'setViewport requires viewportWidth and viewportHeight',
|
|
886
|
+
};
|
|
887
|
+
return {
|
|
888
|
+
args: [
|
|
889
|
+
'set',
|
|
890
|
+
'viewport',
|
|
891
|
+
String(input.viewportWidth),
|
|
892
|
+
String(input.viewportHeight),
|
|
893
|
+
],
|
|
894
|
+
};
|
|
618
895
|
}
|
|
619
896
|
case 'setDevice': {
|
|
620
897
|
const args = ['set', 'device'];
|
|
@@ -637,13 +914,19 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
637
914
|
case 'recordStart': {
|
|
638
915
|
const out = resolvedOutPath ?? input.path;
|
|
639
916
|
if (!out)
|
|
640
|
-
return {
|
|
917
|
+
return {
|
|
918
|
+
error: `${action} 需要已解析的输出路径(由 call() 校验后传入)`,
|
|
919
|
+
};
|
|
641
920
|
return { args: ['record', 'start', out], outPath: out };
|
|
642
921
|
}
|
|
643
922
|
case 'recordStop':
|
|
644
923
|
return { args: ['record', 'stop'] };
|
|
645
924
|
case 'dialogAccept':
|
|
646
|
-
return {
|
|
925
|
+
return {
|
|
926
|
+
args: input.dialogText
|
|
927
|
+
? ['dialog', 'accept', '--text', input.dialogText]
|
|
928
|
+
: ['dialog', 'accept'],
|
|
929
|
+
};
|
|
647
930
|
case 'dialogDismiss':
|
|
648
931
|
return { args: ['dialog', 'dismiss'] };
|
|
649
932
|
case 'pickElement': {
|
|
@@ -663,24 +946,43 @@ function buildSubArgs(input, resolvedOutPath) {
|
|
|
663
946
|
return { error: 'upload requires selector and files parameters' };
|
|
664
947
|
return { args: ['upload', input.selector, ...input.files] };
|
|
665
948
|
}
|
|
949
|
+
case 'download': {
|
|
950
|
+
if (!input.selector)
|
|
951
|
+
return { error: 'download requires selector' };
|
|
952
|
+
const out = resolvedOutPath ?? input.path;
|
|
953
|
+
if (!out)
|
|
954
|
+
return { error: 'download requires output path' };
|
|
955
|
+
return { args: ['download', input.selector, out], outPath: out };
|
|
956
|
+
}
|
|
666
957
|
case 'drag': {
|
|
667
958
|
if (!input.selector || !input.dragTo)
|
|
668
|
-
return {
|
|
959
|
+
return {
|
|
960
|
+
error: 'drag requires selector (from) and dragTo (to) parameters',
|
|
961
|
+
};
|
|
669
962
|
return { args: ['drag', input.selector, input.dragTo] };
|
|
670
963
|
}
|
|
671
|
-
case 'console':
|
|
672
|
-
|
|
673
|
-
case '
|
|
674
|
-
|
|
675
|
-
case '
|
|
676
|
-
|
|
964
|
+
case 'console':
|
|
965
|
+
return { args: ['console'] };
|
|
966
|
+
case 'errors':
|
|
967
|
+
return { args: ['errors'] };
|
|
968
|
+
case 'networkRequests':
|
|
969
|
+
return { args: ['network', 'requests', '--json'] };
|
|
970
|
+
case 'getStyles':
|
|
971
|
+
return { args: ['get', 'styles', input.selector ?? 'body', '--json'] };
|
|
972
|
+
case 'vitals':
|
|
973
|
+
return { args: ['vitals', ...(input.url ? [input.url] : []), '--json'] };
|
|
974
|
+
case 'traceStart':
|
|
975
|
+
return { args: ['trace', 'start'] };
|
|
677
976
|
case 'traceStop': {
|
|
678
977
|
const out = resolvedOutPath ?? input.path;
|
|
679
978
|
if (!out)
|
|
680
|
-
return {
|
|
979
|
+
return {
|
|
980
|
+
error: `${action} 需要已解析的输出路径(由 call() 校验后传入)`,
|
|
981
|
+
};
|
|
681
982
|
return { args: ['trace', 'stop', out], outPath: out };
|
|
682
983
|
}
|
|
683
|
-
case 'reactTree':
|
|
984
|
+
case 'reactTree':
|
|
985
|
+
return { args: ['react', 'tree'] };
|
|
684
986
|
case 'reactInspect':
|
|
685
987
|
if (!input.reactId)
|
|
686
988
|
return { error: 'reactInspect requires reactId parameter' };
|
|
@@ -755,7 +1057,11 @@ async function handleMetaAction(action, input, signal, ctx = {}) {
|
|
|
755
1057
|
const port = stream.getPort();
|
|
756
1058
|
return {
|
|
757
1059
|
ok: true,
|
|
758
|
-
content: JSON.stringify({
|
|
1060
|
+
content: JSON.stringify({
|
|
1061
|
+
connected: stream.isConnected(),
|
|
1062
|
+
port,
|
|
1063
|
+
status,
|
|
1064
|
+
}),
|
|
759
1065
|
};
|
|
760
1066
|
}
|
|
761
1067
|
case 'setMode': {
|
|
@@ -766,15 +1072,29 @@ async function handleMetaAction(action, input, signal, ctx = {}) {
|
|
|
766
1072
|
case 'pickElement': {
|
|
767
1073
|
const el = await stream.pickElement(input.pickX, input.pickY);
|
|
768
1074
|
if (!el)
|
|
769
|
-
return {
|
|
770
|
-
|
|
1075
|
+
return {
|
|
1076
|
+
ok: false,
|
|
1077
|
+
error: 'pickElement: no element found at coordinates',
|
|
1078
|
+
};
|
|
1079
|
+
stream.markElement({
|
|
1080
|
+
ref: el.ref,
|
|
1081
|
+
tag: el.tag,
|
|
1082
|
+
text: el.text,
|
|
1083
|
+
boundingBox: el.boundingBox,
|
|
1084
|
+
label: el.ref,
|
|
1085
|
+
});
|
|
771
1086
|
return {
|
|
772
1087
|
ok: true,
|
|
773
1088
|
content: JSON.stringify(el),
|
|
774
1089
|
};
|
|
775
1090
|
}
|
|
776
1091
|
case 'markElement': {
|
|
777
|
-
const el = {
|
|
1092
|
+
const el = {
|
|
1093
|
+
ref: input.selector,
|
|
1094
|
+
tag: input.selector,
|
|
1095
|
+
label: input.markLabel,
|
|
1096
|
+
color: input.markColor,
|
|
1097
|
+
};
|
|
778
1098
|
stream.markElement(el);
|
|
779
1099
|
return { ok: true, content: `[markElement] marked ${input.selector}` };
|
|
780
1100
|
}
|
|
@@ -794,22 +1114,38 @@ async function handleMetaAction(action, input, signal, ctx = {}) {
|
|
|
794
1114
|
function describeAction(input) {
|
|
795
1115
|
const target = input.selector ?? input.url ?? '';
|
|
796
1116
|
switch (input.action) {
|
|
797
|
-
case 'navigate':
|
|
798
|
-
|
|
799
|
-
case '
|
|
800
|
-
|
|
801
|
-
case '
|
|
802
|
-
|
|
803
|
-
case '
|
|
804
|
-
|
|
805
|
-
case '
|
|
806
|
-
|
|
807
|
-
case '
|
|
808
|
-
|
|
809
|
-
case '
|
|
810
|
-
|
|
811
|
-
case '
|
|
812
|
-
|
|
1117
|
+
case 'navigate':
|
|
1118
|
+
return `正在打开 ${input.url ?? ''}`.trim();
|
|
1119
|
+
case 'click':
|
|
1120
|
+
return `正在点击 ${target}`.trim();
|
|
1121
|
+
case 'fill':
|
|
1122
|
+
return `正在填写 ${input.selector ?? ''}`.trim();
|
|
1123
|
+
case 'type':
|
|
1124
|
+
return `正在输入到 ${input.selector ?? ''}`.trim();
|
|
1125
|
+
case 'hover':
|
|
1126
|
+
return `正在悬停 ${input.selector ?? ''}`.trim();
|
|
1127
|
+
case 'select':
|
|
1128
|
+
return `正在选择 ${input.selector ?? ''}`.trim();
|
|
1129
|
+
case 'press':
|
|
1130
|
+
return `正在按键 ${input.key ?? ''}`.trim();
|
|
1131
|
+
case 'scroll':
|
|
1132
|
+
return '正在滚动页面';
|
|
1133
|
+
case 'drag':
|
|
1134
|
+
return `正在拖拽 ${input.selector ?? ''}`.trim();
|
|
1135
|
+
case 'upload':
|
|
1136
|
+
return '正在上传文件';
|
|
1137
|
+
case 'back':
|
|
1138
|
+
return '正在后退';
|
|
1139
|
+
case 'forward':
|
|
1140
|
+
return '正在前进';
|
|
1141
|
+
case 'reload':
|
|
1142
|
+
return '正在刷新页面';
|
|
1143
|
+
case 'screenshot':
|
|
1144
|
+
return '正在截图';
|
|
1145
|
+
case 'pdf':
|
|
1146
|
+
return '正在导出 PDF';
|
|
1147
|
+
default:
|
|
1148
|
+
return `正在执行 ${input.action}`;
|
|
813
1149
|
}
|
|
814
1150
|
}
|
|
815
1151
|
/** C4:把工具动作映射成指针动效类型(协议 action 枚举)。 */
|
|
@@ -831,7 +1167,10 @@ async function sleep(ms, signal) {
|
|
|
831
1167
|
signal?.removeEventListener('abort', onAbort);
|
|
832
1168
|
resolve();
|
|
833
1169
|
}, ms);
|
|
834
|
-
const onAbort = () => {
|
|
1170
|
+
const onAbort = () => {
|
|
1171
|
+
clearTimeout(timer);
|
|
1172
|
+
resolve();
|
|
1173
|
+
};
|
|
835
1174
|
signal?.addEventListener('abort', onAbort, { once: true });
|
|
836
1175
|
});
|
|
837
1176
|
}
|
|
@@ -839,7 +1178,12 @@ async function sleep(ms, signal) {
|
|
|
839
1178
|
* Pure visual enhancement - failures are silently ignored (never block the action). */
|
|
840
1179
|
function flashTargetElement(stream, selector) {
|
|
841
1180
|
try {
|
|
842
|
-
stream.markElement({
|
|
1181
|
+
stream.markElement({
|
|
1182
|
+
ref: selector,
|
|
1183
|
+
tag: selector,
|
|
1184
|
+
color: '#22c55e',
|
|
1185
|
+
label: '',
|
|
1186
|
+
});
|
|
843
1187
|
setTimeout(() => stream.unmarkElement(selector), 350);
|
|
844
1188
|
}
|
|
845
1189
|
catch {
|
|
@@ -912,12 +1256,12 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
912
1256
|
// 于是排成:scheme 白名单 → 输出路径校验 → argv 拼装 → metaAction → 解析二进制 → 门禁。
|
|
913
1257
|
// `javascript:` 是在页面里执行代码,必须走受门禁约束的 `eval`,不能靠一个看起来像
|
|
914
1258
|
// "打开网址"的动作绕过去;`data:` 可构造带脚本的文档,等价同一件事。
|
|
915
|
-
if ((input.action === 'navigate' || input.action === 'tabNew') &&
|
|
916
|
-
|
|
1259
|
+
if ((input.action === 'navigate' || input.action === 'tabNew') &&
|
|
1260
|
+
input.url &&
|
|
1261
|
+
!isAllowedBrowserScheme(input.url)) {
|
|
917
1262
|
return {
|
|
918
1263
|
ok: false,
|
|
919
|
-
error: `[${input.action}] 不支持的地址协议:${input.url.split(':')[0]}:。仅允许 http/https/about/file
|
|
920
|
-
+ '要在页面里执行脚本请用 eval(需 Developer Mode 授权)。',
|
|
1264
|
+
error: `[${input.action}] 不支持的地址协议:${input.url.split(':')[0]}:。仅允许 http/https/about/file;要在页面里执行脚本请用 eval(需 Developer Mode 授权)。`,
|
|
921
1265
|
};
|
|
922
1266
|
}
|
|
923
1267
|
// 会落盘的动作:先把输出路径解析并校验好,再拼命令。
|
|
@@ -935,10 +1279,33 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
935
1279
|
return { ok: false, error: `[${input.action}] ${checked.error}` };
|
|
936
1280
|
resolvedOutPath = checked.path;
|
|
937
1281
|
}
|
|
1282
|
+
else if (input.action === 'download') {
|
|
1283
|
+
const dir = browserDownloadDir(ctx.browserRun?.conversationKey, ctx.browserRun?.runId ?? ctx.toolCallId);
|
|
1284
|
+
await mkdir(dir, { recursive: true, mode: 0o700 });
|
|
1285
|
+
resolvedOutPath = join(dir, `download-${Date.now()}-${randomUUID().slice(0, 8)}.download`);
|
|
1286
|
+
}
|
|
938
1287
|
else {
|
|
939
1288
|
resolvedOutPath = await createArtifactPath(artifact.prefix, artifact.ext);
|
|
940
1289
|
}
|
|
941
1290
|
}
|
|
1291
|
+
// Asset bundling writes a manifest and fetched files. Validate its root
|
|
1292
|
+
// before starting the browser or evaluating the page, just like the other
|
|
1293
|
+
// browser write actions.
|
|
1294
|
+
if (input.action === 'bundlePageAssets') {
|
|
1295
|
+
if (!input.bundleDir)
|
|
1296
|
+
return { ok: false, error: 'bundlePageAssets requires bundleDir' };
|
|
1297
|
+
const roots = [browserArtifactDir(), getWorkingDir()];
|
|
1298
|
+
const target = await realpath(input.bundleDir).catch(() => resolve(input.bundleDir));
|
|
1299
|
+
const allowed = roots.some((root) => {
|
|
1300
|
+
const child = relative(root, target);
|
|
1301
|
+
return child === '' || (!child.startsWith('..') && !isAbsolute(child));
|
|
1302
|
+
});
|
|
1303
|
+
if (!allowed)
|
|
1304
|
+
return {
|
|
1305
|
+
ok: false,
|
|
1306
|
+
error: `资产打包目录不在浏览器产物或工作区范围内:${input.bundleDir}`,
|
|
1307
|
+
};
|
|
1308
|
+
}
|
|
942
1309
|
const built = buildSubArgs(input, resolvedOutPath);
|
|
943
1310
|
if ('error' in built)
|
|
944
1311
|
return { ok: false, error: built.error };
|
|
@@ -958,8 +1325,13 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
958
1325
|
// Developer Mode 的开关本身是纯配置判定,没有副作用,在这里挡掉最省事。
|
|
959
1326
|
// 但**站点 origin 判定不在这里**——它需要「当前真实 URL」,而那取决于本次动作会
|
|
960
1327
|
// 落在哪个标签页,此刻还没认领。它挪进了下面的动作序列内(见 `requireLiveOrigin`)。
|
|
961
|
-
if (DEVELOPER_MODE_ACTIONS.has(input.action) &&
|
|
962
|
-
|
|
1328
|
+
if (DEVELOPER_MODE_ACTIONS.has(input.action) &&
|
|
1329
|
+
!INTERNAL_SCRIPT_ACTIONS.has(input.action) &&
|
|
1330
|
+
!browserConfig.developerMode) {
|
|
1331
|
+
return {
|
|
1332
|
+
ok: false,
|
|
1333
|
+
error: '完整浏览器调试未授权。请先在“设置 → 浏览器”开启 Developer Mode。',
|
|
1334
|
+
};
|
|
963
1335
|
}
|
|
964
1336
|
const stream = getBrowserStream(BROWSER_SESSION);
|
|
965
1337
|
const runId = ctx.browserRun?.runId ?? ctx.toolCallId ?? randomUUID();
|
|
@@ -1006,12 +1378,23 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1006
1378
|
browserRunRegistry.requireGrant(browserContext, 'submit');
|
|
1007
1379
|
}
|
|
1008
1380
|
catch (error) {
|
|
1009
|
-
return {
|
|
1381
|
+
return {
|
|
1382
|
+
ok: false,
|
|
1383
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1384
|
+
};
|
|
1010
1385
|
}
|
|
1011
1386
|
// 仅当宿主声明「有可见浏览器 + 真人可接管」(Desktop)才允许遇关卡阻塞等人;
|
|
1012
1387
|
// CLI -p / TUI / gateway 不设 browserRun → supervised=false → 据实非阻塞返回,绝不挂起。
|
|
1013
1388
|
const supervised = ctx.browserRun?.supervised === true;
|
|
1014
|
-
const visibleAgentAction = ![
|
|
1389
|
+
const visibleAgentAction = ![
|
|
1390
|
+
'snapshot',
|
|
1391
|
+
'getContent',
|
|
1392
|
+
'console',
|
|
1393
|
+
'errors',
|
|
1394
|
+
'networkRequests',
|
|
1395
|
+
'getStyles',
|
|
1396
|
+
'vitals',
|
|
1397
|
+
].includes(input.action);
|
|
1015
1398
|
// 预备步(认领会话标签页)专用控制器:受父级中断 + 超时约束,但不被「接管」打断(快速一次性)。
|
|
1016
1399
|
const preCtrl = new AbortController();
|
|
1017
1400
|
let preTimer = setTimeout(() => preCtrl.abort(), input.timeout ?? 120_000);
|
|
@@ -1050,10 +1433,12 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1050
1433
|
signal.addEventListener('abort', onParent, { once: true });
|
|
1051
1434
|
}
|
|
1052
1435
|
// subscribe 立即以当前状态回调一次:动作开始时是 running(不掐);中途进入 waiting_human 才掐断。
|
|
1053
|
-
const unsub = stream.subscribeControl((state) => {
|
|
1054
|
-
|
|
1436
|
+
const unsub = stream.subscribeControl((state) => {
|
|
1437
|
+
if (state.status === 'waiting_human')
|
|
1438
|
+
ac.abort();
|
|
1439
|
+
});
|
|
1055
1440
|
try {
|
|
1056
|
-
return await browserService.exclusive({
|
|
1441
|
+
return await withBrowserRecovery(() => browserService.exclusive({
|
|
1057
1442
|
sessionName: BROWSER_SESSION,
|
|
1058
1443
|
bin,
|
|
1059
1444
|
runner: deps.run,
|
|
@@ -1064,14 +1449,41 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1064
1449
|
// 每条都切纯属浪费一次子进程往返。
|
|
1065
1450
|
const scoped = async (args) => {
|
|
1066
1451
|
if (claimedTabId && !tabEnsured) {
|
|
1067
|
-
const switched = await run([
|
|
1068
|
-
|
|
1452
|
+
const switched = await run([
|
|
1453
|
+
...sessionArgs(BROWSER_SESSION),
|
|
1454
|
+
'tab',
|
|
1455
|
+
claimedTabId,
|
|
1456
|
+
]);
|
|
1457
|
+
if (switched.code !== 0) {
|
|
1458
|
+
const recoverable = recoverableAgentBrowserRunFailure(switched);
|
|
1459
|
+
if (recoverable)
|
|
1460
|
+
throw recoverable;
|
|
1069
1461
|
return switched;
|
|
1462
|
+
}
|
|
1070
1463
|
tabEnsured = true;
|
|
1071
1464
|
}
|
|
1072
|
-
|
|
1465
|
+
const result = await run([
|
|
1466
|
+
...sessionArgs(BROWSER_SESSION),
|
|
1467
|
+
...args,
|
|
1468
|
+
]);
|
|
1469
|
+
const recoverable = recoverableAgentBrowserRunFailure(result);
|
|
1470
|
+
if (recoverable)
|
|
1471
|
+
throw recoverable;
|
|
1472
|
+
return result;
|
|
1073
1473
|
};
|
|
1074
1474
|
return body(scoped);
|
|
1475
|
+
}), {
|
|
1476
|
+
signal: ac.signal,
|
|
1477
|
+
reconnect: async () => {
|
|
1478
|
+
await deps.ensureDaemon(BROWSER_SESSION, { signal: ac.signal });
|
|
1479
|
+
},
|
|
1480
|
+
rebindTab: async () => {
|
|
1481
|
+
if (!claimedTabId)
|
|
1482
|
+
return;
|
|
1483
|
+
const rebound = await deps.run(bin, [...sessionArgs(BROWSER_SESSION), 'tab', claimedTabId], ac.signal);
|
|
1484
|
+
if (rebound.code !== 0)
|
|
1485
|
+
throw new Error(rebound.stderr || '浏览器标签页已失效,无法重新绑定');
|
|
1486
|
+
},
|
|
1075
1487
|
});
|
|
1076
1488
|
}
|
|
1077
1489
|
finally {
|
|
@@ -1082,13 +1494,17 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1082
1494
|
};
|
|
1083
1495
|
/** 单条命令 = 只有一步的序列。人工等待之后的收尾快照走这个(此时不该再持锁)。 */
|
|
1084
1496
|
const runAction = (args) => runSequence((run) => run(args));
|
|
1085
|
-
stream.beginAgentAction({ runId, conversationKey, action: input.action });
|
|
1086
1497
|
let pointerShown = false; // C4:本次是否点亮了 AI 指针(finally 淡出用)
|
|
1087
|
-
// ⚠️
|
|
1088
|
-
//
|
|
1089
|
-
//
|
|
1090
|
-
//
|
|
1498
|
+
// ⚠️ 取锁是 try 的**第一行**。这把锁是进程级单例,唯一的释放路径是下面的 finally,
|
|
1499
|
+
// 所以「拿到锁却没能进 try」必须在结构上不可能发生 —— 而不是靠后来者记得别往
|
|
1500
|
+
// begin 与 try 之间插代码。历史上这里正是 begin 在外、旁白(ctx.reportProgress,
|
|
1501
|
+
// 宿主注入的回调)也在外:它一抛错就跳过整个 try/finally,锁再不释放,此后该进程内
|
|
1502
|
+
// **任何**会话都撞「正在执行另一个会话的原子操作」,且现场已经没了。
|
|
1503
|
+
//
|
|
1504
|
+
// 取锁本身失败(锁被别人持有)时 finally 照常执行 finishAgentAction,这是安全的:
|
|
1505
|
+
// finishAgent 校验 state.runId === runId,持有者是别人就不会被误释放。
|
|
1091
1506
|
try {
|
|
1507
|
+
stream.beginAgentAction({ runId, conversationKey, action: input.action });
|
|
1092
1508
|
if (visibleAgentAction) {
|
|
1093
1509
|
stream.setAgentActing(true, input.action, input.selector ?? input.url);
|
|
1094
1510
|
ctx.reportProgress?.(describeAction(input)); // C3:每个可见动作发一句人话旁白到对话/状态行
|
|
@@ -1129,11 +1545,20 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1129
1545
|
clearTimeout(preTimer);
|
|
1130
1546
|
preTimer = null;
|
|
1131
1547
|
}
|
|
1132
|
-
return {
|
|
1548
|
+
return {
|
|
1549
|
+
ok: true,
|
|
1550
|
+
content: JSON.stringify(tabs.filter((tab) => claimed.has(tab.id))),
|
|
1551
|
+
};
|
|
1133
1552
|
}
|
|
1134
1553
|
if (input.action === 'tabSwitch' && input.tabId) {
|
|
1135
|
-
|
|
1136
|
-
|
|
1554
|
+
claimedTabId = await claimConversationTabById({
|
|
1555
|
+
context: browserContext,
|
|
1556
|
+
sessionName: BROWSER_SESSION,
|
|
1557
|
+
bin,
|
|
1558
|
+
globals,
|
|
1559
|
+
runner: deps.run,
|
|
1560
|
+
signal: preCtrl.signal,
|
|
1561
|
+
}, input.tabId);
|
|
1137
1562
|
}
|
|
1138
1563
|
else if (input.action === 'tabClose' && input.tabId) {
|
|
1139
1564
|
browserRunRegistry.assertTab(browserContext, input.tabId);
|
|
@@ -1149,20 +1574,24 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1149
1574
|
signal: preCtrl.signal,
|
|
1150
1575
|
});
|
|
1151
1576
|
}
|
|
1577
|
+
if (input.retention) {
|
|
1578
|
+
browserRunRegistry.markTab(browserContext, claimedTabId, {
|
|
1579
|
+
retention: input.retention,
|
|
1580
|
+
...(input.tabTitle ? { title: input.tabTitle } : {}),
|
|
1581
|
+
});
|
|
1582
|
+
}
|
|
1152
1583
|
if (preTimer) {
|
|
1153
1584
|
clearTimeout(preTimer);
|
|
1154
1585
|
preTimer = null;
|
|
1155
1586
|
}
|
|
1156
1587
|
// 同一 (动作, 选择器) 连续失败到上限 → 拒绝再试,强制换路径。
|
|
1157
1588
|
// 挡的是「原地换着选择器盲猜三五轮然后放弃」这个最常见的失败形状。
|
|
1158
|
-
if (VERIFIED_BROWSER_ACTIONS.has(input.action)
|
|
1159
|
-
|
|
1589
|
+
if (VERIFIED_BROWSER_ACTIONS.has(input.action) &&
|
|
1590
|
+
browserAttempts.failures(conversationKey, input.action, input.selector) >=
|
|
1591
|
+
BROWSER_FAILURE_LIMIT) {
|
|
1160
1592
|
return {
|
|
1161
1593
|
ok: false,
|
|
1162
|
-
error: `[${input.action}] 这个目标(${input.selector ?? '无选择器'})已经连续失败
|
|
1163
|
-
+ `${BROWSER_FAILURE_LIMIT} 次,引擎暂停了对它的重试。\n`
|
|
1164
|
-
+ '请先 snapshot 看清当前页面到底是什么状态(观测不受限制),换一个元素引用或换一条路径;'
|
|
1165
|
-
+ '如果判断不出来,就把现状如实告诉用户,不要继续盲试。',
|
|
1594
|
+
error: `[${input.action}] 这个目标(${input.selector ?? '无选择器'})已经连续失败 ${BROWSER_FAILURE_LIMIT} 次,引擎暂停了对它的重试。\n请先 snapshot 看清当前页面到底是什么状态(观测不受限制),换一个元素引用或换一条路径;如果判断不出来,就把现状如实告诉用户,不要继续盲试。`,
|
|
1166
1595
|
};
|
|
1167
1596
|
}
|
|
1168
1597
|
const outcome = await runSequence(async (run) => {
|
|
@@ -1170,18 +1599,22 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1170
1599
|
// 标签页**上取。缓存不行(navigate 只写「请求的」URL,非监督宿主恒 about:blank,
|
|
1171
1600
|
// 一个 302 就让白名单以为你还在 allowed 站点);锁外也不行(那时标签页还没切)。
|
|
1172
1601
|
if (DEVELOPER_MODE_ACTIONS.has(input.action)) {
|
|
1173
|
-
const originError = requireAllowedDeveloperOrigin(await resolveLiveUrl(run) ?? stream.getCurrentUrl(), browserConfig.allowedCdpOrigins);
|
|
1602
|
+
const originError = requireAllowedDeveloperOrigin((await resolveLiveUrl(run)) ?? stream.getCurrentUrl(), browserConfig.allowedCdpOrigins);
|
|
1174
1603
|
if (originError)
|
|
1175
1604
|
return { blocked: originError };
|
|
1176
1605
|
}
|
|
1177
|
-
if (input.action === 'click' &&
|
|
1606
|
+
if (input.action === 'click' &&
|
|
1607
|
+
input.selector &&
|
|
1608
|
+
input.intent !== 'submit') {
|
|
1178
1609
|
// The model is expected to mark final actions, but safety cannot depend
|
|
1179
1610
|
// solely on model compliance. Inspect the actual target's visible label
|
|
1180
1611
|
// and reject an obvious irreversible click until it is explicitly marked.
|
|
1181
1612
|
const target = await run(['get', 'text', input.selector]);
|
|
1182
1613
|
const targetLabel = `${input.selector}\n${target.code === 0 ? target.stdout : ''}`;
|
|
1183
1614
|
if (isLikelyIrreversibleBrowserTarget(targetLabel)) {
|
|
1184
|
-
return {
|
|
1615
|
+
return {
|
|
1616
|
+
blocked: '该控件看起来会执行发布、提交、购买、删除或发送操作。请仅在用户当前任务已明确授权时,使用 intent="submit" 重试。',
|
|
1617
|
+
};
|
|
1185
1618
|
}
|
|
1186
1619
|
}
|
|
1187
1620
|
// C4:交互动作前点亮 AI 指针——把目标元素中心作为语义终点发给面板,让用户看见「手」移到目标 + 动作动效。
|
|
@@ -1191,7 +1624,12 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1191
1624
|
if (supervised && input.selector && POINTER_ACTIONS.has(input.action)) {
|
|
1192
1625
|
const targetCenter = parsePointerCenter(await run(['get', 'box', input.selector, '--json']));
|
|
1193
1626
|
if (targetCenter) {
|
|
1194
|
-
stream.setAgentPointer({
|
|
1627
|
+
stream.setAgentPointer({
|
|
1628
|
+
x: targetCenter.x,
|
|
1629
|
+
y: targetCenter.y,
|
|
1630
|
+
visible: true,
|
|
1631
|
+
action: pointerActionFor(input.action),
|
|
1632
|
+
});
|
|
1195
1633
|
flashTargetElement(stream, input.selector); // 点亮目标元素绿框,强化「AI 在点这里」
|
|
1196
1634
|
pointerShown = true;
|
|
1197
1635
|
}
|
|
@@ -1231,8 +1669,11 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1231
1669
|
// 再采样纯属白付一次往返 + 一段等待。噪声是页面的持续属性,动作后测与
|
|
1232
1670
|
// 动作前测等价——而放在后面测,这段等待就不再推迟动作本身的发出。
|
|
1233
1671
|
let noiseAfter;
|
|
1234
|
-
if (verified &&
|
|
1235
|
-
|
|
1672
|
+
if (verified &&
|
|
1673
|
+
result.code === 0 &&
|
|
1674
|
+
before !== undefined &&
|
|
1675
|
+
probeSnapshot &&
|
|
1676
|
+
differingLineIndexes(before, probeSnapshot)?.size) {
|
|
1236
1677
|
await sleep(NOISE_BASELINE_GAP_MS, signal);
|
|
1237
1678
|
const again = await run(['snapshot']);
|
|
1238
1679
|
if (again.code === 0)
|
|
@@ -1242,7 +1683,7 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1242
1683
|
});
|
|
1243
1684
|
if (outcome.blocked !== undefined)
|
|
1244
1685
|
return { ok: false, error: outcome.blocked };
|
|
1245
|
-
const { result, before: beforeSnapshot, probeSnapshot, noiseAfter } = outcome;
|
|
1686
|
+
const { result, before: beforeSnapshot, probeSnapshot, noiseAfter, } = outcome;
|
|
1246
1687
|
if (result.code === 0 && input.action === 'tabSwitch') {
|
|
1247
1688
|
browserContext.activeTabId = claimedTabId;
|
|
1248
1689
|
}
|
|
@@ -1262,7 +1703,10 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1262
1703
|
stream.setCurrentUrl(normalizeUrl(input.url));
|
|
1263
1704
|
}
|
|
1264
1705
|
const challengeText = `${result.stdout}\n${result.stderr}\n${probeSnapshot}`;
|
|
1265
|
-
|
|
1706
|
+
// A failed click/navigation is a common way to discover a login, CAPTCHA,
|
|
1707
|
+
// or bot-check page. Do not gate detection on the CLI exit code or the
|
|
1708
|
+
// agent will retry the challenge as if it were an ordinary locator miss.
|
|
1709
|
+
const challenge = detectBrowserChallenge(challengeText);
|
|
1266
1710
|
if (challenge && !supervised) {
|
|
1267
1711
|
// 非受监督宿主(CLI -p / TUI / gateway)无可见浏览器、无人可接管:绝不阻塞等人。
|
|
1268
1712
|
// 拿一张现场快照,连同「需真人验证、当前环境无法越过」如实返回。用 ok:true(非 ok:false)
|
|
@@ -1271,9 +1715,7 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1271
1715
|
const snapshot = probe.code === 0 ? probe.stdout.trim() : '';
|
|
1272
1716
|
return {
|
|
1273
1717
|
ok: true,
|
|
1274
|
-
content: `⚠️ ${challenge.summary}
|
|
1275
|
-
+ `请如实告知用户此页面需人工验证/登录后才能继续,不要反复重试或索要凭据。`
|
|
1276
|
-
+ (snapshot ? `\n\n当前页面快照:\n${snapshot}` : ''),
|
|
1718
|
+
content: `⚠️ ${challenge.summary}。当前为非交互环境(无可见浏览器供真人介入),无法自动越过该关卡。请如实告知用户此页面需人工验证/登录后才能继续,不要反复重试或索要凭据。${snapshot ? `\n\n当前页面快照:\n${snapshot}` : ''}`,
|
|
1277
1719
|
};
|
|
1278
1720
|
}
|
|
1279
1721
|
// challenge 自动暂停 与 用户点「接管」(requestHuman('manual'))都会把控制切到 waiting_human,
|
|
@@ -1290,7 +1732,9 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1290
1732
|
if (pointerShown) {
|
|
1291
1733
|
stream.setAgentPointer({ x: 0, y: 0, visible: false });
|
|
1292
1734
|
}
|
|
1293
|
-
ctx.reportProgress?.(challenge
|
|
1735
|
+
ctx.reportProgress?.(challenge
|
|
1736
|
+
? `${challenge.summary},等待你在浏览器中处理`
|
|
1737
|
+
: '你已接管浏览器,处理完成后点“完成并继续”');
|
|
1294
1738
|
const resolution = await stream.waitForHuman(signal);
|
|
1295
1739
|
if (resolution === 'stopped')
|
|
1296
1740
|
return { ok: false, error: `[${input.action}] 浏览器任务已由用户停止` };
|
|
@@ -1325,10 +1769,9 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1325
1769
|
stream.setAgentCandidates(candidates.map((c) => ({ ref: c.ref, line: c.line })), input.selector);
|
|
1326
1770
|
}
|
|
1327
1771
|
const hint = candidates.length
|
|
1328
|
-
?
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
+ candidates.map((c) => ` ${c.ref} ← ${c.line}`).join('\n')
|
|
1772
|
+
? `\n\n【最接近的候选元素】直接用下面的 @eN 引用重试,不要再猜选择器:\n${candidates
|
|
1773
|
+
.map((c) => ` ${c.ref} ← ${c.line}`)
|
|
1774
|
+
.join('\n')}`
|
|
1332
1775
|
: '';
|
|
1333
1776
|
return {
|
|
1334
1777
|
...base,
|
|
@@ -1339,6 +1782,48 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1339
1782
|
}
|
|
1340
1783
|
return base;
|
|
1341
1784
|
}
|
|
1785
|
+
// WebBrowser's legacy page-assets action used to be only an eval alias.
|
|
1786
|
+
// Materialize the requested bundle at this boundary so the advertised
|
|
1787
|
+
// action actually creates `manifest.json` and asset files.
|
|
1788
|
+
if (input.action === 'bundlePageAssets') {
|
|
1789
|
+
const parsed = parseEvalJson(result.stdout);
|
|
1790
|
+
if (!Array.isArray(parsed)) {
|
|
1791
|
+
return {
|
|
1792
|
+
ok: false,
|
|
1793
|
+
error: '页面资产返回格式无效,无法打包。请先使用 pageAssets 查看页面资源。',
|
|
1794
|
+
};
|
|
1795
|
+
}
|
|
1796
|
+
try {
|
|
1797
|
+
const bundle = await bundlePageAssets(parsed, input.bundleDir, async (url) => {
|
|
1798
|
+
const fetched = parseEvalJson((await runAction(['eval', browserAssetFetchExpression(url)]))
|
|
1799
|
+
.stdout);
|
|
1800
|
+
if (!fetched ||
|
|
1801
|
+
typeof fetched !== 'object' ||
|
|
1802
|
+
fetched.ok !== true ||
|
|
1803
|
+
typeof fetched.base64 !== 'string') {
|
|
1804
|
+
const status = fetched && typeof fetched === 'object'
|
|
1805
|
+
? String(fetched.status ?? 'unknown')
|
|
1806
|
+
: 'unknown';
|
|
1807
|
+
throw new Error(`浏览器上下文抓取资源失败 HTTP ${status}`);
|
|
1808
|
+
}
|
|
1809
|
+
const value = fetched;
|
|
1810
|
+
return new Response(Buffer.from(value.base64, 'base64'), {
|
|
1811
|
+
status: 200,
|
|
1812
|
+
headers: value.type ? { 'content-type': value.type } : undefined,
|
|
1813
|
+
});
|
|
1814
|
+
});
|
|
1815
|
+
return {
|
|
1816
|
+
ok: true,
|
|
1817
|
+
content: clampResult(JSON.stringify({ assets: parsed, bundle }, null, 2)),
|
|
1818
|
+
};
|
|
1819
|
+
}
|
|
1820
|
+
catch (error) {
|
|
1821
|
+
return {
|
|
1822
|
+
ok: false,
|
|
1823
|
+
error: `页面资产打包失败:${error instanceof Error ? error.message : String(error)}`,
|
|
1824
|
+
};
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1342
1827
|
const verdict = VERIFIED_BROWSER_ACTIONS.has(input.action)
|
|
1343
1828
|
? classifyPageEffect(beforeSnapshot, probeSnapshot || undefined, noiseAfter)
|
|
1344
1829
|
: null;
|
|
@@ -1357,22 +1842,27 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1357
1842
|
// 关键节点(navigate 加载完成 / snapshot 读取页面)确定性截当前可见画面,作为「地面真相」附到结果上,
|
|
1358
1843
|
// 由主循环回填成一条带图 user 消息喂视觉模型自我核验。仅基座支持识图时截(否则跳过,不浪费 I/O、不假装核验);
|
|
1359
1844
|
// 截图失败绝不影响原动作结果(纯增强)。
|
|
1360
|
-
if (finalResult.ok &&
|
|
1361
|
-
|
|
1845
|
+
if (finalResult.ok &&
|
|
1846
|
+
KEY_NODE_SHOT_ACTIONS.has(input.action) &&
|
|
1847
|
+
ctx.provider?.supportsVision === true &&
|
|
1848
|
+
!signal?.aborted) {
|
|
1362
1849
|
try {
|
|
1363
1850
|
const shotPath = await createArtifactPath('browser-verify', 'png');
|
|
1364
1851
|
const shot = await runAction(['screenshot', shotPath]);
|
|
1365
1852
|
if (shot.code === 0) {
|
|
1366
|
-
const shots = [
|
|
1853
|
+
const shots = [
|
|
1854
|
+
{ source: shotPath, detail: 'low', maxLongSidePixel: 1400 },
|
|
1855
|
+
];
|
|
1367
1856
|
return {
|
|
1368
1857
|
...finalResult,
|
|
1369
|
-
content: finalResult.content
|
|
1370
|
-
+ '\n\n[已在关键节点截取当前可见页面,见随后的实况画面——请对照核对是否真的到达/呈现了预期状态,不要假设成功。]',
|
|
1858
|
+
content: `${finalResult.content}\n\n[已在关键节点截取当前可见页面,见随后的实况画面——请对照核对是否真的到达/呈现了预期状态,不要假设成功。]`,
|
|
1371
1859
|
visionImages: shots,
|
|
1372
1860
|
};
|
|
1373
1861
|
}
|
|
1374
1862
|
}
|
|
1375
|
-
catch {
|
|
1863
|
+
catch {
|
|
1864
|
+
/* 截图纯增强,失败不影响动作结果 */
|
|
1865
|
+
}
|
|
1376
1866
|
}
|
|
1377
1867
|
return finalResult;
|
|
1378
1868
|
}
|
|
@@ -1380,7 +1870,10 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1380
1870
|
if (signal?.aborted)
|
|
1381
1871
|
return { ok: false, error: `[${input.action}] 浏览器任务已被中断` };
|
|
1382
1872
|
if (error instanceof BrowserTabError) {
|
|
1383
|
-
return {
|
|
1873
|
+
return {
|
|
1874
|
+
ok: false,
|
|
1875
|
+
error: `[${input.action}] ${error.code}: ${error.message}`,
|
|
1876
|
+
};
|
|
1384
1877
|
}
|
|
1385
1878
|
return {
|
|
1386
1879
|
ok: false,
|
|
@@ -1388,14 +1881,29 @@ async function call(input, signal, deps = defaultDeps, ctx = {}) {
|
|
|
1388
1881
|
};
|
|
1389
1882
|
}
|
|
1390
1883
|
finally {
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1884
|
+
// ⚠️ 释放排在**第一位**,且下面两个装饰动作各自吞掉异常。
|
|
1885
|
+
// 它们都会 emit 到宿主注入的 listener(Desktop 面板 / server 推流),listener 一抛错
|
|
1886
|
+
// 就会吃掉它后面的所有语句 —— 排在后面的释放因此永远不执行,锁泄漏。
|
|
1887
|
+
// 一把进程级单例锁的命运不该由 UI 增强的成败决定;反过来,两个装饰之间也不该互相牵连。
|
|
1395
1888
|
stream.finishAgentAction(runId);
|
|
1889
|
+
try {
|
|
1890
|
+
if (visibleAgentAction)
|
|
1891
|
+
stream.setAgentActing(false);
|
|
1892
|
+
}
|
|
1893
|
+
catch {
|
|
1894
|
+
/* 面板装饰,失败不影响动作结果,更不该影响锁 */
|
|
1895
|
+
}
|
|
1896
|
+
try {
|
|
1897
|
+
if (pointerShown)
|
|
1898
|
+
stream.setAgentPointer({ x: 0, y: 0, visible: false }); // C4:动作结束淡出指针
|
|
1899
|
+
}
|
|
1900
|
+
catch {
|
|
1901
|
+
/* 同上 */
|
|
1902
|
+
}
|
|
1396
1903
|
if (preTimer)
|
|
1397
1904
|
clearTimeout(preTimer);
|
|
1398
1905
|
signal?.removeEventListener('abort', abortPreFromParent);
|
|
1906
|
+
browserRunRegistry.finish(conversationKey, runId);
|
|
1399
1907
|
}
|
|
1400
1908
|
}
|
|
1401
1909
|
/** 把「这次操作生效了没有」的结论并进结果。空结论不加噪声。 */
|
|
@@ -1408,9 +1916,33 @@ function appendEffectNote(base, note) {
|
|
|
1408
1916
|
function clampResult(text) {
|
|
1409
1917
|
if (text.length <= DEFAULT_MAX_RESULT_SIZE_CHARS)
|
|
1410
1918
|
return text;
|
|
1411
|
-
return
|
|
1412
|
-
|
|
1413
|
-
|
|
1919
|
+
return `${text.slice(0, DEFAULT_MAX_RESULT_SIZE_CHARS)}\n\n…(输出超过 ${DEFAULT_MAX_RESULT_SIZE_CHARS} 字符已截断;需要完整内容请缩小选择器范围或分段读取)`;
|
|
1920
|
+
}
|
|
1921
|
+
/**
|
|
1922
|
+
* agent-browser returns the value of `eval` as JSON in current releases, but
|
|
1923
|
+
* older versions wrapped JSON strings in a quoted envelope. Keep this
|
|
1924
|
+
* compatibility parsing at the adapter boundary so page-assets remains a
|
|
1925
|
+
* real capability in WebBrowser rather than an alias that only lists assets.
|
|
1926
|
+
*/
|
|
1927
|
+
function parseEvalJson(stdout) {
|
|
1928
|
+
const trimmed = stdout.trim();
|
|
1929
|
+
if (!trimmed)
|
|
1930
|
+
return undefined;
|
|
1931
|
+
try {
|
|
1932
|
+
const value = JSON.parse(trimmed);
|
|
1933
|
+
if (typeof value === 'string') {
|
|
1934
|
+
try {
|
|
1935
|
+
return JSON.parse(value);
|
|
1936
|
+
}
|
|
1937
|
+
catch {
|
|
1938
|
+
return value;
|
|
1939
|
+
}
|
|
1940
|
+
}
|
|
1941
|
+
return value;
|
|
1942
|
+
}
|
|
1943
|
+
catch {
|
|
1944
|
+
return undefined;
|
|
1945
|
+
}
|
|
1414
1946
|
}
|
|
1415
1947
|
function toToolResult(action, r, outPath) {
|
|
1416
1948
|
if (r.code === 0) {
|
|
@@ -1420,7 +1952,10 @@ function toToolResult(action, r, outPath) {
|
|
|
1420
1952
|
// 错误路径截了、withPostActionSnapshot 截了,唯独最常走的这条没截。
|
|
1421
1953
|
const body = clampResult(r.stdout.trim());
|
|
1422
1954
|
if (outPath) {
|
|
1423
|
-
return {
|
|
1955
|
+
return {
|
|
1956
|
+
ok: true,
|
|
1957
|
+
content: body ? `${body}\nSaved: ${outPath}` : `Saved: ${outPath}`,
|
|
1958
|
+
};
|
|
1424
1959
|
}
|
|
1425
1960
|
if (action === 'cookiesGet') {
|
|
1426
1961
|
return { ok: true, content: body || '[]' };
|