minimal-agent 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -5
- package/package.json +62 -5
- package/plugins/computer_use/.claude-plugin/plugin.json +1 -1
- package/plugins/cron-runner/src/commands.js +4 -4
- package/plugins/cron-runner/src/commands.ts +4 -4
- package/plugins/workflow-runner/.claude-plugin/plugin.json +2 -2
- package/plugins/workflow-runner/commands/workflow.md +57 -90
- package/plugins/workflow-runner/commands/workflows.md +4 -4
- package/plugins/workflow-runner/plugin.js +24 -13
- package/plugins/workflow-runner/plugin.ts +28 -13
- package/plugins/workflow-runner/src/discovery.js +160 -0
- package/plugins/workflow-runner/src/discovery.ts +194 -0
- package/plugins/workflow-runner/src/goalAnchor.js +53 -0
- package/plugins/workflow-runner/src/goalAnchor.ts +59 -0
- package/plugins/workflow-runner/src/index.js +204 -185
- package/plugins/workflow-runner/src/index.ts +249 -224
- package/plugins/workflow-runner/src/inputCoercion.js +40 -0
- package/plugins/workflow-runner/src/inputCoercion.ts +43 -0
- package/plugins/workflow-runner/src/journal.js +224 -0
- package/plugins/workflow-runner/src/journal.ts +274 -0
- package/plugins/workflow-runner/src/jsonSchema.js +132 -0
- package/plugins/workflow-runner/src/jsonSchema.ts +162 -0
- package/plugins/workflow-runner/src/moduleResolution.js +64 -0
- package/plugins/workflow-runner/src/moduleResolution.ts +71 -0
- package/plugins/workflow-runner/src/resourceInputs.js +220 -0
- package/plugins/workflow-runner/src/resourceInputs.ts +300 -0
- package/plugins/workflow-runner/src/runner.js +404 -467
- package/plugins/workflow-runner/src/runner.ts +491 -535
- package/plugins/workflow-runner/src/runtime/config.js +46 -0
- package/plugins/workflow-runner/src/runtime/config.ts +69 -0
- package/plugins/workflow-runner/src/runtime/context.js +1304 -0
- package/plugins/workflow-runner/src/runtime/context.ts +1786 -0
- package/plugins/workflow-runner/src/runtime/exec.js +34 -0
- package/plugins/workflow-runner/src/runtime/exec.ts +53 -0
- package/plugins/workflow-runner/src/runtime/flow.js +8 -0
- package/plugins/workflow-runner/src/runtime/flow.ts +12 -0
- package/plugins/workflow-runner/src/runtime/fs.js +27 -0
- package/plugins/workflow-runner/src/runtime/fs.ts +41 -0
- package/plugins/workflow-runner/src/runtime/model.js +6 -0
- package/plugins/workflow-runner/src/runtime/model.ts +8 -0
- package/plugins/workflow-runner/src/runtime/output.js +7 -0
- package/plugins/workflow-runner/src/runtime/output.ts +21 -0
- package/plugins/workflow-runner/src/runtime/scheduler.js +141 -0
- package/plugins/workflow-runner/src/runtime/scheduler.ts +166 -0
- package/plugins/workflow-runner/src/runtime/state.js +16 -0
- package/plugins/workflow-runner/src/runtime/state.ts +19 -0
- package/plugins/workflow-runner/src/runtime/tools.js +3 -0
- package/plugins/workflow-runner/src/runtime/tools.ts +6 -0
- package/plugins/workflow-runner/src/sandbox.js +141 -0
- package/plugins/workflow-runner/src/sandbox.ts +182 -0
- package/plugins/workflow-runner/src/scriptGuard.d.mts +38 -0
- package/plugins/workflow-runner/src/scriptGuard.mjs +449 -0
- package/plugins/workflow-runner/src/taskIdentity.js +172 -152
- package/plugins/workflow-runner/src/taskIdentity.ts +233 -200
- package/plugins/workflow-runner/src/toolRuntime.js +29 -8
- package/plugins/workflow-runner/src/toolRuntime.ts +36 -11
- package/plugins/workflow-runner/src/types.js +7 -105
- package/plugins/workflow-runner/src/types.ts +155 -520
- package/plugins/workflow-runner/src/util.js +47 -0
- package/plugins/workflow-runner/src/util.ts +51 -0
- package/plugins/workflow-runner/src/workflowContext.js +20 -0
- package/plugins/workflow-runner/src/workflowContext.ts +58 -0
- package/plugins/workflow-runner/src/workflowDependencies.js +36 -0
- package/plugins/workflow-runner/src/workflowDependencies.ts +59 -0
- package/plugins/workflow-runner/src/workflowFingerprint.js +118 -0
- package/plugins/workflow-runner/src/workflowFingerprint.ts +122 -0
- package/plugins/workflow-runner/src/workflowState.js +542 -469
- package/plugins/workflow-runner/src/workflowState.ts +639 -589
- package/plugins/workflow-runner/test/api.test.ts +223 -0
- package/plugins/workflow-runner/test/apiCapabilities.test.ts +614 -0
- package/plugins/workflow-runner/test/apiConcurrencyGate.test.ts +104 -0
- package/plugins/workflow-runner/test/apiEcho.test.ts +210 -0
- package/plugins/workflow-runner/test/apiReadText.test.ts +178 -0
- package/plugins/workflow-runner/test/apiResourceLedger.test.ts +184 -0
- package/plugins/workflow-runner/test/childWorkflow.test.ts +304 -0
- package/plugins/workflow-runner/test/discovery.test.ts +234 -0
- package/plugins/workflow-runner/test/goalAnchor.test.ts +106 -0
- package/plugins/workflow-runner/test/goalController.test.ts +317 -0
- package/plugins/workflow-runner/test/journalLanes.test.ts +198 -0
- package/plugins/workflow-runner/test/listResources.test.ts +106 -0
- package/plugins/workflow-runner/test/resourceInputs.test.ts +294 -0
- package/plugins/workflow-runner/test/sandbox.test.ts +126 -0
- package/plugins/workflow-runner/test/scheduler.test.ts +70 -0
- package/plugins/workflow-runner/test/scriptGuard.test.ts +276 -0
- package/plugins/workflow-runner/test/skillDocSync.test.ts +78 -0
- package/plugins/workflow-runner/test/taskIdentity.test.ts +288 -179
- package/plugins/workflow-runner/test/workflowFingerprint.test.ts +106 -0
- package/plugins/workflow-runner/test/workflowState.test.ts +429 -0
- package/skills/pptx/examples/assets/cover-bg.svg +23 -0
- package/skills/pptx/examples/assets/icon-spark.svg +5 -0
- package/skills/skill-creator/scripts/run_eval.py +3 -2
- package/skills/ui-design/SKILL.md +5 -2
- package/skills/ui-design/references/layout-composition.md +1 -1
- package/skills/workflow-creator/SKILL.md +344 -0
- package/skills/workflow-creator/assets/template/workflow.mjs +48 -0
- package/skills/workflow-creator/references/api.md +216 -0
- package/skills/workflow-creator/references/migration.md +103 -0
- package/skills/workflow-creator/references/patterns.md +779 -0
- package/skills/workflow-creator/scripts/list-resources.mjs +205 -0
- package/skills/workflow-creator/scripts/scaffold.mjs +148 -0
- package/skills/workflow-creator/scripts/validate.mjs +275 -0
- package/skills/workflow-creator/scripts/workflow-creator.test.mjs +82 -0
- package/src/advisor/advisorCoordinator.js +503 -0
- package/src/advisor/advisorCoordinator.ts +657 -0
- package/src/cli/configMenu.tsx +0 -1
- package/src/cli/print.js +44 -6
- package/src/cli/print.ts +73 -4
- package/src/cli/streamJson.js +5 -6
- package/src/cli/streamJson.ts +8 -4
- package/src/cli/wizards/advisorWizard.js +4 -2
- package/src/cli/wizards/advisorWizard.tsx +8 -2
- package/src/cli/wizards/contextAdvisorWizard.js +5 -4
- package/src/cli/wizards/contextAdvisorWizard.tsx +7 -6
- package/src/collaboration/taskFlow.js +115 -0
- package/src/collaboration/taskFlow.ts +169 -0
- package/src/config/configDefaults.js +41 -1
- package/src/config/configDefaults.ts +43 -1
- package/src/config/configDiagnostics.ts +2 -2
- package/src/config/configFile.js +40 -9
- package/src/config/configFile.ts +60 -11
- package/src/config/configService.js +443 -74
- package/src/config/configService.ts +723 -142
- package/src/config/numericLimits.js +22 -0
- package/src/config/numericLimits.ts +24 -0
- package/src/config.js +1 -0
- package/src/config.ts +3 -2
- package/src/context/budgetPlan.js +1 -1
- package/src/context/budgetPlan.ts +1 -1
- package/src/context/compact.js +7 -2
- package/src/context/compact.ts +9 -2
- package/src/context/microCompactLite.js +212 -13
- package/src/context/microCompactLite.ts +261 -12
- package/src/context/sessionRegistry.js +8 -1
- package/src/context/sessionRegistry.ts +18 -5
- package/src/context/toolResultArchive.js +72 -0
- package/src/context/toolResultArchive.ts +95 -0
- package/src/goal/errors.js +8 -0
- package/src/goal/errors.ts +18 -0
- package/src/goal/goalRuntime.js +344 -0
- package/src/goal/goalRuntime.ts +466 -0
- package/src/goal/goalStore.js +200 -0
- package/src/goal/goalStore.ts +219 -0
- package/src/goal/index.js +4 -0
- package/src/goal/index.ts +4 -0
- package/src/goal/statusMapping.js +71 -0
- package/src/goal/statusMapping.ts +95 -0
- package/src/goal/types.js +7 -0
- package/src/goal/types.ts +179 -0
- package/src/goal/workflowController.js +306 -0
- package/src/goal/workflowController.ts +360 -0
- package/src/harness/evidenceLedger.js +2 -0
- package/src/harness/evidenceLedger.ts +1 -0
- package/src/harness/stallGuard.js +9 -3
- package/src/harness/stallGuard.ts +7 -3
- package/src/images/imageResolver.js +161 -0
- package/src/images/imageResolver.ts +189 -0
- package/src/infra/verification.js +56 -20
- package/src/infra/verification.ts +87 -20
- package/src/llm/client.js +79 -21
- package/src/llm/client.ts +182 -35
- package/src/llm/gatewayClient.js +201 -0
- package/src/llm/gatewayClient.ts +299 -0
- package/src/llm/messageProvenance.js +46 -0
- package/src/llm/messageProvenance.ts +58 -0
- package/src/llm/modelProvider.js +1 -0
- package/src/llm/modelProvider.ts +22 -0
- package/src/llm/providers/anthropicFormat.js +108 -1
- package/src/llm/providers/anthropicFormat.ts +115 -4
- package/src/llm/providers/openaiFormat.js +9 -0
- package/src/llm/providers/openaiFormat.ts +12 -3
- package/src/loop.js +319 -54
- package/src/loop.ts +373 -57
- package/src/main.js +4 -1
- package/src/main.tsx +4 -1
- package/src/managedExecution/index.js +2 -0
- package/src/managedExecution/index.ts +2 -0
- package/src/managedExecution/registry.js +15 -0
- package/src/managedExecution/registry.ts +20 -0
- package/src/managedExecution/types.js +1 -0
- package/src/managedExecution/types.ts +86 -0
- package/src/modelRef.js +9 -0
- package/src/modelRef.ts +31 -0
- package/src/multimodal.js +14 -28
- package/src/multimodal.ts +17 -33
- package/src/net/lanDiscovery.js +65 -0
- package/src/net/lanDiscovery.ts +82 -0
- package/src/plugin-sdk.js +28 -3
- package/src/plugin-sdk.ts +39 -6
- package/src/plugins/pluginApi.ts +16 -1
- package/src/plugins/pluginLoader.js +4 -0
- package/src/plugins/pluginLoader.ts +6 -0
- package/src/plugins/pluginRunner.js +152 -9
- package/src/plugins/pluginRunner.ts +186 -8
- package/src/plugins/slashFallback.js +113 -0
- package/src/plugins/slashFallback.ts +137 -0
- package/src/prompts/policyRegistry.js +15 -8
- package/src/prompts/policyRegistry.ts +18 -8
- package/src/prompts/promptAssembly.js +103 -12
- package/src/prompts/promptAssembly.ts +114 -12
- package/src/prompts/skillList.js +21 -3
- package/src/prompts/skillList.ts +27 -2
- package/src/prompts/stableSystemPrompt.js +18 -0
- package/src/prompts/stableSystemPrompt.ts +33 -0
- package/src/runtime/agentRuntimeContext.js +8 -2
- package/src/runtime/agentRuntimeContext.ts +17 -4
- package/src/session/runAgentTurn.js +78 -12
- package/src/session/runAgentTurn.ts +105 -23
- package/src/session/runTurn.js +10 -9
- package/src/session/runTurn.ts +17 -8
- package/src/session/sessionState.ts +2 -0
- package/src/team/auth.js +186 -0
- package/src/team/auth.ts +298 -0
- package/src/team/cli.js +105 -0
- package/src/team/cli.ts +122 -0
- package/src/team/onlineToolContract.js +64 -0
- package/src/team/onlineToolContract.ts +109 -0
- package/src/team/toolPolicy.js +36 -0
- package/src/team/toolPolicy.ts +75 -0
- package/src/tools/advisor/advisor.js +148 -37
- package/src/tools/advisor/advisor.ts +161 -38
- package/src/tools/advisor/advisorDecision.js +71 -0
- package/src/tools/advisor/advisorDecision.ts +85 -0
- package/src/tools/agent/agent.js +18 -5
- package/src/tools/agent/agent.ts +21 -7
- package/src/tools/agent/agentControl.js +246 -0
- package/src/tools/agent/agentControl.ts +284 -0
- package/src/tools/agent/agentTaskService.js +40 -1
- package/src/tools/agent/agentTaskService.ts +44 -1
- package/src/tools/agent/agentTypes.js +14 -2
- package/src/tools/agent/agentTypes.ts +15 -2
- package/src/tools/applypatch/applyPatch.js +362 -0
- package/src/tools/applypatch/applyPatch.ts +428 -0
- package/src/tools/bash/bash.js +70 -18
- package/src/tools/bash/bash.ts +75 -20
- package/src/tools/capabilities.js +10 -5
- package/src/tools/capabilities.ts +10 -5
- package/src/tools/computeruse/computerIntent.js +32 -0
- package/src/tools/computeruse/computerIntent.ts +46 -0
- package/src/tools/computeruse/telemetryLog.js +12 -0
- package/src/tools/computeruse/telemetryLog.ts +16 -0
- package/src/tools/computeruse/vision/telemetry.js +1 -0
- package/src/tools/computeruse/vision/telemetry.ts +1 -0
- package/src/tools/effectiveToolSet.js +4 -0
- package/src/tools/effectiveToolSet.ts +4 -0
- package/src/tools/index.js +21 -2
- package/src/tools/index.ts +33 -2
- package/src/tools/powershell/powershell.js +18 -1
- package/src/tools/powershell/powershell.ts +18 -2
- package/src/tools/presentresult/presentresult.js +6 -15
- package/src/tools/presentresult/presentresult.ts +6 -15
- package/src/tools/read/read.js +69 -11
- package/src/tools/read/read.ts +70 -11
- package/src/tools/shared/fileUtils.js +1 -2
- package/src/tools/shared/fileUtils.ts +1 -2
- package/src/tools/shared/permanentDeleteGuard.js +34 -3
- package/src/tools/shared/permanentDeleteGuard.ts +43 -6
- package/src/tools/shared/toolOutputStore.js +23 -1
- package/src/tools/shared/toolOutputStore.ts +22 -1
- package/src/tools/skill/skill.js +9 -3
- package/src/tools/skill/skill.ts +8 -4
- package/src/tools/toolPromptGuides.js +117 -0
- package/src/tools/toolPromptGuides.ts +137 -0
- package/src/tools/webbrowser/agentBrowserPath.js +19 -9
- package/src/tools/webbrowser/agentBrowserPath.ts +70 -28
- package/src/tools/webbrowser/browser.js +27 -11
- package/src/tools/webbrowser/browser.ts +68 -24
- package/src/tools/webbrowser/browserAutomation.js +329 -0
- package/src/tools/webbrowser/browserAutomation.ts +467 -0
- package/src/tools/webbrowser/browserAxState.js +97 -0
- package/src/tools/webbrowser/browserAxState.ts +142 -0
- package/src/tools/webbrowser/browserBatch.js +1578 -0
- package/src/tools/webbrowser/browserBatch.ts +2382 -0
- package/src/tools/webbrowser/browserCdp.js +208 -0
- package/src/tools/webbrowser/browserCdp.ts +255 -0
- package/src/tools/webbrowser/browserConditions.js +24 -0
- package/src/tools/webbrowser/browserConditions.ts +45 -0
- package/src/tools/webbrowser/browserContent.js +281 -0
- package/src/tools/webbrowser/browserContent.ts +438 -0
- package/src/tools/webbrowser/browserControl.js +131 -30
- package/src/tools/webbrowser/browserControl.ts +210 -50
- package/src/tools/webbrowser/browserDaemon.js +22 -14
- package/src/tools/webbrowser/browserDaemon.ts +54 -21
- package/src/tools/webbrowser/browserFrames.js +47 -0
- package/src/tools/webbrowser/browserFrames.ts +54 -0
- package/src/tools/webbrowser/browserInteractionResolver.js +17 -0
- package/src/tools/webbrowser/browserInteractionResolver.ts +40 -0
- package/src/tools/webbrowser/browserLocator.js +140 -0
- package/src/tools/webbrowser/browserLocator.ts +194 -0
- package/src/tools/webbrowser/browserOutcome.js +10 -17
- package/src/tools/webbrowser/browserOutcome.ts +48 -31
- package/src/tools/webbrowser/browserRecovery.js +58 -0
- package/src/tools/webbrowser/browserRecovery.ts +83 -0
- package/src/tools/webbrowser/browserRunRegistry.js +35 -3
- package/src/tools/webbrowser/browserRunRegistry.ts +91 -15
- package/src/tools/webbrowser/browserService.js +28 -2
- package/src/tools/webbrowser/browserService.ts +49 -9
- package/src/tools/webbrowser/browserStream.js +132 -33
- package/src/tools/webbrowser/browserStream.ts +224 -50
- package/src/tools/webbrowser/browserTabs.js +48 -15
- package/src/tools/webbrowser/browserTabs.ts +178 -96
- package/src/tools/webbrowser/webbrowser.js +683 -144
- package/src/tools/webbrowser/webbrowser.ts +979 -236
- package/src/tools/webfetch/preapproved.js +0 -2
- package/src/tools/webfetch/preapproved.ts +0 -2
- package/src/tools/websearch/websearch.js +129 -8
- package/src/tools/websearch/websearch.ts +138 -6
- package/src/types.ts +450 -42
- package/src/ui/App.js +377 -58
- package/src/ui/App.tsx +576 -111
- package/src/ui/LiveArea.js +14 -8
- package/src/ui/LiveArea.tsx +14 -10
- package/src/ui/MessageList.js +63 -20
- package/src/ui/MessageList.tsx +101 -16
- package/src/ui/ModelPicker.js +7 -4
- package/src/ui/ModelPicker.tsx +55 -8
- package/src/ui/hooks/useChat.js +152 -163
- package/src/ui/hooks/useChat.ts +396 -334
- package/src/ui/loginCommand.js +56 -0
- package/src/ui/loginCommand.ts +63 -0
- package/src/ui/toolProgress.js +41 -0
- package/src/ui/toolProgress.ts +44 -0
- package/src/web/browserBridge.js +20 -2
- package/src/web/browserBridge.ts +20 -2
- package/src/web/protocol.ts +1897 -194
- package/workflows/adaptive-research/workflow.mjs +86 -0
- package/workflows/e2e-capabilities/workflow.mjs +102 -0
- package/workflows/e2e-llm/workflow.mjs +52 -0
- package/workflows/e2e-smoke/workflow.mjs +59 -0
- package/workflows/e2e-vision-schema/README.md +48 -0
- package/workflows/e2e-vision-schema/workflow.mjs +149 -0
- package/workflows/e2e-write-greet/workflow.mjs +37 -0
- package/workflows/file-census/README.md +14 -0
- package/workflows/file-census/scripts/.gitkeep +2 -0
- package/workflows/file-census/workflow.mjs +110 -0
- package/workflows/goal/README.md +166 -0
- package/workflows/goal/prompts/audit.md +57 -0
- package/workflows/goal/prompts/contract.md +56 -0
- package/workflows/goal/prompts/iterate.md +37 -0
- package/workflows/goal/workflow.mjs +501 -0
- package/workflows/illustrate/README.md +90 -0
- package/workflows/illustrate/assets/engines.json +20 -0
- package/workflows/illustrate/assets/extract-baidu.js +21 -0
- package/workflows/illustrate/assets/extract-bing.js +33 -0
- package/workflows/illustrate/assets/extract-google.js +31 -0
- package/workflows/illustrate/workflow.mjs +301 -0
- package/workflows/plan/workflow.mjs +389 -0
- package/workflows/question-crops/README.md +35 -0
- package/workflows/question-crops/prompts/inventory.md +23 -0
- package/workflows/question-crops/scripts/image-ops.mjs +305 -0
- package/workflows/question-crops/scripts/image-ops.test.mjs +95 -0
- package/workflows/question-crops/workflow.mjs +172 -0
- package/workflows/ralph-loop/workflow.mjs +162 -0
- package/workflows/research/DESIGN.md +449 -0
- package/workflows/research/README.md +74 -0
- package/workflows/research/prompts/s0-goal.md +43 -0
- package/workflows/research/prompts/s1-early-exit.md +22 -0
- package/workflows/research/prompts/s1-map.md +58 -0
- package/workflows/research/prompts/s2-options.md +53 -0
- package/workflows/research/prompts/s3-deep-dive.md +56 -0
- package/workflows/research/prompts/s4-verify.md +74 -0
- package/workflows/research/prompts/s5-synthesis.md +81 -0
- package/workflows/research/prompts/s5b-fill-gaps.md +32 -0
- package/workflows/research/prompts/s6-package.md +32 -0
- package/workflows/research/scripts/build-report.mjs +98 -0
- package/workflows/research/workflow.mjs +288 -0
- package/workflows/wechat-reading-shots/prompts/collect-params.md +16 -0
- package/workflows/wechat-reading-shots/prompts/evaluate.md +20 -0
- package/workflows/wechat-reading-shots/prompts/open-book.md +28 -0
- package/workflows/wechat-reading-shots/prompts/read-page.md +13 -0
- package/workflows/wechat-reading-shots/prompts/turn-batch.md +24 -0
- package/workflows/wechat-reading-shots/test/control-flow.test.ts +329 -0
- package/workflows/wechat-reading-shots/workflow.mjs +266 -0
- package/workflows/wrong-question-docx/README.md +55 -0
- package/workflows/wrong-question-docx/prompts/figure-qa.md +8 -0
- package/workflows/wrong-question-docx/prompts/transcribe.md +25 -0
- package/workflows/wrong-question-docx/scripts/workbook.mjs +857 -0
- package/workflows/wrong-question-docx/test/control-flow.test.ts +165 -0
- package/workflows/wrong-question-docx/test/workbook.test.mjs +303 -0
- package/workflows/wrong-question-docx/workflow.mjs +485 -0
- package/plugins/workflow-runner/src/agentCommon.js +0 -354
- package/plugins/workflow-runner/src/agentCommon.ts +0 -435
- package/plugins/workflow-runner/src/controlFlow/branch.js +0 -40
- package/plugins/workflow-runner/src/controlFlow/branch.ts +0 -46
- package/plugins/workflow-runner/src/controlFlow/loop.js +0 -211
- package/plugins/workflow-runner/src/controlFlow/loop.ts +0 -248
- package/plugins/workflow-runner/src/controlFlow/pause.js +0 -129
- package/plugins/workflow-runner/src/controlFlow/pause.ts +0 -141
- package/plugins/workflow-runner/src/controlFlow/signal.js +0 -21
- package/plugins/workflow-runner/src/controlFlow/signal.ts +0 -29
- package/plugins/workflow-runner/src/controlFlow/types.js +0 -18
- package/plugins/workflow-runner/src/controlFlow/types.ts +0 -63
- package/plugins/workflow-runner/src/dslManifest.js +0 -72
- package/plugins/workflow-runner/src/dslManifest.ts +0 -127
- package/plugins/workflow-runner/src/expressions.js +0 -618
- package/plugins/workflow-runner/src/expressions.ts +0 -622
- package/plugins/workflow-runner/src/loader.js +0 -573
- package/plugins/workflow-runner/src/loader.ts +0 -721
- package/plugins/workflow-runner/src/multimodal.js +0 -38
- package/plugins/workflow-runner/src/multimodal.ts +0 -64
- package/plugins/workflow-runner/src/recovery.js +0 -119
- package/plugins/workflow-runner/src/recovery.ts +0 -149
- package/plugins/workflow-runner/src/stepExecutors/agent.js +0 -173
- package/plugins/workflow-runner/src/stepExecutors/agent.ts +0 -234
- package/plugins/workflow-runner/src/stepExecutors/assert.js +0 -41
- package/plugins/workflow-runner/src/stepExecutors/assert.ts +0 -50
- package/plugins/workflow-runner/src/stepExecutors/echo.js +0 -31
- package/plugins/workflow-runner/src/stepExecutors/echo.ts +0 -38
- package/plugins/workflow-runner/src/stepExecutors/llm.js +0 -99
- package/plugins/workflow-runner/src/stepExecutors/llm.ts +0 -133
- package/plugins/workflow-runner/src/stepExecutors/set.js +0 -49
- package/plugins/workflow-runner/src/stepExecutors/set.ts +0 -56
- package/plugins/workflow-runner/src/stepExecutors/skill.js +0 -111
- package/plugins/workflow-runner/src/stepExecutors/skill.ts +0 -140
- package/plugins/workflow-runner/src/stepExecutors/tool.js +0 -208
- package/plugins/workflow-runner/src/stepExecutors/tool.ts +0 -256
- package/plugins/workflow-runner/src/stepExecutors/verify.js +0 -58
- package/plugins/workflow-runner/src/stepExecutors/verify.ts +0 -68
- package/plugins/workflow-runner/src/stepPipeline.js +0 -277
- package/plugins/workflow-runner/src/stepPipeline.ts +0 -366
- package/plugins/workflow-runner/test/agent.test.ts +0 -587
- package/plugins/workflow-runner/test/agentLlmError.test.ts +0 -373
- package/plugins/workflow-runner/test/bashBlacklist.sweep.test.ts +0 -65
- package/plugins/workflow-runner/test/captureMisbind.test.ts +0 -99
- package/plugins/workflow-runner/test/cli.e2e.test.ts +0 -147
- package/plugins/workflow-runner/test/context-files.test.ts +0 -320
- package/plugins/workflow-runner/test/e2e.test.ts +0 -370
- package/plugins/workflow-runner/test/engineRobustness.test.ts +0 -226
- package/plugins/workflow-runner/test/expressions.test.ts +0 -397
- package/plugins/workflow-runner/test/foundation.test.ts +0 -126
- package/plugins/workflow-runner/test/graceful.test.ts +0 -139
- package/plugins/workflow-runner/test/greedyInputs.load.test.ts +0 -47
- package/plugins/workflow-runner/test/hostCapability.test.ts +0 -108
- package/plugins/workflow-runner/test/illustrate.load.test.ts +0 -183
- package/plugins/workflow-runner/test/jsonPayload.test.ts +0 -83
- package/plugins/workflow-runner/test/loader.test.ts +0 -617
- package/plugins/workflow-runner/test/maxTurnsRetryDelay.test.ts +0 -33
- package/plugins/workflow-runner/test/multimodal.test.ts +0 -342
- package/plugins/workflow-runner/test/output-schema.test.ts +0 -213
- package/plugins/workflow-runner/test/parseWorkflowArgs.test.ts +0 -87
- package/plugins/workflow-runner/test/pluginRunner.isolation.test.ts +0 -269
- package/plugins/workflow-runner/test/ppt.load.test.ts +0 -1220
- package/plugins/workflow-runner/test/ralphLoop.e2e.test.ts +0 -220
- package/plugins/workflow-runner/test/recovery.test.ts +0 -265
- package/plugins/workflow-runner/test/runner.test.ts +0 -911
- package/plugins/workflow-runner/test/runnerPreflight.test.ts +0 -379
- package/plugins/workflow-runner/test/skillTurnDone.test.ts +0 -71
- package/plugins/workflow-runner/test/stateLifecycle.test.ts +0 -150
- package/plugins/workflow-runner/test/stepPipeline.test.ts +0 -114
- package/plugins/workflow-runner/test/taskArbitration.test.ts +0 -338
- package/plugins/workflow-runner/test/taskLifecycle.test.ts +0 -270
- package/plugins/workflow-runner/test/toolArtifactOverwrite.test.ts +0 -221
- package/plugins/workflow-runner/test/toolCapabilityBoundary.test.ts +0 -29
- package/plugins/workflow-runner/test/toolRuntimeIntegration.test.ts +0 -168
- package/plugins/workflow-runner/test/webppt.load.test.ts +0 -250
- package/plugins/workflow-runner/test/word.load.test.ts +0 -208
- package/plugins/workflow-runner/test/workflows.smoke.test.ts +0 -76
- package/plugins/workflow-runner/test/yamlGuardConditions.test.ts +0 -207
- package/src/advisorState.js +0 -22
- package/src/cli/configWizard.js +0 -293
- package/src/tools/toolProgressBridge.js +0 -48
- package/workflows/adaptive-research.yaml +0 -50
- package/workflows/book-review-short.yaml +0 -125
- package/workflows/e2e-write-greet.yaml +0 -31
- package/workflows/illustrate.yaml +0 -227
- package/workflows/plan.yaml +0 -470
- package/workflows/ppt.yaml +0 -1670
- package/workflows/ralph-loop.yaml +0 -199
- package/workflows/schema.json +0 -176
- package/workflows/webppt.yaml +0 -496
- package/workflows/wechat-reading-shots.yaml +0 -315
- package/workflows/word.yaml +0 -720
- package/workflows/youtube-shorts.yaml +0 -202
|
@@ -1,5 +1,5 @@
|
|
|
1
|
+
import { BrowserControlCoordinator, } from './browserControl.js';
|
|
1
2
|
import { getStreamPort, waitForStreamPort } from './browserDaemon.js';
|
|
2
|
-
import { BrowserControlCoordinator } from './browserControl.js';
|
|
3
3
|
class SessionBrowserStream {
|
|
4
4
|
sessionName;
|
|
5
5
|
socket = null;
|
|
@@ -8,26 +8,39 @@ class SessionBrowserStream {
|
|
|
8
8
|
port = null;
|
|
9
9
|
mode = 'browse';
|
|
10
10
|
currentUrl = 'about:blank';
|
|
11
|
-
status = {
|
|
11
|
+
status = {
|
|
12
|
+
connected: false,
|
|
13
|
+
screencasting: false,
|
|
14
|
+
};
|
|
15
|
+
reconnectTimer = null;
|
|
16
|
+
reconnectAttempt = 0;
|
|
17
|
+
disposed = false;
|
|
12
18
|
picker;
|
|
13
19
|
marked = [];
|
|
14
20
|
control;
|
|
15
21
|
constructor(sessionName) {
|
|
16
22
|
this.sessionName = sessionName;
|
|
17
|
-
this.control = new BrowserControlCoordinator({
|
|
23
|
+
this.control = new BrowserControlCoordinator({
|
|
24
|
+
conversationKey: `browser:${sessionName}`,
|
|
25
|
+
});
|
|
18
26
|
this.control.subscribe((state) => this.emit({ type: 'control_state', state }));
|
|
19
27
|
this.control.subscribeInterventions((event) => this.emit({
|
|
20
|
-
type: event.type === 'started'
|
|
28
|
+
type: event.type === 'started'
|
|
29
|
+
? 'human_intervention_started'
|
|
30
|
+
: 'human_intervention_resolved',
|
|
21
31
|
event,
|
|
22
32
|
}));
|
|
23
33
|
}
|
|
24
34
|
async connect() {
|
|
35
|
+
this.disposed = false;
|
|
25
36
|
if (this.socket?.readyState === WebSocket.OPEN)
|
|
26
37
|
return;
|
|
27
38
|
if (this.connecting)
|
|
28
39
|
return this.connecting;
|
|
29
40
|
this.connecting = (async () => {
|
|
30
|
-
this.port =
|
|
41
|
+
this.port =
|
|
42
|
+
(await getStreamPort(this.sessionName)) ??
|
|
43
|
+
(await waitForStreamPort(this.sessionName));
|
|
31
44
|
await new Promise((resolve, reject) => {
|
|
32
45
|
const socket = new WebSocket(`ws://127.0.0.1:${this.port}`);
|
|
33
46
|
const timer = setTimeout(() => {
|
|
@@ -37,6 +50,7 @@ class SessionBrowserStream {
|
|
|
37
50
|
socket.onopen = () => {
|
|
38
51
|
clearTimeout(timer);
|
|
39
52
|
this.socket = socket;
|
|
53
|
+
this.reconnectAttempt = 0;
|
|
40
54
|
this.status = { ...this.status, connected: true };
|
|
41
55
|
this.emit({ type: 'status', ...this.status });
|
|
42
56
|
resolve();
|
|
@@ -48,16 +62,33 @@ class SessionBrowserStream {
|
|
|
48
62
|
socket.onclose = () => {
|
|
49
63
|
if (this.socket === socket)
|
|
50
64
|
this.socket = null;
|
|
51
|
-
this.status = {
|
|
65
|
+
this.status = {
|
|
66
|
+
...this.status,
|
|
67
|
+
connected: false,
|
|
68
|
+
screencasting: false,
|
|
69
|
+
};
|
|
52
70
|
this.emit({ type: 'status', ...this.status });
|
|
71
|
+
this.scheduleReconnect();
|
|
53
72
|
};
|
|
54
73
|
socket.onmessage = (event) => this.onMessage(event.data);
|
|
55
74
|
});
|
|
56
|
-
})().finally(() => {
|
|
75
|
+
})().finally(() => {
|
|
76
|
+
this.connecting = null;
|
|
77
|
+
});
|
|
57
78
|
return this.connecting;
|
|
58
79
|
}
|
|
59
|
-
disconnect() {
|
|
60
|
-
|
|
80
|
+
disconnect() {
|
|
81
|
+
this.disposed = true;
|
|
82
|
+
if (this.reconnectTimer)
|
|
83
|
+
clearTimeout(this.reconnectTimer);
|
|
84
|
+
this.reconnectTimer = null;
|
|
85
|
+
this.socket?.close();
|
|
86
|
+
this.socket = null;
|
|
87
|
+
}
|
|
88
|
+
dispose() {
|
|
89
|
+
this.disconnect();
|
|
90
|
+
this.listeners.clear();
|
|
91
|
+
}
|
|
61
92
|
send(frame) {
|
|
62
93
|
if (this.socket?.readyState === WebSocket.OPEN)
|
|
63
94
|
this.socket.send(JSON.stringify(frame));
|
|
@@ -66,22 +97,49 @@ class SessionBrowserStream {
|
|
|
66
97
|
this.listeners.add(listener);
|
|
67
98
|
return () => this.listeners.delete(listener);
|
|
68
99
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
100
|
+
publish(event) {
|
|
101
|
+
this.emit(event);
|
|
102
|
+
}
|
|
103
|
+
getStatus() {
|
|
104
|
+
return this.status;
|
|
105
|
+
}
|
|
106
|
+
getPort() {
|
|
107
|
+
return this.port;
|
|
108
|
+
}
|
|
109
|
+
isConnected() {
|
|
110
|
+
return this.socket?.readyState === WebSocket.OPEN;
|
|
111
|
+
}
|
|
112
|
+
setMode(mode) {
|
|
113
|
+
this.mode = mode;
|
|
114
|
+
this.emit({ type: 'mode', mode });
|
|
115
|
+
}
|
|
116
|
+
getMode() {
|
|
117
|
+
return this.mode;
|
|
118
|
+
}
|
|
119
|
+
getCurrentUrl() {
|
|
120
|
+
return this.currentUrl;
|
|
121
|
+
}
|
|
122
|
+
setCurrentUrl(url) {
|
|
123
|
+
this.currentUrl = url;
|
|
124
|
+
this.emit({ type: 'url', url });
|
|
125
|
+
}
|
|
126
|
+
setPicker(picker) {
|
|
127
|
+
this.picker = picker;
|
|
128
|
+
}
|
|
77
129
|
pickElement(x, y) {
|
|
78
130
|
return this.picker ? this.picker(x, y) : Promise.resolve(null);
|
|
79
131
|
}
|
|
80
132
|
markElement(element) {
|
|
81
|
-
this.marked = [
|
|
133
|
+
this.marked = [
|
|
134
|
+
...this.marked.filter((item) => item.ref !== element.ref),
|
|
135
|
+
element,
|
|
136
|
+
];
|
|
82
137
|
this.emit({ type: 'marked_elements', elements: this.marked });
|
|
83
138
|
}
|
|
84
|
-
clearMarkedElements() {
|
|
139
|
+
clearMarkedElements() {
|
|
140
|
+
this.marked = [];
|
|
141
|
+
this.emit({ type: 'marked_elements', elements: [] });
|
|
142
|
+
}
|
|
85
143
|
unmarkElement(ref) {
|
|
86
144
|
const next = this.marked.filter((item) => item.ref !== ref);
|
|
87
145
|
if (next.length !== this.marked.length) {
|
|
@@ -102,29 +160,67 @@ class SessionBrowserStream {
|
|
|
102
160
|
// 只发语义终点(目标元素中心 + 动作类型);平滑移动交给面板 CSS 过渡,内核不做逐帧插值(避免 WS 洪泛 + 双层缓动打架)。
|
|
103
161
|
this.emit({ type: 'agent_pointer', ...pointer });
|
|
104
162
|
}
|
|
105
|
-
getControlState() {
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
163
|
+
getControlState() {
|
|
164
|
+
return this.control.getState();
|
|
165
|
+
}
|
|
166
|
+
subscribeControl(listener) {
|
|
167
|
+
return this.control.subscribe(listener);
|
|
168
|
+
}
|
|
169
|
+
subscribeInterventions(listener) {
|
|
170
|
+
return this.control.subscribeInterventions(listener);
|
|
171
|
+
}
|
|
172
|
+
beginAgentAction(input) {
|
|
173
|
+
this.control.beginAgent(input);
|
|
174
|
+
}
|
|
175
|
+
finishAgentAction(runId) {
|
|
176
|
+
this.control.finishAgent(runId);
|
|
177
|
+
}
|
|
178
|
+
requestHuman(reason, input) {
|
|
179
|
+
return this.control.requestHuman(reason, input);
|
|
180
|
+
}
|
|
181
|
+
takeControl(input) {
|
|
182
|
+
this.control.takeInteractive(input);
|
|
183
|
+
}
|
|
184
|
+
resumeControl(token) {
|
|
185
|
+
return this.control.resume(token);
|
|
186
|
+
}
|
|
187
|
+
stopControl(token) {
|
|
188
|
+
return this.control.stop(token);
|
|
189
|
+
}
|
|
114
190
|
resumeIntervention(credential) {
|
|
115
191
|
return this.control.resumeIntervention(credential);
|
|
116
192
|
}
|
|
117
193
|
cancelIntervention(credential) {
|
|
118
194
|
return this.control.cancelIntervention(credential);
|
|
119
195
|
}
|
|
120
|
-
waitForHuman(signal) {
|
|
196
|
+
waitForHuman(signal) {
|
|
197
|
+
return this.control.waitForHuman(signal);
|
|
198
|
+
}
|
|
121
199
|
emit(event) {
|
|
122
|
-
for (const listener of this.listeners)
|
|
123
|
-
|
|
200
|
+
for (const listener of this.listeners) {
|
|
201
|
+
try {
|
|
202
|
+
listener(event);
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
// A disconnected/failed UI observer must not break browser control or
|
|
206
|
+
// prevent the remaining observers from receiving the event.
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
scheduleReconnect() {
|
|
211
|
+
if (this.disposed || !this.listeners.size || this.reconnectTimer)
|
|
212
|
+
return;
|
|
213
|
+
const delay = Math.min(10_000, 250 * 2 ** this.reconnectAttempt++);
|
|
214
|
+
this.reconnectTimer = setTimeout(() => {
|
|
215
|
+
this.reconnectTimer = null;
|
|
216
|
+
void this.connect().catch(() => this.scheduleReconnect());
|
|
217
|
+
}, delay);
|
|
124
218
|
}
|
|
125
219
|
onMessage(data) {
|
|
126
220
|
try {
|
|
127
|
-
const text = typeof data === 'string'
|
|
221
|
+
const text = typeof data === 'string'
|
|
222
|
+
? data
|
|
223
|
+
: Buffer.from(data).toString('utf8');
|
|
128
224
|
const event = JSON.parse(text);
|
|
129
225
|
if (event.type === 'url' && typeof event.url === 'string')
|
|
130
226
|
this.currentUrl = event.url;
|
|
@@ -133,7 +229,10 @@ class SessionBrowserStream {
|
|
|
133
229
|
this.emit(event);
|
|
134
230
|
}
|
|
135
231
|
catch (error) {
|
|
136
|
-
this.emit({
|
|
232
|
+
this.emit({
|
|
233
|
+
type: 'error',
|
|
234
|
+
message: error instanceof Error ? error.message : String(error),
|
|
235
|
+
});
|
|
137
236
|
}
|
|
138
237
|
}
|
|
139
238
|
}
|
|
@@ -1,6 +1,13 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
HumanInterventionCredential,
|
|
3
|
+
HumanInterventionEvent,
|
|
4
|
+
} from '../../humanIntervention.js';
|
|
5
|
+
import {
|
|
6
|
+
BrowserControlCoordinator,
|
|
7
|
+
type BrowserControlState,
|
|
8
|
+
type BrowserHumanReason,
|
|
9
|
+
} from './browserControl.js';
|
|
1
10
|
import { getStreamPort, waitForStreamPort } from './browserDaemon.js';
|
|
2
|
-
import { BrowserControlCoordinator, type BrowserControlState, type BrowserHumanReason } from './browserControl.js';
|
|
3
|
-
import type { HumanInterventionCredential, HumanInterventionEvent } from '../../humanIntervention.js';
|
|
4
11
|
|
|
5
12
|
export type BrowserMode = 'browse' | 'assist' | 'auto';
|
|
6
13
|
export type PickedBrowserElement = {
|
|
@@ -12,7 +19,10 @@ export type PickedBrowserElement = {
|
|
|
12
19
|
boundingBox?: { x: number; y: number; width: number; height: number };
|
|
13
20
|
attributes?: Record<string, string>;
|
|
14
21
|
};
|
|
15
|
-
export type MarkedBrowserElement = PickedBrowserElement & {
|
|
22
|
+
export type MarkedBrowserElement = PickedBrowserElement & {
|
|
23
|
+
label?: string;
|
|
24
|
+
color?: string;
|
|
25
|
+
};
|
|
16
26
|
export type BrowserStreamEvent = Record<string, unknown> & { type: string };
|
|
17
27
|
|
|
18
28
|
export interface BrowserStream {
|
|
@@ -21,6 +31,8 @@ export interface BrowserStream {
|
|
|
21
31
|
send(frame: unknown): void;
|
|
22
32
|
dispose(): void;
|
|
23
33
|
subscribe(listener: (event: BrowserStreamEvent) => void): () => void;
|
|
34
|
+
/** Publish a local engine event to Desktop observers (downloads/CDP lifecycle). */
|
|
35
|
+
publish(event: BrowserStreamEvent): void;
|
|
24
36
|
getStatus(): Record<string, unknown>;
|
|
25
37
|
getPort(): number | null;
|
|
26
38
|
isConnected(): boolean;
|
|
@@ -29,7 +41,9 @@ export interface BrowserStream {
|
|
|
29
41
|
getCurrentUrl(): string;
|
|
30
42
|
setCurrentUrl(url: string): void;
|
|
31
43
|
pickElement(x: number, y: number): Promise<PickedBrowserElement | null>;
|
|
32
|
-
setPicker(
|
|
44
|
+
setPicker(
|
|
45
|
+
picker: (x: number, y: number) => Promise<PickedBrowserElement | null>,
|
|
46
|
+
): void;
|
|
33
47
|
markElement(element: MarkedBrowserElement): void;
|
|
34
48
|
clearMarkedElements(): void;
|
|
35
49
|
/** Remove a single marked element by ref (does not affect other marks). */
|
|
@@ -41,7 +55,10 @@ export interface BrowserStream {
|
|
|
41
55
|
* 候选只进对话的话,盯着浏览器的人看到的是 AI 忽然停住;推到面板上他可以一眼看出
|
|
42
56
|
* AI 卡在哪,并点一下把正确的那个喂回去 —— 人的判断比模型再猜一轮便宜得多。
|
|
43
57
|
*/
|
|
44
|
-
setAgentCandidates(
|
|
58
|
+
setAgentCandidates(
|
|
59
|
+
candidates: Array<{ ref: string; line: string }>,
|
|
60
|
+
selector?: string,
|
|
61
|
+
): void;
|
|
45
62
|
/**
|
|
46
63
|
* 发布一次动作**做完之后**的生效结论(只在结论不是「变了」时调用)。
|
|
47
64
|
*
|
|
@@ -55,14 +72,34 @@ export interface BrowserStream {
|
|
|
55
72
|
maskedLines?: number;
|
|
56
73
|
}): void;
|
|
57
74
|
/** C4:发布 AI 指针语义终点给可见面板(宿主用 CSS 过渡平滑移动 + 渲染点击动效)。 */
|
|
58
|
-
setAgentPointer(pointer: {
|
|
75
|
+
setAgentPointer(pointer: {
|
|
76
|
+
x: number;
|
|
77
|
+
y: number;
|
|
78
|
+
visible: boolean;
|
|
79
|
+
action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle';
|
|
80
|
+
elementRef?: string;
|
|
81
|
+
}): void;
|
|
59
82
|
getControlState(): BrowserControlState;
|
|
60
83
|
/** 订阅控制权状态变化(立即以当前状态回调一次);供工具即时感知「用户接管 / 进入 waiting_human」以掐断在跑动作。 */
|
|
61
84
|
subscribeControl(listener: (state: BrowserControlState) => void): () => void;
|
|
62
|
-
subscribeInterventions(
|
|
63
|
-
|
|
85
|
+
subscribeInterventions(
|
|
86
|
+
listener: (event: HumanInterventionEvent) => void,
|
|
87
|
+
): () => void;
|
|
88
|
+
beginAgentAction(input: {
|
|
89
|
+
runId: string;
|
|
90
|
+
conversationKey?: string;
|
|
91
|
+
action?: string;
|
|
92
|
+
}): void;
|
|
64
93
|
finishAgentAction(runId: string): void;
|
|
65
|
-
requestHuman(
|
|
94
|
+
requestHuman(
|
|
95
|
+
reason: BrowserHumanReason,
|
|
96
|
+
input?: {
|
|
97
|
+
runId?: string;
|
|
98
|
+
conversationKey?: string;
|
|
99
|
+
ownerId?: string;
|
|
100
|
+
summary?: string;
|
|
101
|
+
},
|
|
102
|
+
): string;
|
|
66
103
|
takeControl(input?: { runId?: string; conversationKey?: string }): void;
|
|
67
104
|
resumeControl(token?: string): boolean;
|
|
68
105
|
stopControl(token?: string): boolean;
|
|
@@ -78,25 +115,46 @@ class SessionBrowserStream implements BrowserStream {
|
|
|
78
115
|
private port: number | null = null;
|
|
79
116
|
private mode: BrowserMode = 'browse';
|
|
80
117
|
private currentUrl = 'about:blank';
|
|
81
|
-
private status: Record<string, unknown> = {
|
|
82
|
-
|
|
118
|
+
private status: Record<string, unknown> = {
|
|
119
|
+
connected: false,
|
|
120
|
+
screencasting: false,
|
|
121
|
+
};
|
|
122
|
+
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
|
123
|
+
private reconnectAttempt = 0;
|
|
124
|
+
private disposed = false;
|
|
125
|
+
private picker?: (
|
|
126
|
+
x: number,
|
|
127
|
+
y: number,
|
|
128
|
+
) => Promise<PickedBrowserElement | null>;
|
|
83
129
|
private marked: MarkedBrowserElement[] = [];
|
|
84
130
|
private control: BrowserControlCoordinator;
|
|
85
131
|
|
|
86
132
|
constructor(private readonly sessionName: string) {
|
|
87
|
-
this.control = new BrowserControlCoordinator({
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
133
|
+
this.control = new BrowserControlCoordinator({
|
|
134
|
+
conversationKey: `browser:${sessionName}`,
|
|
135
|
+
});
|
|
136
|
+
this.control.subscribe((state) =>
|
|
137
|
+
this.emit({ type: 'control_state', state }),
|
|
138
|
+
);
|
|
139
|
+
this.control.subscribeInterventions((event) =>
|
|
140
|
+
this.emit({
|
|
141
|
+
type:
|
|
142
|
+
event.type === 'started'
|
|
143
|
+
? 'human_intervention_started'
|
|
144
|
+
: 'human_intervention_resolved',
|
|
145
|
+
event,
|
|
146
|
+
}),
|
|
147
|
+
);
|
|
93
148
|
}
|
|
94
149
|
|
|
95
150
|
async connect(): Promise<void> {
|
|
151
|
+
this.disposed = false;
|
|
96
152
|
if (this.socket?.readyState === WebSocket.OPEN) return;
|
|
97
153
|
if (this.connecting) return this.connecting;
|
|
98
154
|
this.connecting = (async () => {
|
|
99
|
-
this.port =
|
|
155
|
+
this.port =
|
|
156
|
+
(await getStreamPort(this.sessionName)) ??
|
|
157
|
+
(await waitForStreamPort(this.sessionName));
|
|
100
158
|
await new Promise<void>((resolve, reject) => {
|
|
101
159
|
const socket = new WebSocket(`ws://127.0.0.1:${this.port}`);
|
|
102
160
|
const timer = setTimeout(() => {
|
|
@@ -106,6 +164,7 @@ class SessionBrowserStream implements BrowserStream {
|
|
|
106
164
|
socket.onopen = () => {
|
|
107
165
|
clearTimeout(timer);
|
|
108
166
|
this.socket = socket;
|
|
167
|
+
this.reconnectAttempt = 0;
|
|
109
168
|
this.status = { ...this.status, connected: true };
|
|
110
169
|
this.emit({ type: 'status', ...this.status });
|
|
111
170
|
resolve();
|
|
@@ -116,40 +175,86 @@ class SessionBrowserStream implements BrowserStream {
|
|
|
116
175
|
};
|
|
117
176
|
socket.onclose = () => {
|
|
118
177
|
if (this.socket === socket) this.socket = null;
|
|
119
|
-
this.status = {
|
|
178
|
+
this.status = {
|
|
179
|
+
...this.status,
|
|
180
|
+
connected: false,
|
|
181
|
+
screencasting: false,
|
|
182
|
+
};
|
|
120
183
|
this.emit({ type: 'status', ...this.status });
|
|
184
|
+
this.scheduleReconnect();
|
|
121
185
|
};
|
|
122
186
|
socket.onmessage = (event) => this.onMessage(event.data);
|
|
123
187
|
});
|
|
124
|
-
})().finally(() => {
|
|
188
|
+
})().finally(() => {
|
|
189
|
+
this.connecting = null;
|
|
190
|
+
});
|
|
125
191
|
return this.connecting;
|
|
126
192
|
}
|
|
127
193
|
|
|
128
|
-
disconnect(): void {
|
|
129
|
-
|
|
194
|
+
disconnect(): void {
|
|
195
|
+
this.disposed = true;
|
|
196
|
+
if (this.reconnectTimer) clearTimeout(this.reconnectTimer);
|
|
197
|
+
this.reconnectTimer = null;
|
|
198
|
+
this.socket?.close();
|
|
199
|
+
this.socket = null;
|
|
200
|
+
}
|
|
201
|
+
dispose(): void {
|
|
202
|
+
this.disconnect();
|
|
203
|
+
this.listeners.clear();
|
|
204
|
+
}
|
|
130
205
|
send(frame: unknown): void {
|
|
131
|
-
if (this.socket?.readyState === WebSocket.OPEN)
|
|
206
|
+
if (this.socket?.readyState === WebSocket.OPEN)
|
|
207
|
+
this.socket.send(JSON.stringify(frame));
|
|
132
208
|
}
|
|
133
209
|
subscribe(listener: (event: BrowserStreamEvent) => void): () => void {
|
|
134
210
|
this.listeners.add(listener);
|
|
135
211
|
return () => this.listeners.delete(listener);
|
|
136
212
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
213
|
+
publish(event: BrowserStreamEvent): void {
|
|
214
|
+
this.emit(event);
|
|
215
|
+
}
|
|
216
|
+
getStatus(): Record<string, unknown> {
|
|
217
|
+
return this.status;
|
|
218
|
+
}
|
|
219
|
+
getPort(): number | null {
|
|
220
|
+
return this.port;
|
|
221
|
+
}
|
|
222
|
+
isConnected(): boolean {
|
|
223
|
+
return this.socket?.readyState === WebSocket.OPEN;
|
|
224
|
+
}
|
|
225
|
+
setMode(mode: BrowserMode): void {
|
|
226
|
+
this.mode = mode;
|
|
227
|
+
this.emit({ type: 'mode', mode });
|
|
228
|
+
}
|
|
229
|
+
getMode(): BrowserMode {
|
|
230
|
+
return this.mode;
|
|
231
|
+
}
|
|
232
|
+
getCurrentUrl(): string {
|
|
233
|
+
return this.currentUrl;
|
|
234
|
+
}
|
|
235
|
+
setCurrentUrl(url: string): void {
|
|
236
|
+
this.currentUrl = url;
|
|
237
|
+
this.emit({ type: 'url', url });
|
|
238
|
+
}
|
|
239
|
+
setPicker(
|
|
240
|
+
picker: (x: number, y: number) => Promise<PickedBrowserElement | null>,
|
|
241
|
+
): void {
|
|
242
|
+
this.picker = picker;
|
|
243
|
+
}
|
|
145
244
|
pickElement(x: number, y: number): Promise<PickedBrowserElement | null> {
|
|
146
245
|
return this.picker ? this.picker(x, y) : Promise.resolve(null);
|
|
147
246
|
}
|
|
148
247
|
markElement(element: MarkedBrowserElement): void {
|
|
149
|
-
this.marked = [
|
|
248
|
+
this.marked = [
|
|
249
|
+
...this.marked.filter((item) => item.ref !== element.ref),
|
|
250
|
+
element,
|
|
251
|
+
];
|
|
150
252
|
this.emit({ type: 'marked_elements', elements: this.marked });
|
|
151
253
|
}
|
|
152
|
-
clearMarkedElements(): void {
|
|
254
|
+
clearMarkedElements(): void {
|
|
255
|
+
this.marked = [];
|
|
256
|
+
this.emit({ type: 'marked_elements', elements: [] });
|
|
257
|
+
}
|
|
153
258
|
unmarkElement(ref: string): void {
|
|
154
259
|
const next = this.marked.filter((item) => item.ref !== ref);
|
|
155
260
|
if (next.length !== this.marked.length) {
|
|
@@ -160,7 +265,10 @@ class SessionBrowserStream implements BrowserStream {
|
|
|
160
265
|
setAgentActing(acting: boolean, action?: string, target?: string): void {
|
|
161
266
|
this.emit({ type: 'agent_acting', acting, action, target });
|
|
162
267
|
}
|
|
163
|
-
setAgentCandidates(
|
|
268
|
+
setAgentCandidates(
|
|
269
|
+
candidates: Array<{ ref: string; line: string }>,
|
|
270
|
+
selector?: string,
|
|
271
|
+
): void {
|
|
164
272
|
this.emit({ type: 'agent_candidates', candidates, selector });
|
|
165
273
|
}
|
|
166
274
|
setAgentOutcome(outcome: {
|
|
@@ -171,39 +279,102 @@ class SessionBrowserStream implements BrowserStream {
|
|
|
171
279
|
}): void {
|
|
172
280
|
this.emit({ type: 'agent_outcome', ...outcome });
|
|
173
281
|
}
|
|
174
|
-
setAgentPointer(pointer: {
|
|
282
|
+
setAgentPointer(pointer: {
|
|
283
|
+
x: number;
|
|
284
|
+
y: number;
|
|
285
|
+
visible: boolean;
|
|
286
|
+
action?: 'move' | 'click' | 'type' | 'hover' | 'scroll' | 'idle';
|
|
287
|
+
elementRef?: string;
|
|
288
|
+
}): void {
|
|
175
289
|
// 只发语义终点(目标元素中心 + 动作类型);平滑移动交给面板 CSS 过渡,内核不做逐帧插值(避免 WS 洪泛 + 双层缓动打架)。
|
|
176
290
|
this.emit({ type: 'agent_pointer', ...pointer });
|
|
177
291
|
}
|
|
178
|
-
getControlState(): BrowserControlState {
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
292
|
+
getControlState(): BrowserControlState {
|
|
293
|
+
return this.control.getState();
|
|
294
|
+
}
|
|
295
|
+
subscribeControl(listener: (state: BrowserControlState) => void): () => void {
|
|
296
|
+
return this.control.subscribe(listener);
|
|
297
|
+
}
|
|
298
|
+
subscribeInterventions(
|
|
299
|
+
listener: (event: HumanInterventionEvent) => void,
|
|
300
|
+
): () => void {
|
|
301
|
+
return this.control.subscribeInterventions(listener);
|
|
302
|
+
}
|
|
303
|
+
beginAgentAction(input: {
|
|
304
|
+
runId: string;
|
|
305
|
+
conversationKey?: string;
|
|
306
|
+
action?: string;
|
|
307
|
+
}): void {
|
|
308
|
+
this.control.beginAgent(input);
|
|
309
|
+
}
|
|
310
|
+
finishAgentAction(runId: string): void {
|
|
311
|
+
this.control.finishAgent(runId);
|
|
312
|
+
}
|
|
313
|
+
requestHuman(
|
|
314
|
+
reason: BrowserHumanReason,
|
|
315
|
+
input?: {
|
|
316
|
+
runId?: string;
|
|
317
|
+
conversationKey?: string;
|
|
318
|
+
ownerId?: string;
|
|
319
|
+
summary?: string;
|
|
320
|
+
},
|
|
321
|
+
): string {
|
|
322
|
+
return this.control.requestHuman(reason, input);
|
|
323
|
+
}
|
|
324
|
+
takeControl(input?: { runId?: string; conversationKey?: string }): void {
|
|
325
|
+
this.control.takeInteractive(input);
|
|
326
|
+
}
|
|
327
|
+
resumeControl(token?: string): boolean {
|
|
328
|
+
return this.control.resume(token);
|
|
329
|
+
}
|
|
330
|
+
stopControl(token?: string): boolean {
|
|
331
|
+
return this.control.stop(token);
|
|
332
|
+
}
|
|
187
333
|
resumeIntervention(credential: HumanInterventionCredential): boolean {
|
|
188
334
|
return this.control.resumeIntervention(credential);
|
|
189
335
|
}
|
|
190
336
|
cancelIntervention(credential: HumanInterventionCredential): boolean {
|
|
191
337
|
return this.control.cancelIntervention(credential);
|
|
192
338
|
}
|
|
193
|
-
waitForHuman(signal?: AbortSignal): Promise<'resumed' | 'stopped'> {
|
|
339
|
+
waitForHuman(signal?: AbortSignal): Promise<'resumed' | 'stopped'> {
|
|
340
|
+
return this.control.waitForHuman(signal);
|
|
341
|
+
}
|
|
194
342
|
|
|
195
343
|
private emit(event: BrowserStreamEvent): void {
|
|
196
|
-
for (const listener of this.listeners)
|
|
344
|
+
for (const listener of this.listeners) {
|
|
345
|
+
try {
|
|
346
|
+
listener(event);
|
|
347
|
+
} catch {
|
|
348
|
+
// A disconnected/failed UI observer must not break browser control or
|
|
349
|
+
// prevent the remaining observers from receiving the event.
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
private scheduleReconnect(): void {
|
|
355
|
+
if (this.disposed || !this.listeners.size || this.reconnectTimer) return;
|
|
356
|
+
const delay = Math.min(10_000, 250 * 2 ** this.reconnectAttempt++);
|
|
357
|
+
this.reconnectTimer = setTimeout(() => {
|
|
358
|
+
this.reconnectTimer = null;
|
|
359
|
+
void this.connect().catch(() => this.scheduleReconnect());
|
|
360
|
+
}, delay);
|
|
197
361
|
}
|
|
198
362
|
private onMessage(data: unknown): void {
|
|
199
363
|
try {
|
|
200
|
-
const text =
|
|
364
|
+
const text =
|
|
365
|
+
typeof data === 'string'
|
|
366
|
+
? data
|
|
367
|
+
: Buffer.from(data as ArrayBuffer).toString('utf8');
|
|
201
368
|
const event = JSON.parse(text) as BrowserStreamEvent;
|
|
202
|
-
if (event.type === 'url' && typeof event.url === 'string')
|
|
369
|
+
if (event.type === 'url' && typeof event.url === 'string')
|
|
370
|
+
this.currentUrl = event.url;
|
|
203
371
|
if (event.type === 'status') this.status = { ...this.status, ...event };
|
|
204
372
|
this.emit(event);
|
|
205
373
|
} catch (error) {
|
|
206
|
-
this.emit({
|
|
374
|
+
this.emit({
|
|
375
|
+
type: 'error',
|
|
376
|
+
message: error instanceof Error ? error.message : String(error),
|
|
377
|
+
});
|
|
207
378
|
}
|
|
208
379
|
}
|
|
209
380
|
}
|
|
@@ -211,6 +382,9 @@ class SessionBrowserStream implements BrowserStream {
|
|
|
211
382
|
const streams = new Map<string, SessionBrowserStream>();
|
|
212
383
|
export function getBrowserStream(sessionName = 'minimal-agent'): BrowserStream {
|
|
213
384
|
let stream = streams.get(sessionName);
|
|
214
|
-
if (!stream) {
|
|
385
|
+
if (!stream) {
|
|
386
|
+
stream = new SessionBrowserStream(sessionName);
|
|
387
|
+
streams.set(sessionName, stream);
|
|
388
|
+
}
|
|
215
389
|
return stream;
|
|
216
390
|
}
|