@beeos-ai/device-agent 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.pi/extensions/android-tools.ts +139 -0
- package/.pi/skills/.gitkeep +0 -0
- package/.pi/skills/ground-mobile-fallback/SKILL.md +50 -0
- package/LICENSE +201 -0
- package/dist/action/handler.d.ts +9 -0
- package/dist/action/handler.js +124 -0
- package/dist/action/handler.js.map +1 -0
- package/dist/adapters/agent-gateway-client.d.ts +256 -0
- package/dist/adapters/agent-gateway-client.js +332 -0
- package/dist/adapters/agent-gateway-client.js.map +1 -0
- package/dist/adapters/agentcore-inner-loop.d.ts +29 -0
- package/dist/adapters/agentcore-inner-loop.js +72 -0
- package/dist/adapters/agentcore-inner-loop.js.map +1 -0
- package/dist/adapters/agentcore-pi.d.ts +215 -0
- package/dist/adapters/agentcore-pi.js +464 -0
- package/dist/adapters/agentcore-pi.js.map +1 -0
- package/dist/adapters/agentcore-runner.d.ts +70 -0
- package/dist/adapters/agentcore-runner.js +255 -0
- package/dist/adapters/agentcore-runner.js.map +1 -0
- package/dist/adapters/device-android-adb.d.ts +41 -0
- package/dist/adapters/device-android-adb.js +279 -0
- package/dist/adapters/device-android-adb.js.map +1 -0
- package/dist/adapters/device-desktop.d.ts +37 -0
- package/dist/adapters/device-desktop.js +101 -0
- package/dist/adapters/device-desktop.js.map +1 -0
- package/dist/adapters/identity-file.d.ts +30 -0
- package/dist/adapters/identity-file.js +50 -0
- package/dist/adapters/identity-file.js.map +1 -0
- package/dist/adapters/transport-bridge-ws.d.ts +39 -0
- package/dist/adapters/transport-bridge-ws.js +49 -0
- package/dist/adapters/transport-bridge-ws.js.map +1 -0
- package/dist/adapters/vlm-inner-loop.d.ts +88 -0
- package/dist/adapters/vlm-inner-loop.js +199 -0
- package/dist/adapters/vlm-inner-loop.js.map +1 -0
- package/dist/agent-runtime/adapters/base.d.ts +145 -0
- package/dist/agent-runtime/adapters/base.js +29 -0
- package/dist/agent-runtime/adapters/base.js.map +1 -0
- package/dist/agent-runtime/adapters/index.d.ts +17 -0
- package/dist/agent-runtime/adapters/index.js +17 -0
- package/dist/agent-runtime/adapters/index.js.map +1 -0
- package/dist/agent-runtime/adapters/mock.d.ts +44 -0
- package/dist/agent-runtime/adapters/mock.js +54 -0
- package/dist/agent-runtime/adapters/mock.js.map +1 -0
- package/dist/agent-runtime/adapters/openai.d.ts +63 -0
- package/dist/agent-runtime/adapters/openai.js +302 -0
- package/dist/agent-runtime/adapters/openai.js.map +1 -0
- package/dist/agent-runtime/adapters/registry.d.ts +49 -0
- package/dist/agent-runtime/adapters/registry.js +129 -0
- package/dist/agent-runtime/adapters/registry.js.map +1 -0
- package/dist/agent-runtime/adapters/ui-tars.d.ts +13 -0
- package/dist/agent-runtime/adapters/ui-tars.js +22 -0
- package/dist/agent-runtime/adapters/ui-tars.js.map +1 -0
- package/dist/agent-runtime/agent-runner.d.ts +149 -0
- package/dist/agent-runtime/agent-runner.js +356 -0
- package/dist/agent-runtime/agent-runner.js.map +1 -0
- package/dist/agent-runtime/core.d.ts +170 -0
- package/dist/agent-runtime/core.js +624 -0
- package/dist/agent-runtime/core.js.map +1 -0
- package/dist/agent-runtime/index.d.ts +16 -0
- package/dist/agent-runtime/index.js +17 -0
- package/dist/agent-runtime/index.js.map +1 -0
- package/dist/agent-runtime/plan-executor.d.ts +118 -0
- package/dist/agent-runtime/plan-executor.js +501 -0
- package/dist/agent-runtime/plan-executor.js.map +1 -0
- package/dist/agent-runtime/planner/base.d.ts +128 -0
- package/dist/agent-runtime/planner/base.js +157 -0
- package/dist/agent-runtime/planner/base.js.map +1 -0
- package/dist/agent-runtime/planner/llm-planner.d.ts +58 -0
- package/dist/agent-runtime/planner/llm-planner.js +112 -0
- package/dist/agent-runtime/planner/llm-planner.js.map +1 -0
- package/dist/agent-runtime/planner/noop-planner.d.ts +13 -0
- package/dist/agent-runtime/planner/noop-planner.js +30 -0
- package/dist/agent-runtime/planner/noop-planner.js.map +1 -0
- package/dist/agent-runtime/planner/prompts.d.ts +33 -0
- package/dist/agent-runtime/planner/prompts.js +165 -0
- package/dist/agent-runtime/planner/prompts.js.map +1 -0
- package/dist/agent-runtime/reflection.d.ts +81 -0
- package/dist/agent-runtime/reflection.js +189 -0
- package/dist/agent-runtime/reflection.js.map +1 -0
- package/dist/agent-runtime/reflectors/action-reflector.d.ts +97 -0
- package/dist/agent-runtime/reflectors/action-reflector.js +233 -0
- package/dist/agent-runtime/reflectors/action-reflector.js.map +1 -0
- package/dist/agent-runtime/reflectors/policy.d.ts +53 -0
- package/dist/agent-runtime/reflectors/policy.js +106 -0
- package/dist/agent-runtime/reflectors/policy.js.map +1 -0
- package/dist/agent-runtime/reflectors/trajectory-reflector.d.ts +94 -0
- package/dist/agent-runtime/reflectors/trajectory-reflector.js +223 -0
- package/dist/agent-runtime/reflectors/trajectory-reflector.js.map +1 -0
- package/dist/agent-runtime/skills/embedding.d.ts +53 -0
- package/dist/agent-runtime/skills/embedding.js +134 -0
- package/dist/agent-runtime/skills/embedding.js.map +1 -0
- package/dist/agent-runtime/skills/fs-store.d.ts +88 -0
- package/dist/agent-runtime/skills/fs-store.js +483 -0
- package/dist/agent-runtime/skills/fs-store.js.map +1 -0
- package/dist/agent-runtime/skills/index.d.ts +10 -0
- package/dist/agent-runtime/skills/index.js +10 -0
- package/dist/agent-runtime/skills/index.js.map +1 -0
- package/dist/agent-runtime/skills/matcher-evolver.d.ts +88 -0
- package/dist/agent-runtime/skills/matcher-evolver.js +110 -0
- package/dist/agent-runtime/skills/matcher-evolver.js.map +1 -0
- package/dist/agent-runtime/skills/memory-store.d.ts +23 -0
- package/dist/agent-runtime/skills/memory-store.js +94 -0
- package/dist/agent-runtime/skills/memory-store.js.map +1 -0
- package/dist/agent-runtime/skills/normalise.d.ts +40 -0
- package/dist/agent-runtime/skills/normalise.js +152 -0
- package/dist/agent-runtime/skills/normalise.js.map +1 -0
- package/dist/agent-runtime/skills/replay.d.ts +46 -0
- package/dist/agent-runtime/skills/replay.js +106 -0
- package/dist/agent-runtime/skills/replay.js.map +1 -0
- package/dist/agent-runtime/skills/sqlite-store.d.ts +60 -0
- package/dist/agent-runtime/skills/sqlite-store.js +234 -0
- package/dist/agent-runtime/skills/sqlite-store.js.map +1 -0
- package/dist/agent-runtime/skills/types.d.ts +99 -0
- package/dist/agent-runtime/skills/types.js +14 -0
- package/dist/agent-runtime/skills/types.js.map +1 -0
- package/dist/agent.d.ts +110 -0
- package/dist/agent.js +162 -0
- package/dist/agent.js.map +1 -0
- package/dist/api.d.ts +27 -0
- package/dist/api.js +48 -0
- package/dist/api.js.map +1 -0
- package/dist/app/agent-host.d.ts +59 -0
- package/dist/app/agent-host.js +263 -0
- package/dist/app/agent-host.js.map +1 -0
- package/dist/app/device-manager.d.ts +28 -0
- package/dist/app/device-manager.js +64 -0
- package/dist/app/device-manager.js.map +1 -0
- package/dist/app/message-store.d.ts +55 -0
- package/dist/app/message-store.js +99 -0
- package/dist/app/message-store.js.map +1 -0
- package/dist/app/session-manager.d.ts +132 -0
- package/dist/app/session-manager.js +350 -0
- package/dist/app/session-manager.js.map +1 -0
- package/dist/bootstrap.d.ts +94 -0
- package/dist/bootstrap.js +459 -0
- package/dist/bootstrap.js.map +1 -0
- package/dist/chat.d.ts +16 -0
- package/dist/chat.js +175 -0
- package/dist/chat.js.map +1 -0
- package/dist/config/loader.d.ts +77 -0
- package/dist/config/loader.js +292 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/schema.d.ts +162 -0
- package/dist/config/schema.js +243 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/core/actions.d.ts +60 -0
- package/dist/core/actions.js +120 -0
- package/dist/core/actions.js.map +1 -0
- package/dist/core/errors.d.ts +8 -0
- package/dist/core/errors.js +9 -0
- package/dist/core/errors.js.map +1 -0
- package/dist/core/screen.d.ts +7 -0
- package/dist/core/screen.js +8 -0
- package/dist/core/screen.js.map +1 -0
- package/dist/core/session.d.ts +42 -0
- package/dist/core/session.js +29 -0
- package/dist/core/session.js.map +1 -0
- package/dist/debug/dump-system-prompt.d.ts +1 -0
- package/dist/debug/dump-system-prompt.js +34 -0
- package/dist/debug/dump-system-prompt.js.map +1 -0
- package/dist/device/android.d.ts +25 -0
- package/dist/device/android.js +231 -0
- package/dist/device/android.js.map +1 -0
- package/dist/device/apps.d.ts +3 -0
- package/dist/device/apps.js +77 -0
- package/dist/device/apps.js.map +1 -0
- package/dist/device/base.d.ts +20 -0
- package/dist/device/base.js +2 -0
- package/dist/device/base.js.map +1 -0
- package/dist/device/ui-xml.d.ts +29 -0
- package/dist/device/ui-xml.js +194 -0
- package/dist/device/ui-xml.js.map +1 -0
- package/dist/fleet/adapters/process/node-process-manager.d.ts +157 -0
- package/dist/fleet/adapters/process/node-process-manager.js +664 -0
- package/dist/fleet/adapters/process/node-process-manager.js.map +1 -0
- package/dist/fleet/adapters/sources/local-adb.d.ts +105 -0
- package/dist/fleet/adapters/sources/local-adb.js +353 -0
- package/dist/fleet/adapters/sources/local-adb.js.map +1 -0
- package/dist/fleet/adapters/sources/remote-adb.d.ts +97 -0
- package/dist/fleet/adapters/sources/remote-adb.js +138 -0
- package/dist/fleet/adapters/sources/remote-adb.js.map +1 -0
- package/dist/fleet/adapters/state/beeos-cli-store.d.ts +72 -0
- package/dist/fleet/adapters/state/beeos-cli-store.js +250 -0
- package/dist/fleet/adapters/state/beeos-cli-store.js.map +1 -0
- package/dist/fleet/adapters/state/port-allocator.d.ts +43 -0
- package/dist/fleet/adapters/state/port-allocator.js +173 -0
- package/dist/fleet/adapters/state/port-allocator.js.map +1 -0
- package/dist/fleet/domain/device.d.ts +138 -0
- package/dist/fleet/domain/device.js +49 -0
- package/dist/fleet/domain/device.js.map +1 -0
- package/dist/fleet/domain/events.d.ts +102 -0
- package/dist/fleet/domain/events.js +15 -0
- package/dist/fleet/domain/events.js.map +1 -0
- package/dist/fleet/domain/fsm.d.ts +122 -0
- package/dist/fleet/domain/fsm.js +475 -0
- package/dist/fleet/domain/fsm.js.map +1 -0
- package/dist/fleet/http/status-client.d.ts +31 -0
- package/dist/fleet/http/status-client.js +115 -0
- package/dist/fleet/http/status-client.js.map +1 -0
- package/dist/fleet/http/status-server.d.ts +64 -0
- package/dist/fleet/http/status-server.js +200 -0
- package/dist/fleet/http/status-server.js.map +1 -0
- package/dist/fleet/index.d.ts +27 -0
- package/dist/fleet/index.js +17 -0
- package/dist/fleet/index.js.map +1 -0
- package/dist/fleet/installer/index.d.ts +12 -0
- package/dist/fleet/installer/index.js +13 -0
- package/dist/fleet/installer/index.js.map +1 -0
- package/dist/fleet/installer/launchd-mac.d.ts +73 -0
- package/dist/fleet/installer/launchd-mac.js +291 -0
- package/dist/fleet/installer/launchd-mac.js.map +1 -0
- package/dist/fleet/installer/service-spec.d.ts +70 -0
- package/dist/fleet/installer/service-spec.js +43 -0
- package/dist/fleet/installer/service-spec.js.map +1 -0
- package/dist/fleet/installer/spawn-env.d.ts +45 -0
- package/dist/fleet/installer/spawn-env.js +224 -0
- package/dist/fleet/installer/spawn-env.js.map +1 -0
- package/dist/fleet/ports/clock.d.ts +19 -0
- package/dist/fleet/ports/clock.js +26 -0
- package/dist/fleet/ports/clock.js.map +1 -0
- package/dist/fleet/ports/device-source.d.ts +28 -0
- package/dist/fleet/ports/device-source.js +18 -0
- package/dist/fleet/ports/device-source.js.map +1 -0
- package/dist/fleet/ports/process-manager.d.ts +110 -0
- package/dist/fleet/ports/process-manager.js +27 -0
- package/dist/fleet/ports/process-manager.js.map +1 -0
- package/dist/fleet/ports/state-registry.d.ts +49 -0
- package/dist/fleet/ports/state-registry.js +12 -0
- package/dist/fleet/ports/state-registry.js.map +1 -0
- package/dist/fleet/supervisor.d.ts +107 -0
- package/dist/fleet/supervisor.js +508 -0
- package/dist/fleet/supervisor.js.map +1 -0
- package/dist/grounding/grounder.d.ts +46 -0
- package/dist/grounding/grounder.js +82 -0
- package/dist/grounding/grounder.js.map +1 -0
- package/dist/inbound/acp/handlers/device.d.ts +43 -0
- package/dist/inbound/acp/handlers/device.js +153 -0
- package/dist/inbound/acp/handlers/device.js.map +1 -0
- package/dist/inbound/acp/handlers/initialize.d.ts +71 -0
- package/dist/inbound/acp/handlers/initialize.js +53 -0
- package/dist/inbound/acp/handlers/initialize.js.map +1 -0
- package/dist/inbound/acp/handlers/models.d.ts +16 -0
- package/dist/inbound/acp/handlers/models.js +23 -0
- package/dist/inbound/acp/handlers/models.js.map +1 -0
- package/dist/inbound/acp/handlers/session.d.ts +51 -0
- package/dist/inbound/acp/handlers/session.js +61 -0
- package/dist/inbound/acp/handlers/session.js.map +1 -0
- package/dist/inbound/acp/handlers/skills.d.ts +8 -0
- package/dist/inbound/acp/handlers/skills.js +9 -0
- package/dist/inbound/acp/handlers/skills.js.map +1 -0
- package/dist/inbound/acp/notifier.d.ts +35 -0
- package/dist/inbound/acp/notifier.js +45 -0
- package/dist/inbound/acp/notifier.js.map +1 -0
- package/dist/inbound/acp/router.d.ts +67 -0
- package/dist/inbound/acp/router.js +179 -0
- package/dist/inbound/acp/router.js.map +1 -0
- package/dist/inbound/cli/fleet-command.d.ts +34 -0
- package/dist/inbound/cli/fleet-command.js +792 -0
- package/dist/inbound/cli/fleet-command.js.map +1 -0
- package/dist/inbound/cli/main.d.ts +38 -0
- package/dist/inbound/cli/main.js +364 -0
- package/dist/inbound/cli/main.js.map +1 -0
- package/dist/inbound/cli/run-command.d.ts +101 -0
- package/dist/inbound/cli/run-command.js +405 -0
- package/dist/inbound/cli/run-command.js.map +1 -0
- package/dist/inbound/cli/skills-command.d.ts +39 -0
- package/dist/inbound/cli/skills-command.js +333 -0
- package/dist/inbound/cli/skills-command.js.map +1 -0
- package/dist/inbound/http/server.d.ts +7 -0
- package/dist/inbound/http/server.js +8 -0
- package/dist/inbound/http/server.js.map +1 -0
- package/dist/index.d.ts +66 -0
- package/dist/index.js +48 -0
- package/dist/index.js.map +1 -0
- package/dist/logger/index.d.ts +37 -0
- package/dist/logger/index.js +84 -0
- package/dist/logger/index.js.map +1 -0
- package/dist/mcp-client/autospawn.d.ts +67 -0
- package/dist/mcp-client/autospawn.js +173 -0
- package/dist/mcp-client/autospawn.js.map +1 -0
- package/dist/mcp-client/client.d.ts +78 -0
- package/dist/mcp-client/client.js +168 -0
- package/dist/mcp-client/client.js.map +1 -0
- package/dist/mcp-client/index.d.ts +7 -0
- package/dist/mcp-client/index.js +4 -0
- package/dist/mcp-client/index.js.map +1 -0
- package/dist/mcp-client/inproc.d.ts +47 -0
- package/dist/mcp-client/inproc.js +229 -0
- package/dist/mcp-client/inproc.js.map +1 -0
- package/dist/mcp-client/runtime.d.ts +34 -0
- package/dist/mcp-client/runtime.js +15 -0
- package/dist/mcp-client/runtime.js.map +1 -0
- package/dist/model/adapters/autoglm.d.ts +31 -0
- package/dist/model/adapters/autoglm.js +409 -0
- package/dist/model/adapters/autoglm.js.map +1 -0
- package/dist/model/adapters/base.d.ts +13 -0
- package/dist/model/adapters/base.js +27 -0
- package/dist/model/adapters/base.js.map +1 -0
- package/dist/model/adapters/guiowl.d.ts +1 -0
- package/dist/model/adapters/guiowl.js +253 -0
- package/dist/model/adapters/guiowl.js.map +1 -0
- package/dist/model/adapters/index.d.ts +6 -0
- package/dist/model/adapters/index.js +8 -0
- package/dist/model/adapters/index.js.map +1 -0
- package/dist/model/adapters/maiui.d.ts +1 -0
- package/dist/model/adapters/maiui.js +117 -0
- package/dist/model/adapters/maiui.js.map +1 -0
- package/dist/model/adapters/qwenvl.d.ts +1 -0
- package/dist/model/adapters/qwenvl.js +108 -0
- package/dist/model/adapters/qwenvl.js.map +1 -0
- package/dist/model/adapters/showui.d.ts +1 -0
- package/dist/model/adapters/showui.js +132 -0
- package/dist/model/adapters/showui.js.map +1 -0
- package/dist/model/adapters/tool-call.d.ts +6 -0
- package/dist/model/adapters/tool-call.js +102 -0
- package/dist/model/adapters/tool-call.js.map +1 -0
- package/dist/model/adapters/uitars.d.ts +1 -0
- package/dist/model/adapters/uitars.js +180 -0
- package/dist/model/adapters/uitars.js.map +1 -0
- package/dist/model/base.d.ts +32 -0
- package/dist/model/base.js +2 -0
- package/dist/model/base.js.map +1 -0
- package/dist/model/client.d.ts +26 -0
- package/dist/model/client.js +300 -0
- package/dist/model/client.js.map +1 -0
- package/dist/model/image/compress.d.ts +53 -0
- package/dist/model/image/compress.js +89 -0
- package/dist/model/image/compress.js.map +1 -0
- package/dist/model/image/transport.d.ts +134 -0
- package/dist/model/image/transport.js +154 -0
- package/dist/model/image/transport.js.map +1 -0
- package/dist/orchestrator-system-prompt.d.ts +6 -0
- package/dist/orchestrator-system-prompt.js +81 -0
- package/dist/orchestrator-system-prompt.js.map +1 -0
- package/dist/pi-context.d.ts +82 -0
- package/dist/pi-context.js +62 -0
- package/dist/pi-context.js.map +1 -0
- package/dist/pi-local-skills.d.ts +17 -0
- package/dist/pi-local-skills.js +24 -0
- package/dist/pi-local-skills.js.map +1 -0
- package/dist/plugins/acp/index.d.ts +46 -0
- package/dist/plugins/acp/index.js +111 -0
- package/dist/plugins/acp/index.js.map +1 -0
- package/dist/plugins/agent-sync/index.d.ts +40 -0
- package/dist/plugins/agent-sync/index.js +98 -0
- package/dist/plugins/agent-sync/index.js.map +1 -0
- package/dist/plugins/http-debug/index.d.ts +30 -0
- package/dist/plugins/http-debug/index.js +64 -0
- package/dist/plugins/http-debug/index.js.map +1 -0
- package/dist/plugins/http-debug/server.d.ts +92 -0
- package/dist/plugins/http-debug/server.js +464 -0
- package/dist/plugins/http-debug/server.js.map +1 -0
- package/dist/ports/agent-core.d.ts +158 -0
- package/dist/ports/agent-core.js +2 -0
- package/dist/ports/agent-core.js.map +1 -0
- package/dist/ports/agent-host.d.ts +125 -0
- package/dist/ports/agent-host.js +23 -0
- package/dist/ports/agent-host.js.map +1 -0
- package/dist/ports/device.d.ts +53 -0
- package/dist/ports/device.js +13 -0
- package/dist/ports/device.js.map +1 -0
- package/dist/ports/identity.d.ts +22 -0
- package/dist/ports/identity.js +9 -0
- package/dist/ports/identity.js.map +1 -0
- package/dist/ports/plugin.d.ts +62 -0
- package/dist/ports/plugin.js +35 -0
- package/dist/ports/plugin.js.map +1 -0
- package/dist/ports/transport.d.ts +32 -0
- package/dist/ports/transport.js +12 -0
- package/dist/ports/transport.js.map +1 -0
- package/dist/ports/vlm.d.ts +59 -0
- package/dist/ports/vlm.js +14 -0
- package/dist/ports/vlm.js.map +1 -0
- package/dist/test-lan-autoglm.d.ts +7 -0
- package/dist/test-lan-autoglm.js +46 -0
- package/dist/test-lan-autoglm.js.map +1 -0
- package/dist/test-mem-breakdown.d.ts +1 -0
- package/dist/test-mem-breakdown.js +103 -0
- package/dist/test-mem-breakdown.js.map +1 -0
- package/dist/test-mem.d.ts +1 -0
- package/dist/test-mem.js +60 -0
- package/dist/test-mem.js.map +1 -0
- package/dist/test-mobile-use.d.ts +1 -0
- package/dist/test-mobile-use.js +18 -0
- package/dist/test-mobile-use.js.map +1 -0
- package/dist/test-screenshot-loop.d.ts +1 -0
- package/dist/test-screenshot-loop.js +43 -0
- package/dist/test-screenshot-loop.js.map +1 -0
- package/dist/test-screenshot.d.ts +1 -0
- package/dist/test-screenshot.js +10 -0
- package/dist/test-screenshot.js.map +1 -0
- package/dist/test-ui-xml-dump.d.ts +15 -0
- package/dist/test-ui-xml-dump.js +49 -0
- package/dist/test-ui-xml-dump.js.map +1 -0
- package/dist/types.d.ts +69 -0
- package/dist/types.js +23 -0
- package/dist/types.js.map +1 -0
- package/package.json +94 -0
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
import { ModelAdapter, registerAdapter } from "./base.js";
|
|
2
|
+
import { parseToolCall } from "./tool-call.js";
|
|
3
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
4
|
+
// GUI-Owl / GUI-Plus adapter
|
|
5
|
+
//
|
|
6
|
+
// Shared format used by:
|
|
7
|
+
// - mPLUG/GUI-Owl-7B/32B/1.5 (open-source, https://github.com/X-PLUG/MobileAgent)
|
|
8
|
+
// - Alibaba Tongyi gui-plus-* (DashScope, e.g. gui-plus-2026-02-26)
|
|
9
|
+
//
|
|
10
|
+
// Protocol (mirrors ClawGUI prompts_guiowl.py exactly):
|
|
11
|
+
// - system prompt contains a full <tools></tools> XML schema for `mobile_use`
|
|
12
|
+
// - model output: "Action: <short desc>\n<tool_call>{"name":"mobile_use","arguments":{...}}</tool_call>"
|
|
13
|
+
// - coordinates are 0–999 (resolution declared in the schema as 1000x1000)
|
|
14
|
+
// - only ONE user message is ever sent per turn; action history is injected as
|
|
15
|
+
// "Previous actions: Step1: ... Step2: ..." text, no assistant messages accumulated
|
|
16
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
17
|
+
const SCREEN_W = 1000;
|
|
18
|
+
const SCREEN_H = 1000;
|
|
19
|
+
function buildToolSchema(w, h) {
|
|
20
|
+
const schema = {
|
|
21
|
+
type: "function",
|
|
22
|
+
function: {
|
|
23
|
+
name_for_human: "mobile_use",
|
|
24
|
+
name: "mobile_use",
|
|
25
|
+
description: "Use a touchscreen to interact with a mobile device, and take screenshots.\n" +
|
|
26
|
+
"* This is an interface to a mobile device with touchscreen. " +
|
|
27
|
+
"You can perform actions like clicking, typing, swiping, etc.\n" +
|
|
28
|
+
"* Some applications may take time to start or process actions, " +
|
|
29
|
+
"so you may need to wait and take successive screenshots to see the results of your actions.\n" +
|
|
30
|
+
`* The screen's resolution is ${w}x${h}.\n` +
|
|
31
|
+
"* Make sure to click any buttons, links, icons, etc with the cursor tip " +
|
|
32
|
+
"in the center of the element. Don't click boxes on their edges unless asked.",
|
|
33
|
+
parameters: {
|
|
34
|
+
properties: {
|
|
35
|
+
action: {
|
|
36
|
+
description: "The action to perform. The available actions are:\n" +
|
|
37
|
+
"* `key`: Perform a key event on the mobile device.\n" +
|
|
38
|
+
" - This supports adb's `keyevent` syntax.\n" +
|
|
39
|
+
' - Examples: "volume_up", "volume_down", "power", "camera", "clear".\n' +
|
|
40
|
+
"* `click`: Click the point on the screen with coordinate (x, y).\n" +
|
|
41
|
+
"* `long_press`: Press the point on the screen with coordinate (x, y) for specified seconds.\n" +
|
|
42
|
+
"* `swipe`: Swipe from the starting point with coordinate (x, y) to the end point with coordinates2 (x2, y2).\n" +
|
|
43
|
+
"* `type`: Input the specified text into the activated input box.\n" +
|
|
44
|
+
"* `system_button`: Press the system button.\n" +
|
|
45
|
+
"* `open`: Open an app on the device.\n" +
|
|
46
|
+
"* `wait`: Wait specified seconds for the change to happen.\n" +
|
|
47
|
+
"* `answer`: Terminate the current task and output the answer.\n" +
|
|
48
|
+
"* `terminate`: Terminate the current task and report its completion status.",
|
|
49
|
+
enum: [
|
|
50
|
+
"key", "click", "long_press", "swipe", "type",
|
|
51
|
+
"system_button", "open", "wait", "answer", "terminate",
|
|
52
|
+
],
|
|
53
|
+
type: "string",
|
|
54
|
+
},
|
|
55
|
+
coordinate: {
|
|
56
|
+
description: "(x, y): The x (pixels from the left edge) and y (pixels from the top edge). " +
|
|
57
|
+
"Required only by `action=click`, `action=long_press`, and `action=swipe`.",
|
|
58
|
+
type: "array",
|
|
59
|
+
},
|
|
60
|
+
coordinate2: {
|
|
61
|
+
description: "(x, y): swipe endpoint. Required only by `action=swipe`.",
|
|
62
|
+
type: "array",
|
|
63
|
+
},
|
|
64
|
+
text: {
|
|
65
|
+
description: "Required only by `action=key`, `action=type`, `action=open`, and `action=answer`.",
|
|
66
|
+
type: "string",
|
|
67
|
+
},
|
|
68
|
+
time: {
|
|
69
|
+
description: "Seconds. Required only by `action=long_press` and `action=wait`.",
|
|
70
|
+
type: "number",
|
|
71
|
+
},
|
|
72
|
+
button: {
|
|
73
|
+
description: "Back = return to previous screen; Home = desktop; Menu = app background menu; Enter = press enter. " +
|
|
74
|
+
"Required only by `action=system_button`.",
|
|
75
|
+
enum: ["Back", "Home", "Menu", "Enter"],
|
|
76
|
+
type: "string",
|
|
77
|
+
},
|
|
78
|
+
status: {
|
|
79
|
+
description: "Task status. Required only by `action=terminate`.",
|
|
80
|
+
type: "string",
|
|
81
|
+
enum: ["success", "failure"],
|
|
82
|
+
},
|
|
83
|
+
},
|
|
84
|
+
required: ["action"],
|
|
85
|
+
type: "object",
|
|
86
|
+
},
|
|
87
|
+
args_format: "Format the arguments as a JSON object.",
|
|
88
|
+
},
|
|
89
|
+
};
|
|
90
|
+
return JSON.stringify(schema);
|
|
91
|
+
}
|
|
92
|
+
function systemPromptEn(w, h) {
|
|
93
|
+
return `# Tools
|
|
94
|
+
|
|
95
|
+
You may call one or more functions to assist with the user query.
|
|
96
|
+
|
|
97
|
+
You are provided with function signatures within <tools></tools> XML tags:
|
|
98
|
+
<tools>
|
|
99
|
+
${buildToolSchema(w, h)}
|
|
100
|
+
</tools>
|
|
101
|
+
|
|
102
|
+
For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
|
|
103
|
+
<tool_call>
|
|
104
|
+
{"name": <function-name>, "arguments": <args-json-object>}
|
|
105
|
+
</tool_call>
|
|
106
|
+
|
|
107
|
+
# Response format
|
|
108
|
+
|
|
109
|
+
Response format for every step:
|
|
110
|
+
1) Action: a short imperative describing what to do in the UI.
|
|
111
|
+
2) A single <tool_call>...</tool_call> block containing only the JSON: {"name": <function-name>, "arguments": <args-json-object>}.
|
|
112
|
+
|
|
113
|
+
Rules:
|
|
114
|
+
- Output exactly in the order: Action, <tool_call>.
|
|
115
|
+
- Be brief: one line for Action.
|
|
116
|
+
- Do not output anything else outside those two parts.
|
|
117
|
+
- If finishing, use mobile_use with action=terminate in the tool call.`;
|
|
118
|
+
}
|
|
119
|
+
function systemPromptCn(w, h) {
|
|
120
|
+
return `# 工具
|
|
121
|
+
|
|
122
|
+
你可以调用一个或多个函数来完成用户的请求。
|
|
123
|
+
|
|
124
|
+
可用的函数定义在 <tools></tools> XML 标签内:
|
|
125
|
+
<tools>
|
|
126
|
+
${buildToolSchema(w, h)}
|
|
127
|
+
</tools>
|
|
128
|
+
|
|
129
|
+
每次调用函数时,请在 <tool_call></tool_call> XML 标签内返回一个包含函数名和参数的 JSON 对象:
|
|
130
|
+
<tool_call>
|
|
131
|
+
{"name": <函数名>, "arguments": <参数JSON对象>}
|
|
132
|
+
</tool_call>
|
|
133
|
+
|
|
134
|
+
# 回复格式
|
|
135
|
+
|
|
136
|
+
每一步的回复格式:
|
|
137
|
+
1) Action: 一句简短的祈使句,描述在UI上要执行的操作。
|
|
138
|
+
2) 一个 <tool_call>...</tool_call> 代码块,只包含 JSON:{"name": <函数名>, "arguments": <参数JSON对象>}。
|
|
139
|
+
|
|
140
|
+
规则:
|
|
141
|
+
- 严格按照 Action、<tool_call> 的顺序输出。
|
|
142
|
+
- 保持简洁:Action 只需一句话。
|
|
143
|
+
- 不要输出这两部分以外的任何内容。
|
|
144
|
+
- 如果任务完成,在 tool_call 中使用 mobile_use 的 action=terminate。`;
|
|
145
|
+
}
|
|
146
|
+
function buildUserQuery(task, history, lang, opts) {
|
|
147
|
+
const isEn = lang !== "cn";
|
|
148
|
+
const ctxHint = !opts.hasScreenshot && opts.hasXml
|
|
149
|
+
? (isEn
|
|
150
|
+
? "according to the UI hierarchy (XML) below, the instruction, and previous actions"
|
|
151
|
+
: "根据下方 UI 结构(XML)、指令和之前的操作")
|
|
152
|
+
: opts.hasScreenshot && opts.hasXml
|
|
153
|
+
? (isEn
|
|
154
|
+
? "according to the UI screenshot, the UI hierarchy (XML) below, the instruction, and previous actions"
|
|
155
|
+
: "根据 UI 截图、下方 UI 结构(XML)、指令和之前的操作")
|
|
156
|
+
: (isEn
|
|
157
|
+
? "according to the UI screenshot, the instruction, and previous actions"
|
|
158
|
+
: "根据UI截图、指令和之前的操作");
|
|
159
|
+
if (history.length === 0) {
|
|
160
|
+
return isEn
|
|
161
|
+
? `Please generate the next move ${ctxHint}.\n\nInstruction: ${task}`
|
|
162
|
+
: `请${ctxHint},生成下一步操作。\n\n指令: ${task}`;
|
|
163
|
+
}
|
|
164
|
+
const prev = history.map((d, i) => `Step${i + 1}: ${d} Tool response: None`).join("\n");
|
|
165
|
+
return isEn
|
|
166
|
+
? `Please generate the next move ${ctxHint}.\n\nInstruction: ${task}\n\nPrevious actions: \n${prev}`
|
|
167
|
+
: `请${ctxHint},生成下一步操作。\n\n指令: ${task}\n\n之前的操作: \n${prev}`;
|
|
168
|
+
}
|
|
169
|
+
class GUIOwlAdapter extends ModelAdapter {
|
|
170
|
+
name = "guiowl";
|
|
171
|
+
// Matches GUI-Owl-{7B,32B,1.5} open-source AND Alibaba DashScope gui-plus-*
|
|
172
|
+
patterns = [/guiowl/, /gui[-_]?owl/, /owl.*gui/, /gui[-_]?plus/];
|
|
173
|
+
actionHistory = [];
|
|
174
|
+
buildMessages(task, screenInfo, _context, isFirst, lang) {
|
|
175
|
+
if (isFirst)
|
|
176
|
+
this.actionHistory = [];
|
|
177
|
+
const sys = lang === "cn" ? systemPromptCn(SCREEN_W, SCREEN_H) : systemPromptEn(SCREEN_W, SCREEN_H);
|
|
178
|
+
const hasShot = !!screenInfo.imageRef;
|
|
179
|
+
const hasXml = !!screenInfo.uiXml;
|
|
180
|
+
let userText = buildUserQuery(task, this.actionHistory, lang, { hasScreenshot: hasShot, hasXml });
|
|
181
|
+
if (hasXml && screenInfo.uiXml) {
|
|
182
|
+
userText += lang !== "cn"
|
|
183
|
+
? `\n\n** UI Hierarchy (XML) **\n${screenInfo.uiXml}`
|
|
184
|
+
: `\n\n** UI 结构 (XML) **\n${screenInfo.uiXml}`;
|
|
185
|
+
}
|
|
186
|
+
const userContent = [
|
|
187
|
+
{ type: "text", text: userText },
|
|
188
|
+
];
|
|
189
|
+
if (screenInfo.imageRef) {
|
|
190
|
+
userContent.push({
|
|
191
|
+
type: "image_url",
|
|
192
|
+
image_url: { url: screenInfo.imageRef.url },
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
return [
|
|
196
|
+
{ role: "system", content: sys },
|
|
197
|
+
{ role: "user", content: userContent },
|
|
198
|
+
];
|
|
199
|
+
}
|
|
200
|
+
parseResponse(raw) {
|
|
201
|
+
// Extract the "Action: ..." line as thinking/description for history
|
|
202
|
+
const actionLineMatch = raw.match(/^\s*Action:\s*(.+?)$/m);
|
|
203
|
+
const description = actionLineMatch ? actionLineMatch[1].trim().replace(/^"|"$/g, "") : "";
|
|
204
|
+
const parsed = parseToolCall(raw, description) ?? fallbackWait(description);
|
|
205
|
+
// Coordinates come back 0-999; handler treats "0-1000" the same bucket.
|
|
206
|
+
parsed.coordSystem = "0-1000";
|
|
207
|
+
return parsed;
|
|
208
|
+
}
|
|
209
|
+
updateContext(_context, _response, parsed) {
|
|
210
|
+
if (!parsed.shouldFinish) {
|
|
211
|
+
const desc = (parsed.thinking || actionSummary(parsed)).slice(0, 200);
|
|
212
|
+
if (desc)
|
|
213
|
+
this.actionHistory.push(desc);
|
|
214
|
+
}
|
|
215
|
+
// GUI-Owl / GUI-Plus rebuild messages from scratch each turn (no accumulation)
|
|
216
|
+
return [];
|
|
217
|
+
}
|
|
218
|
+
clearHistory() {
|
|
219
|
+
this.actionHistory = [];
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
registerAdapter(new GUIOwlAdapter());
|
|
223
|
+
function fallbackWait(thinking) {
|
|
224
|
+
return {
|
|
225
|
+
actionType: "wait",
|
|
226
|
+
params: { duration_seconds: 2 },
|
|
227
|
+
thinking,
|
|
228
|
+
coordSystem: "0-1000",
|
|
229
|
+
shouldFinish: false,
|
|
230
|
+
finishMessage: null,
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
function actionSummary(parsed) {
|
|
234
|
+
const { actionType: t, params: p } = parsed;
|
|
235
|
+
if (t === "tap")
|
|
236
|
+
return `click(${p.x},${p.y})`;
|
|
237
|
+
if (t === "double_tap")
|
|
238
|
+
return `double_tap(${p.x},${p.y})`;
|
|
239
|
+
if (t === "long_press")
|
|
240
|
+
return `long_press(${p.x},${p.y})`;
|
|
241
|
+
if (t === "type")
|
|
242
|
+
return `type("${String(p.text).slice(0, 40)}")`;
|
|
243
|
+
if (t === "swipe")
|
|
244
|
+
return `swipe(${p.start_x},${p.start_y}→${p.end_x},${p.end_y})`;
|
|
245
|
+
if (t === "open_app")
|
|
246
|
+
return `open(${p.app_name})`;
|
|
247
|
+
if (t === "press_back")
|
|
248
|
+
return "system_button(Back)";
|
|
249
|
+
if (t === "press_home")
|
|
250
|
+
return "system_button(Home)";
|
|
251
|
+
return t;
|
|
252
|
+
}
|
|
253
|
+
//# sourceMappingURL=guiowl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"guiowl.js","sourceRoot":"","sources":["../../../src/model/adapters/guiowl.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,gFAAgF;AAChF,6BAA6B;AAC7B,EAAE;AACF,yBAAyB;AACzB,sFAAsF;AACtF,uEAAuE;AACvE,EAAE;AACF,wDAAwD;AACxD,gFAAgF;AAChF,4GAA4G;AAC5G,6EAA6E;AAC7E,iFAAiF;AACjF,wFAAwF;AACxF,gFAAgF;AAEhF,MAAM,QAAQ,GAAG,IAAI,CAAC;AACtB,MAAM,QAAQ,GAAG,IAAI,CAAC;AAEtB,SAAS,eAAe,CAAC,CAAS,EAAE,CAAS;IAC3C,MAAM,MAAM,GAAG;QACb,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,cAAc,EAAE,YAAY;YAC5B,IAAI,EAAE,YAAY;YAClB,WAAW,EACT,6EAA6E;gBAC7E,8DAA8D;gBAC9D,gEAAgE;gBAChE,iEAAiE;gBACjE,+FAA+F;gBAC/F,gCAAgC,CAAC,IAAI,CAAC,KAAK;gBAC3C,0EAA0E;gBAC1E,8EAA8E;YAChF,UAAU,EAAE;gBACV,UAAU,EAAE;oBACV,MAAM,EAAE;wBACN,WAAW,EACT,qDAAqD;4BACrD,sDAAsD;4BACtD,gDAAgD;4BAChD,2EAA2E;4BAC3E,oEAAoE;4BACpE,+FAA+F;4BAC/F,gHAAgH;4BAChH,oEAAoE;4BACpE,+CAA+C;4BAC/C,wCAAwC;4BACxC,8DAA8D;4BAC9D,iEAAiE;4BACjE,6EAA6E;wBAC/E,IAAI,EAAE;4BACJ,KAAK,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM;4BAC7C,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW;yBACvD;wBACD,IAAI,EAAE,QAAQ;qBACf;oBACD,UAAU,EAAE;wBACV,WAAW,EACT,8EAA8E;4BAC9E,2EAA2E;wBAC7E,IAAI,EAAE,OAAO;qBACd;oBACD,WAAW,EAAE;wBACX,WAAW,EACT,0DAA0D;wBAC5D,IAAI,EAAE,OAAO;qBACd;oBACD,IAAI,EAAE;wBACJ,WAAW,EACT,mFAAmF;wBACrF,IAAI,EAAE,QAAQ;qBACf;oBACD,IAAI,EAAE;wBACJ,WAAW,EAAE,kEAAkE;wBAC/E,IAAI,EAAE,QAAQ;qBACf;oBACD,MAAM,EAAE;wBACN,WAAW,EACT,qGAAqG;4BACrG,0CAA0C;wBAC5C,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;wBACvC,IAAI,EAAE,QAAQ;qBACf;oBACD,MAAM,EAAE;wBACN,WAAW,EAAE,mDAAmD;wBAChE,IAAI,EAAE,QAAQ;wBACd,IAAI,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC;qBAC7B;iBACF;gBACD,QAAQ,EAAE,CAAC,QAAQ,CAAC;gBACpB,IAAI,EAAE,QAAQ;aACf;YACD,WAAW,EAAE,wCAAwC;SACtD;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,cAAc,CAAC,CAAS,EAAE,CAAS;IAC1C,OAAO;;;;;;EAMP,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC;;;;;;;;;;;;;;;;;;uEAkBgD,CAAC;AACxE,CAAC;AAED,SAAS,cAAc,CAAC,CAAS,EAAE,CAAS;IAC1C,OAAO;;;;;;EAMP,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC;;;;;;;;;;;;;;;;;;wDAkBiC,CAAC;AACzD,CAAC;AAED,SAAS,cAAc,CACrB,IAAY,EACZ,OAAiB,EACjB,IAAY,EACZ,IAAiD;IAEjD,MAAM,IAAI,GAAG,IAAI,KAAK,IAAI,CAAC;IAC3B,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM;QAChD,CAAC,CAAC,CAAC,IAAI;YACL,CAAC,CAAC,kFAAkF;YACpF,CAAC,CAAC,0BAA0B,CAAC;QAC/B,CAAC,CAAC,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM;YACjC,CAAC,CAAC,CAAC,IAAI;gBACL,CAAC,CAAC,qGAAqG;gBACvG,CAAC,CAAC,iCAAiC,CAAC;YACtC,CAAC,CAAC,CAAC,IAAI;gBACL,CAAC,CAAC,uEAAuE;gBACzE,CAAC,CAAC,iBAAiB,CAAC,CAAC;IAE3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI;YACT,CAAC,CAAC,iCAAiC,OAAO,qBAAqB,IAAI,EAAE;YACrE,CAAC,CAAC,IAAI,OAAO,oBAAoB,IAAI,EAAE,CAAC;IAC5C,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxF,OAAO,IAAI;QACT,CAAC,CAAC,iCAAiC,OAAO,qBAAqB,IAAI,2BAA2B,IAAI,EAAE;QACpG,CAAC,CAAC,IAAI,OAAO,oBAAoB,IAAI,gBAAgB,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,aAAc,SAAQ,YAAY;IAC7B,IAAI,GAAG,QAAQ,CAAC;IACzB,4EAA4E;IACnE,QAAQ,GAAG,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,cAAc,CAAC,CAAC;IAElE,aAAa,GAAa,EAAE,CAAC;IAErC,aAAa,CACX,IAAY,EACZ,UAAsB,EACtB,QAAmB,EACnB,OAAgB,EAChB,IAAY;QAEZ,IAAI,OAAO;YAAE,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;QAErC,MAAM,GAAG,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACpG,MAAM,OAAO,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC;QACtC,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC;QAClC,IAAI,QAAQ,GAAG,cAAc,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,EAAE,IAAI,EAAE,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAClG,IAAI,MAAM,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;YAC/B,QAAQ,IAAI,IAAI,KAAK,IAAI;gBACvB,CAAC,CAAC,iCAAiC,UAAU,CAAC,KAAK,EAAE;gBACrD,CAAC,CAAC,0BAA0B,UAAU,CAAC,KAAK,EAAE,CAAC;QACnD,CAAC;QAED,MAAM,WAAW,GAAkD;YACjE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE;SACjC,CAAC;QACF,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxB,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,WAAW;gBACjB,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE;aAC5C,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE;YAChC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE;SACvC,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,qEAAqE;QACrE,MAAM,eAAe,GAAG,GAAG,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC3D,MAAM,WAAW,GAAG,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE3F,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC;QAC5E,wEAAwE;QACxE,MAAM,CAAC,WAAW,GAAG,QAAQ,CAAC;QAC9B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,aAAa,CAAC,QAAmB,EAAE,SAAwB,EAAE,MAAoB;QAC/E,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACtE,IAAI,IAAI;gBAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,CAAC;QACD,+EAA+E;QAC/E,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,YAAY;QACV,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;IAC1B,CAAC;CACF;AAED,eAAe,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAErC,SAAS,YAAY,CAAC,QAAgB;IACpC,OAAO;QACL,UAAU,EAAE,MAAM;QAClB,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE;QAC/B,QAAQ;QACR,WAAW,EAAE,QAAQ;QACrB,YAAY,EAAE,KAAK;QACnB,aAAa,EAAE,IAAI;KACpB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,MAAoB;IACzC,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC;IAC5C,IAAI,CAAC,KAAK,KAAK;QAAE,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC/C,IAAI,CAAC,KAAK,YAAY;QAAE,OAAO,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC3D,IAAI,CAAC,KAAK,YAAY;QAAE,OAAO,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC3D,IAAI,CAAC,KAAK,MAAM;QAAE,OAAO,SAAS,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC;IAClE,IAAI,CAAC,KAAK,OAAO;QAAE,OAAO,SAAS,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC;IACnF,IAAI,CAAC,KAAK,UAAU;QAAE,OAAO,QAAQ,CAAC,CAAC,QAAQ,GAAG,CAAC;IACnD,IAAI,CAAC,KAAK,YAAY;QAAE,OAAO,qBAAqB,CAAC;IACrD,IAAI,CAAC,KAAK,YAAY;QAAE,OAAO,qBAAqB,CAAC;IACrD,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// Import each adapter to trigger registerAdapter() side-effects
|
|
2
|
+
export * from "./autoglm.js";
|
|
3
|
+
export * from "./uitars.js";
|
|
4
|
+
export * from "./qwenvl.js";
|
|
5
|
+
export * from "./maiui.js";
|
|
6
|
+
export * from "./guiowl.js";
|
|
7
|
+
export * from "./showui.js";
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/model/adapters/index.ts"],"names":[],"mappings":"AAAA,gEAAgE;AAChE,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC;AAC5B,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,aAAa,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { ModelAdapter, registerAdapter } from "./base.js";
|
|
2
|
+
import { parseToolCall } from "./tool-call.js";
|
|
3
|
+
// MAIUI adapter
|
|
4
|
+
// Same <tool_call> JSON format as QwenVL but accumulates multi-turn context
|
|
5
|
+
// with a max of 3 images in the window. Handles <thinking> tags.
|
|
6
|
+
const MAX_IMAGES = 3;
|
|
7
|
+
const SYSTEM_PROMPT_EN = `\
|
|
8
|
+
You are a mobile GUI automation agent. Analyze the screenshot and output the next action as a tool call.
|
|
9
|
+
|
|
10
|
+
<tool_call>
|
|
11
|
+
{"name": "mobile_use", "arguments": {"action": "<action>", ...}}
|
|
12
|
+
</tool_call>
|
|
13
|
+
|
|
14
|
+
Actions and parameters:
|
|
15
|
+
- {"action":"tap","coordinate":[x,y]}
|
|
16
|
+
- {"action":"double_tap","coordinate":[x,y]}
|
|
17
|
+
- {"action":"long_press","coordinate":[x,y]}
|
|
18
|
+
- {"action":"swipe","start_coordinate":[x,y],"end_coordinate":[x,y]}
|
|
19
|
+
- {"action":"type","text":"..."}
|
|
20
|
+
- {"action":"press_back"}
|
|
21
|
+
- {"action":"press_home"}
|
|
22
|
+
- {"action":"open_app","text":"App Name"}
|
|
23
|
+
- {"action":"wait"}
|
|
24
|
+
- {"action":"finish","text":"completion reason"}
|
|
25
|
+
|
|
26
|
+
Coordinates are 0–1000 relative to screen size.`;
|
|
27
|
+
const SYSTEM_PROMPT_CN = `\
|
|
28
|
+
你是一个手机 GUI 自动化 Agent。分析截图并以工具调用形式输出下一步操作。
|
|
29
|
+
|
|
30
|
+
<tool_call>
|
|
31
|
+
{"name": "mobile_use", "arguments": {"action": "<动作>", ...}}
|
|
32
|
+
</tool_call>
|
|
33
|
+
|
|
34
|
+
动作及参数:
|
|
35
|
+
- {"action":"tap","coordinate":[x,y]}
|
|
36
|
+
- {"action":"double_tap","coordinate":[x,y]}
|
|
37
|
+
- {"action":"long_press","coordinate":[x,y]}
|
|
38
|
+
- {"action":"swipe","start_coordinate":[x,y],"end_coordinate":[x,y]}
|
|
39
|
+
- {"action":"type","text":"..."}
|
|
40
|
+
- {"action":"press_back"}
|
|
41
|
+
- {"action":"press_home"}
|
|
42
|
+
- {"action":"open_app","text":"App名称"}
|
|
43
|
+
- {"action":"wait"}
|
|
44
|
+
- {"action":"finish","text":"完成原因"}
|
|
45
|
+
|
|
46
|
+
坐标范围 0–1000,相对于屏幕尺寸。`;
|
|
47
|
+
class MAIUIAdapter extends ModelAdapter {
|
|
48
|
+
name = "maiui";
|
|
49
|
+
patterns = [/maiui/, /mai[-_]?ui/, /mai_ui/];
|
|
50
|
+
buildMessages(task, screenInfo, context, isFirst, lang) {
|
|
51
|
+
const systemPrompt = lang === "cn" ? SYSTEM_PROMPT_CN : SYSTEM_PROMPT_EN;
|
|
52
|
+
const msgs = [{ role: "system", content: systemPrompt }];
|
|
53
|
+
if (isFirst) {
|
|
54
|
+
const content = [];
|
|
55
|
+
if (screenInfo.imageRef) {
|
|
56
|
+
content.push({ type: "image_url", image_url: { url: screenInfo.imageRef.url } });
|
|
57
|
+
}
|
|
58
|
+
content.push({ type: "text", text: `Task: ${task}` });
|
|
59
|
+
msgs.push({ role: "user", content });
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
// Prune old images to keep at most MAX_IMAGES across all messages
|
|
63
|
+
const pruned = pruneImages([...context], MAX_IMAGES - 1); // -1 for current screenshot
|
|
64
|
+
msgs.push(...pruned);
|
|
65
|
+
const content = [];
|
|
66
|
+
if (screenInfo.imageRef) {
|
|
67
|
+
content.push({ type: "image_url", image_url: { url: screenInfo.imageRef.url } });
|
|
68
|
+
}
|
|
69
|
+
content.push({ type: "text", text: "Current screen. What is the next action?" });
|
|
70
|
+
msgs.push({ role: "user", content });
|
|
71
|
+
}
|
|
72
|
+
return msgs;
|
|
73
|
+
}
|
|
74
|
+
parseResponse(raw) {
|
|
75
|
+
const thinking = extractThinking(raw);
|
|
76
|
+
const cleaned = raw.replace(/<thinking>[\s\S]*?<\/thinking>/gi, "").trim();
|
|
77
|
+
const toolBlock = cleaned.includes("<tool_call>") ? cleaned : `<tool_call>${cleaned}</tool_call>`;
|
|
78
|
+
return parseToolCall(toolBlock, thinking) ?? wait(thinking);
|
|
79
|
+
}
|
|
80
|
+
updateContext(context, response, parsed) {
|
|
81
|
+
context.push({ role: "assistant", content: response.rawContent });
|
|
82
|
+
return context;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
registerAdapter(new MAIUIAdapter());
|
|
86
|
+
// ── Helpers ──────────────────────────────────────────────────────────────────
|
|
87
|
+
function extractThinking(raw) {
|
|
88
|
+
const m = raw.match(/<thinking>([\s\S]*?)<\/thinking>/i);
|
|
89
|
+
return m ? m[1].trim() : "";
|
|
90
|
+
}
|
|
91
|
+
function pruneImages(context, keepCount) {
|
|
92
|
+
// Find all messages with images, remove oldest ones exceeding keepCount
|
|
93
|
+
let imageCount = 0;
|
|
94
|
+
// Count images from newest to oldest
|
|
95
|
+
for (let i = context.length - 1; i >= 0; i--) {
|
|
96
|
+
const msg = context[i];
|
|
97
|
+
if (Array.isArray(msg.content)) {
|
|
98
|
+
const parts = msg.content;
|
|
99
|
+
const hasImage = parts.some((p) => p.type === "image_url");
|
|
100
|
+
if (hasImage) {
|
|
101
|
+
imageCount++;
|
|
102
|
+
if (imageCount > keepCount) {
|
|
103
|
+
// Strip images from this message
|
|
104
|
+
context[i] = {
|
|
105
|
+
...msg,
|
|
106
|
+
content: parts.filter((p) => p.type !== "image_url"),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return context;
|
|
113
|
+
}
|
|
114
|
+
function wait(thinking) {
|
|
115
|
+
return { actionType: "wait", params: { duration_seconds: 2 }, thinking, coordSystem: "0-1000", shouldFinish: false, finishMessage: null };
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=maiui.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"maiui.js","sourceRoot":"","sources":["../../../src/model/adapters/maiui.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,gBAAgB;AAChB,4EAA4E;AAC5E,iEAAiE;AAEjE,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;gDAmBuB,CAAC;AAEjD,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;qBAmBJ,CAAC;AAEtB,MAAM,YAAa,SAAQ,YAAY;IAC5B,IAAI,GAAG,OAAO,CAAC;IACf,QAAQ,GAAG,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;IAEtD,aAAa,CACX,IAAY,EACZ,UAAsB,EACtB,OAAkB,EAClB,OAAgB,EAChB,IAAY;QAEZ,MAAM,YAAY,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEzE,MAAM,IAAI,GAAc,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;QAEpE,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,OAAO,GAAkD,EAAE,CAAC;YAClE,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,IAAI,EAAE,EAAE,CAAC,CAAC;YACtD,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,kEAAkE;YAClE,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,GAAG,OAAO,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,4BAA4B;YACtF,IAAI,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAErB,MAAM,OAAO,GAAkD,EAAE,CAAC;YAClE,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,0CAA0C,EAAE,CAAC,CAAC;YACjF,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACvC,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3E,MAAM,SAAS,GAAG,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,OAAO,cAAc,CAAC;QAClG,OAAO,aAAa,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC9D,CAAC;IAED,aAAa,CAAC,OAAkB,EAAE,QAAuB,EAAE,MAAoB;QAC7E,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAClE,OAAO,OAAO,CAAC;IACjB,CAAC;CACF;AAED,eAAe,CAAC,IAAI,YAAY,EAAE,CAAC,CAAC;AAEpC,gFAAgF;AAEhF,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACzD,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,WAAW,CAAC,OAAkB,EAAE,SAAiB;IACxD,wEAAwE;IACxE,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,qCAAqC;IACrC,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACvB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,OAAkC,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;YAC3D,IAAI,QAAQ,EAAE,CAAC;gBACb,UAAU,EAAE,CAAC;gBACb,IAAI,UAAU,GAAG,SAAS,EAAE,CAAC;oBAC3B,iCAAiC;oBACjC,OAAO,CAAC,CAAC,CAAC,GAAG;wBACX,GAAG,GAAG;wBACN,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC;qBACrD,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,IAAI,CAAC,QAAgB;IAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC5I,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { ModelAdapter, registerAdapter } from "./base.js";
|
|
2
|
+
import { parseToolCall } from "./tool-call.js";
|
|
3
|
+
// Qwen-VL adapter
|
|
4
|
+
// Output: <tool_call>{"name":"mobile_use","arguments":{...}}</tool_call>
|
|
5
|
+
// Always rebuilds: 1 system + 1 user message (no multi-turn context)
|
|
6
|
+
// Coordinates: 0–999 normalized (treated as 0-1000)
|
|
7
|
+
const SYSTEM_PROMPT_EN = `\
|
|
8
|
+
You are a mobile automation assistant. Given a screenshot and task history, output the next action as a tool call.
|
|
9
|
+
|
|
10
|
+
Use the mobile_use tool:
|
|
11
|
+
<tool_call>
|
|
12
|
+
{"name": "mobile_use", "arguments": {"action": "<action>", ...}}
|
|
13
|
+
</tool_call>
|
|
14
|
+
|
|
15
|
+
Actions:
|
|
16
|
+
- tap: {"action":"tap","coordinate":[x,y]}
|
|
17
|
+
- long_press: {"action":"long_press","coordinate":[x,y]}
|
|
18
|
+
- swipe: {"action":"swipe","start_coordinate":[x,y],"end_coordinate":[x,y]}
|
|
19
|
+
- type: {"action":"type","text":"..."}
|
|
20
|
+
- press_back: {"action":"press_back"}
|
|
21
|
+
- press_home: {"action":"press_home"}
|
|
22
|
+
- open_app: {"action":"open_app","text":"App Name"}
|
|
23
|
+
- wait: {"action":"wait"}
|
|
24
|
+
- finish: {"action":"finish","text":"reason"}
|
|
25
|
+
|
|
26
|
+
Coordinates are 0–999 (relative to screen dimensions).`;
|
|
27
|
+
const SYSTEM_PROMPT_CN = `\
|
|
28
|
+
你是一个手机自动化助手。根据截图和历史操作,输出下一个动作的工具调用。
|
|
29
|
+
|
|
30
|
+
使用 mobile_use 工具:
|
|
31
|
+
<tool_call>
|
|
32
|
+
{"name": "mobile_use", "arguments": {"action": "<动作>", ...}}
|
|
33
|
+
</tool_call>
|
|
34
|
+
|
|
35
|
+
动作列表:
|
|
36
|
+
- tap: {"action":"tap","coordinate":[x,y]}
|
|
37
|
+
- long_press: {"action":"long_press","coordinate":[x,y]}
|
|
38
|
+
- swipe: {"action":"swipe","start_coordinate":[x,y],"end_coordinate":[x,y]}
|
|
39
|
+
- type: {"action":"type","text":"..."}
|
|
40
|
+
- press_back: {"action":"press_back"}
|
|
41
|
+
- press_home: {"action":"press_home"}
|
|
42
|
+
- open_app: {"action":"open_app","text":"App名称"}
|
|
43
|
+
- wait: {"action":"wait"}
|
|
44
|
+
- finish: {"action":"finish","text":"原因"}
|
|
45
|
+
|
|
46
|
+
坐标范围 0–999(相对屏幕尺寸)。`;
|
|
47
|
+
class QwenVLAdapter extends ModelAdapter {
|
|
48
|
+
name = "qwenvl";
|
|
49
|
+
patterns = [/qwen.*vl/, /qwen2.*vl/, /qvq/, /qwen-vl/];
|
|
50
|
+
actionHistory = [];
|
|
51
|
+
buildMessages(task, screenInfo, context, isFirst, lang) {
|
|
52
|
+
const systemPrompt = lang === "cn" ? SYSTEM_PROMPT_CN : SYSTEM_PROMPT_EN;
|
|
53
|
+
// Build single user message with task + history + screenshot
|
|
54
|
+
const historyText = this.actionHistory.length > 0
|
|
55
|
+
? `\n\nAction history:\n${this.actionHistory.map((a, i) => `${i + 1}. ${a}`).join("\n")}`
|
|
56
|
+
: "";
|
|
57
|
+
const taskText = isFirst
|
|
58
|
+
? `Task: ${task}${historyText}`
|
|
59
|
+
: `Task: ${task}${historyText}\n\nWhat is the next action?`;
|
|
60
|
+
const content = [];
|
|
61
|
+
if (screenInfo.imageRef) {
|
|
62
|
+
content.push({ type: "image_url", image_url: { url: screenInfo.imageRef.url } });
|
|
63
|
+
}
|
|
64
|
+
content.push({ type: "text", text: taskText });
|
|
65
|
+
return [
|
|
66
|
+
{ role: "system", content: systemPrompt },
|
|
67
|
+
{ role: "user", content },
|
|
68
|
+
];
|
|
69
|
+
}
|
|
70
|
+
parseResponse(raw) {
|
|
71
|
+
const thinking = extractThinking(raw);
|
|
72
|
+
const toolBlock = raw.includes("<tool_call>") ? raw : `<tool_call>${raw}</tool_call>`;
|
|
73
|
+
return parseToolCall(toolBlock, thinking) ?? wait(thinking);
|
|
74
|
+
}
|
|
75
|
+
updateContext(context, response, parsed) {
|
|
76
|
+
// Record action in history for next rebuild
|
|
77
|
+
if (!parsed.shouldFinish) {
|
|
78
|
+
this.actionHistory.push(actionSummary(parsed));
|
|
79
|
+
}
|
|
80
|
+
// QwenVL always rebuilds messages, so context is not used for multi-turn
|
|
81
|
+
return context;
|
|
82
|
+
}
|
|
83
|
+
clearHistory() {
|
|
84
|
+
this.actionHistory = [];
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
registerAdapter(new QwenVLAdapter());
|
|
88
|
+
// ── Helpers ──────────────────────────────────────────────────────────────────
|
|
89
|
+
function extractThinking(raw) {
|
|
90
|
+
const m = raw.match(/<think(?:ing)?>([\s\S]*?)<\/think(?:ing)?>/i);
|
|
91
|
+
return m ? m[1].trim() : "";
|
|
92
|
+
}
|
|
93
|
+
function wait(thinking) {
|
|
94
|
+
return { actionType: "wait", params: { duration_seconds: 2 }, thinking, coordSystem: "0-1000", shouldFinish: false, finishMessage: null };
|
|
95
|
+
}
|
|
96
|
+
function actionSummary(parsed) {
|
|
97
|
+
const { actionType: t, params: p } = parsed;
|
|
98
|
+
if (t === "tap")
|
|
99
|
+
return `tap(${p.x},${p.y})`;
|
|
100
|
+
if (t === "type")
|
|
101
|
+
return `type("${p.text}")`;
|
|
102
|
+
if (t === "swipe")
|
|
103
|
+
return `swipe(${p.start_x},${p.start_y}→${p.end_x},${p.end_y})`;
|
|
104
|
+
if (t === "open_app")
|
|
105
|
+
return `open_app(${p.app_name})`;
|
|
106
|
+
return t;
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=qwenvl.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"qwenvl.js","sourceRoot":"","sources":["../../../src/model/adapters/qwenvl.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAE/C,kBAAkB;AAClB,yEAAyE;AACzE,qEAAqE;AACrE,oDAAoD;AAEpD,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;uDAmB8B,CAAC;AAExD,MAAM,gBAAgB,GAAG;;;;;;;;;;;;;;;;;;;oBAmBL,CAAC;AAErB,MAAM,aAAc,SAAQ,YAAY;IAC7B,IAAI,GAAG,QAAQ,CAAC;IAChB,QAAQ,GAAG,CAAC,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;IAExD,aAAa,GAAa,EAAE,CAAC;IAErC,aAAa,CACX,IAAY,EACZ,UAAsB,EACtB,OAAkB,EAClB,OAAgB,EAChB,IAAY;QAEZ,MAAM,YAAY,GAAG,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAEzE,6DAA6D;QAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC;YAC/C,CAAC,CAAC,wBAAwB,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YACzF,CAAC,CAAC,EAAE,CAAC;QAEP,MAAM,QAAQ,GAAG,OAAO;YACtB,CAAC,CAAC,SAAS,IAAI,GAAG,WAAW,EAAE;YAC/B,CAAC,CAAC,SAAS,IAAI,GAAG,WAAW,8BAA8B,CAAC;QAE9D,MAAM,OAAO,GAAkD,EAAE,CAAC;QAClE,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QAE/C,OAAO;YACL,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;YACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE;SAC1B,CAAC;IACJ,CAAC;IAED,aAAa,CAAC,GAAW;QACvB,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,cAAc,GAAG,cAAc,CAAC;QACtF,OAAO,aAAa,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC9D,CAAC;IAED,aAAa,CAAC,OAAkB,EAAE,QAAuB,EAAE,MAAoB;QAC7E,4CAA4C;QAC5C,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,yEAAyE;QACzE,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,YAAY;QACV,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC;IAC1B,CAAC;CACF;AAED,eAAe,CAAC,IAAI,aAAa,EAAE,CAAC,CAAC;AAErC,gFAAgF;AAEhF,SAAS,eAAe,CAAC,GAAW;IAClC,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACnE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,IAAI,CAAC,QAAgB;IAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,gBAAgB,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;AAC5I,CAAC;AAED,SAAS,aAAa,CAAC,MAAoB;IACzC,MAAM,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC;IAC5C,IAAI,CAAC,KAAK,KAAK;QAAE,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC7C,IAAI,CAAC,KAAK,MAAM;QAAE,OAAO,SAAS,CAAC,CAAC,IAAI,IAAI,CAAC;IAC7C,IAAI,CAAC,KAAK,OAAO;QAAE,OAAO,SAAS,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC;IACnF,IAAI,CAAC,KAAK,UAAU;QAAE,OAAO,YAAY,CAAC,CAAC,QAAQ,GAAG,CAAC;IACvD,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|