@waq666/pneuma 0.0.0-stage → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +144 -2
- package/bin/dsc.js +91 -0
- package/bin/headless.cjs +40 -0
- package/lib/adapter/core-runtime.d.ts +11 -0
- package/lib/adapter/core-runtime.js +259 -0
- package/lib/adapter/host.d.ts +18 -0
- package/lib/adapter/host.js +14 -0
- package/lib/adapter/runtime.d.ts +15 -0
- package/lib/adapter/runtime.js +290 -0
- package/lib/adapter/transcript.d.ts +184 -0
- package/lib/adapter/transcript.js +605 -0
- package/lib/app/AgentsOverlay.d.ts +18 -0
- package/lib/app/AgentsOverlay.js +63 -0
- package/lib/app/App.d.ts +13 -0
- package/lib/app/App.js +1498 -0
- package/lib/app/ApprovalCard.d.ts +12 -0
- package/lib/app/ApprovalCard.js +76 -0
- package/lib/app/AskCard.d.ts +20 -0
- package/lib/app/AskCard.js +45 -0
- package/lib/app/ChatView.d.ts +32 -0
- package/lib/app/ChatView.js +198 -0
- package/lib/app/Composer.d.ts +50 -0
- package/lib/app/Composer.js +282 -0
- package/lib/app/DiffBlock.d.ts +6 -0
- package/lib/app/DiffBlock.js +37 -0
- package/lib/app/MarkdownView.d.ts +6 -0
- package/lib/app/MarkdownView.js +99 -0
- package/lib/app/ModelPicker.d.ts +7 -0
- package/lib/app/ModelPicker.js +18 -0
- package/lib/app/PlanReviewCard.d.ts +13 -0
- package/lib/app/PlanReviewCard.js +54 -0
- package/lib/app/PresetPicker.d.ts +11 -0
- package/lib/app/PresetPicker.js +19 -0
- package/lib/app/PreviewOverlay.d.ts +14 -0
- package/lib/app/PreviewOverlay.js +22 -0
- package/lib/app/SessionPicker.d.ts +43 -0
- package/lib/app/SessionPicker.js +56 -0
- package/lib/app/SettingsOverlay.d.ts +32 -0
- package/lib/app/SettingsOverlay.js +157 -0
- package/lib/app/SkillsPicker.d.ts +7 -0
- package/lib/app/SkillsPicker.js +19 -0
- package/lib/app/StatusBar.d.ts +22 -0
- package/lib/app/StatusBar.js +272 -0
- package/lib/app/TaskStrips.d.ts +13 -0
- package/lib/app/TaskStrips.js +40 -0
- package/lib/app/ToolCard.d.ts +5 -0
- package/lib/app/ToolCard.js +40 -0
- package/lib/app/TranscriptOverlay.d.ts +12 -0
- package/lib/app/TranscriptOverlay.js +17 -0
- package/lib/app/Welcome.d.ts +6 -0
- package/lib/app/Welcome.js +66 -0
- package/lib/app/attach.d.ts +16 -0
- package/lib/app/attach.js +58 -0
- package/lib/app/click.d.ts +27 -0
- package/lib/app/click.js +89 -0
- package/lib/app/clipboard-image.d.ts +11 -0
- package/lib/app/clipboard-image.js +66 -0
- package/lib/app/cursor.d.ts +4 -0
- package/lib/app/cursor.js +24 -0
- package/lib/app/hello.d.ts +11 -0
- package/lib/app/hello.js +8 -0
- package/lib/app/history-store.d.ts +4 -0
- package/lib/app/history-store.js +41 -0
- package/lib/app/image-blocks.d.ts +12 -0
- package/lib/app/image-blocks.js +125 -0
- package/lib/app/markdown.d.ts +54 -0
- package/lib/app/markdown.js +189 -0
- package/lib/app/settings-model.d.ts +69 -0
- package/lib/app/settings-model.js +114 -0
- package/lib/app/theme.d.ts +166 -0
- package/lib/app/theme.js +141 -0
- package/lib/boot.d.ts +1 -0
- package/lib/boot.js +34 -0
- package/lib/commands.d.ts +49 -0
- package/lib/commands.js +115 -0
- package/lib/contract.d.ts +1973 -0
- package/lib/contract.js +36 -0
- package/lib/core/agent-roles.d.ts +52 -0
- package/lib/core/agent-roles.js +274 -0
- package/lib/core/approval-floor.d.ts +183 -0
- package/lib/core/approval-floor.js +679 -0
- package/lib/core/approval.d.ts +42 -0
- package/lib/core/approval.js +8 -0
- package/lib/core/audit.d.ts +61 -0
- package/lib/core/audit.js +59 -0
- package/lib/core/boot-notices.d.ts +6 -0
- package/lib/core/boot-notices.js +61 -0
- package/lib/core/cdp/actions.d.ts +182 -0
- package/lib/core/cdp/actions.js +649 -0
- package/lib/core/cdp/launch.d.ts +192 -0
- package/lib/core/cdp/launch.js +491 -0
- package/lib/core/cdp/snapshot.d.ts +139 -0
- package/lib/core/cdp/snapshot.js +316 -0
- package/lib/core/cdp/transport.d.ts +160 -0
- package/lib/core/cdp/transport.js +359 -0
- package/lib/core/command-policy.d.ts +108 -0
- package/lib/core/command-policy.js +580 -0
- package/lib/core/commands-completion.d.ts +78 -0
- package/lib/core/commands-completion.js +167 -0
- package/lib/core/compact-anchors.d.ts +76 -0
- package/lib/core/compact-anchors.js +284 -0
- package/lib/core/compact.d.ts +110 -0
- package/lib/core/compact.js +210 -0
- package/lib/core/config-store.d.ts +38 -0
- package/lib/core/config-store.js +307 -0
- package/lib/core/config.d.ts +36 -0
- package/lib/core/config.js +124 -0
- package/lib/core/connection-test.d.ts +60 -0
- package/lib/core/connection-test.js +84 -0
- package/lib/core/desktop-control.d.ts +298 -0
- package/lib/core/desktop-control.js +1311 -0
- package/lib/core/diff-text.d.ts +74 -0
- package/lib/core/diff-text.js +257 -0
- package/lib/core/dock-browser.d.ts +137 -0
- package/lib/core/dock-browser.js +146 -0
- package/lib/core/dock-ops.d.ts +59 -0
- package/lib/core/dock-ops.js +104 -0
- package/lib/core/dock-session.d.ts +39 -0
- package/lib/core/dock-session.js +28 -0
- package/lib/core/dsh-compat/hook.d.ts +18 -0
- package/lib/core/dsh-compat/hook.js +52 -0
- package/lib/core/dsh-compat/resolve-hook.d.ts +5 -0
- package/lib/core/dsh-compat/resolve-hook.js +59 -0
- package/lib/core/dsh-compat/tools-facade.d.ts +47 -0
- package/lib/core/dsh-compat/tools-facade.js +108 -0
- package/lib/core/err-text.d.ts +13 -0
- package/lib/core/err-text.js +15 -0
- package/lib/core/events.d.ts +149 -0
- package/lib/core/events.js +1 -0
- package/lib/core/git-exec.d.ts +23 -0
- package/lib/core/git-exec.js +72 -0
- package/lib/core/git-info.d.ts +47 -0
- package/lib/core/git-info.js +157 -0
- package/lib/core/goal.d.ts +89 -0
- package/lib/core/goal.js +189 -0
- package/lib/core/hooks.d.ts +272 -0
- package/lib/core/hooks.js +818 -0
- package/lib/core/host-browser-link.d.ts +59 -0
- package/lib/core/host-browser-link.js +9 -0
- package/lib/core/host-methods.d.ts +25 -0
- package/lib/core/host-methods.js +103 -0
- package/lib/core/image-diet.d.ts +77 -0
- package/lib/core/image-diet.js +127 -0
- package/lib/core/learnings/ledger.d.ts +73 -0
- package/lib/core/learnings/ledger.js +273 -0
- package/lib/core/learnings/skill-write.d.ts +250 -0
- package/lib/core/learnings/skill-write.js +854 -0
- package/lib/core/learnings/store.d.ts +102 -0
- package/lib/core/learnings/store.js +266 -0
- package/lib/core/lifecycle-hooks.d.ts +189 -0
- package/lib/core/lifecycle-hooks.js +576 -0
- package/lib/core/llm.d.ts +272 -0
- package/lib/core/llm.js +548 -0
- package/lib/core/lockfile.d.ts +17 -0
- package/lib/core/lockfile.js +110 -0
- package/lib/core/loop.d.ts +185 -0
- package/lib/core/loop.js +587 -0
- package/lib/core/lsp/client.d.ts +189 -0
- package/lib/core/lsp/client.js +690 -0
- package/lib/core/lsp/framing.d.ts +84 -0
- package/lib/core/lsp/framing.js +181 -0
- package/lib/core/lsp/line-shift.d.ts +28 -0
- package/lib/core/lsp/line-shift.js +127 -0
- package/lib/core/lsp/manager.d.ts +93 -0
- package/lib/core/lsp/manager.js +418 -0
- package/lib/core/lsp/normalize.d.ts +32 -0
- package/lib/core/lsp/normalize.js +139 -0
- package/lib/core/lsp/servers.d.ts +106 -0
- package/lib/core/lsp/servers.js +352 -0
- package/lib/core/lsp/uri.d.ts +76 -0
- package/lib/core/lsp/uri.js +179 -0
- package/lib/core/market.d.ts +51 -0
- package/lib/core/market.js +358 -0
- package/lib/core/mcp.d.ts +201 -0
- package/lib/core/mcp.js +836 -0
- package/lib/core/memory-eval.d.ts +158 -0
- package/lib/core/memory-eval.js +215 -0
- package/lib/core/memory.d.ts +266 -0
- package/lib/core/memory.js +703 -0
- package/lib/core/mention.d.ts +57 -0
- package/lib/core/mention.js +129 -0
- package/lib/core/microcompact.d.ts +48 -0
- package/lib/core/microcompact.js +79 -0
- package/lib/core/migrate.d.ts +56 -0
- package/lib/core/migrate.js +134 -0
- package/lib/core/model-caps.d.ts +83 -0
- package/lib/core/model-caps.js +150 -0
- package/lib/core/model-discovery.d.ts +19 -0
- package/lib/core/model-discovery.js +69 -0
- package/lib/core/modes.d.ts +72 -0
- package/lib/core/modes.js +152 -0
- package/lib/core/native-bridge.d.ts +135 -0
- package/lib/core/native-bridge.js +563 -0
- package/lib/core/path-policy.d.ts +32 -0
- package/lib/core/path-policy.js +248 -0
- package/lib/core/permission-presets.d.ts +67 -0
- package/lib/core/permission-presets.js +75 -0
- package/lib/core/plugin-loader.d.ts +36 -0
- package/lib/core/plugin-loader.js +294 -0
- package/lib/core/plugin-registry.d.ts +113 -0
- package/lib/core/plugin-registry.js +256 -0
- package/lib/core/prefs.d.ts +82 -0
- package/lib/core/prefs.js +372 -0
- package/lib/core/presets.d.ts +80 -0
- package/lib/core/presets.js +373 -0
- package/lib/core/pricing.d.ts +90 -0
- package/lib/core/pricing.js +126 -0
- package/lib/core/prompt.d.ts +92 -0
- package/lib/core/prompt.js +339 -0
- package/lib/core/ptc.d.ts +66 -0
- package/lib/core/ptc.js +70 -0
- package/lib/core/remote/frames.d.ts +91 -0
- package/lib/core/remote/frames.js +174 -0
- package/lib/core/remote/notify.d.ts +54 -0
- package/lib/core/remote/notify.js +77 -0
- package/lib/core/remote/owner.d.ts +32 -0
- package/lib/core/remote/owner.js +75 -0
- package/lib/core/remote/pairing.d.ts +104 -0
- package/lib/core/remote/pairing.js +261 -0
- package/lib/core/remote/push.d.ts +80 -0
- package/lib/core/remote/push.js +222 -0
- package/lib/core/remote/store.d.ts +18 -0
- package/lib/core/remote/store.js +59 -0
- package/lib/core/remote/tickets.d.ts +20 -0
- package/lib/core/remote/tickets.js +49 -0
- package/lib/core/remote/uploads.d.ts +63 -0
- package/lib/core/remote/uploads.js +161 -0
- package/lib/core/request-attribution.d.ts +26 -0
- package/lib/core/request-attribution.js +86 -0
- package/lib/core/request-invariant.d.ts +61 -0
- package/lib/core/request-invariant.js +199 -0
- package/lib/core/sandbox/backends.d.ts +64 -0
- package/lib/core/sandbox/backends.js +194 -0
- package/lib/core/sandbox/execpolicy.d.ts +93 -0
- package/lib/core/sandbox/execpolicy.js +452 -0
- package/lib/core/sandbox/net-proxy.d.ts +31 -0
- package/lib/core/sandbox/net-proxy.js +162 -0
- package/lib/core/sandbox/policy.d.ts +211 -0
- package/lib/core/sandbox/policy.js +559 -0
- package/lib/core/sandbox/win/abi.d.ts +161 -0
- package/lib/core/sandbox/win/abi.js +169 -0
- package/lib/core/sandbox/win/account.d.ts +71 -0
- package/lib/core/sandbox/win/account.js +206 -0
- package/lib/core/sandbox/win/acl.d.ts +54 -0
- package/lib/core/sandbox/win/acl.js +339 -0
- package/lib/core/sandbox/win/backend.d.ts +61 -0
- package/lib/core/sandbox/win/backend.js +249 -0
- package/lib/core/sandbox/win/doctor.d.ts +24 -0
- package/lib/core/sandbox/win/doctor.js +272 -0
- package/lib/core/sandbox/win/ffi.d.ts +107 -0
- package/lib/core/sandbox/win/ffi.js +166 -0
- package/lib/core/sandbox/win/net-abi.d.ts +154 -0
- package/lib/core/sandbox/win/net-abi.js +316 -0
- package/lib/core/sandbox/win/runner-entry.d.ts +35 -0
- package/lib/core/sandbox/win/runner-entry.js +391 -0
- package/lib/core/sandbox/win/setup.d.ts +60 -0
- package/lib/core/sandbox/win/setup.js +986 -0
- package/lib/core/sandbox/win/token.d.ts +35 -0
- package/lib/core/sandbox/win/token.js +238 -0
- package/lib/core/sandbox/win/wfp.d.ts +41 -0
- package/lib/core/sandbox/win/wfp.js +200 -0
- package/lib/core/sandbox/win/workspace-sid.d.ts +9 -0
- package/lib/core/sandbox/win/workspace-sid.js +45 -0
- package/lib/core/schedule/rule.d.ts +184 -0
- package/lib/core/schedule/rule.js +745 -0
- package/lib/core/schedule/runner.d.ts +185 -0
- package/lib/core/schedule/runner.js +678 -0
- package/lib/core/schedule/store.d.ts +182 -0
- package/lib/core/schedule/store.js +455 -0
- package/lib/core/secrets.d.ts +41 -0
- package/lib/core/secrets.js +150 -0
- package/lib/core/session-export.d.ts +18 -0
- package/lib/core/session-export.js +73 -0
- package/lib/core/session-index.d.ts +159 -0
- package/lib/core/session-index.js +683 -0
- package/lib/core/session-meta.d.ts +41 -0
- package/lib/core/session-meta.js +97 -0
- package/lib/core/session.d.ts +356 -0
- package/lib/core/session.js +998 -0
- package/lib/core/skills.d.ts +60 -0
- package/lib/core/skills.js +171 -0
- package/lib/core/spill.d.ts +159 -0
- package/lib/core/spill.js +329 -0
- package/lib/core/surface-registry.d.ts +45 -0
- package/lib/core/surface-registry.js +67 -0
- package/lib/core/team-board.d.ts +95 -0
- package/lib/core/team-board.js +316 -0
- package/lib/core/throughput.d.ts +46 -0
- package/lib/core/throughput.js +40 -0
- package/lib/core/todo-reminder.d.ts +44 -0
- package/lib/core/todo-reminder.js +55 -0
- package/lib/core/todo.d.ts +81 -0
- package/lib/core/todo.js +222 -0
- package/lib/core/token-estimate.d.ts +55 -0
- package/lib/core/token-estimate.js +82 -0
- package/lib/core/tool-guards.d.ts +103 -0
- package/lib/core/tool-guards.js +77 -0
- package/lib/core/tool-search.d.ts +152 -0
- package/lib/core/tool-search.js +311 -0
- package/lib/core/tools/bash.d.ts +281 -0
- package/lib/core/tools/bash.js +871 -0
- package/lib/core/tools/command-runner.d.ts +84 -0
- package/lib/core/tools/command-runner.js +68 -0
- package/lib/core/tools/fs-tools.d.ts +23 -0
- package/lib/core/tools/fs-tools.js +247 -0
- package/lib/core/tools/index.d.ts +8 -0
- package/lib/core/tools/index.js +4 -0
- package/lib/core/tools/sandbox-args.d.ts +51 -0
- package/lib/core/tools/sandbox-args.js +90 -0
- package/lib/core/tools/search-tools.d.ts +3 -0
- package/lib/core/tools/search-tools.js +153 -0
- package/lib/core/tools.d.ts +96 -0
- package/lib/core/tools.js +53 -0
- package/lib/core/ui-boot-source.d.ts +71 -0
- package/lib/core/ui-boot-source.js +121 -0
- package/lib/core/ui-bundles.d.ts +249 -0
- package/lib/core/ui-bundles.js +830 -0
- package/lib/core/ui-combo.d.ts +49 -0
- package/lib/core/ui-combo.js +55 -0
- package/lib/core/ui-events.d.ts +81 -0
- package/lib/core/ui-events.js +364 -0
- package/lib/core/ui-hmr.d.ts +36 -0
- package/lib/core/ui-hmr.js +128 -0
- package/lib/core/ui-methods.d.ts +69 -0
- package/lib/core/ui-methods.js +341 -0
- package/lib/core/ui-route.d.ts +69 -0
- package/lib/core/ui-route.js +122 -0
- package/lib/core/ui-server.d.ts +75 -0
- package/lib/core/ui-server.js +369 -0
- package/lib/core/ui-trust.d.ts +47 -0
- package/lib/core/ui-trust.js +164 -0
- package/lib/core/untrusted.d.ts +30 -0
- package/lib/core/untrusted.js +48 -0
- package/lib/core/update-check.d.ts +81 -0
- package/lib/core/update-check.js +177 -0
- package/lib/core/usage-log.d.ts +82 -0
- package/lib/core/usage-log.js +306 -0
- package/lib/core/version.d.ts +1 -0
- package/lib/core/version.js +37 -0
- package/lib/core/web-http.d.ts +57 -0
- package/lib/core/web-http.js +137 -0
- package/lib/core/web-static.d.ts +31 -0
- package/lib/core/web-static.js +75 -0
- package/lib/headless.d.ts +1 -0
- package/lib/headless.js +31 -0
- package/lib/host/kernel.d.ts +33 -0
- package/lib/host/kernel.js +451 -0
- package/lib/index.d.ts +41 -0
- package/lib/index.js +35 -0
- package/lib/plugins/agent.d.ts +2 -0
- package/lib/plugins/agent.js +274 -0
- package/lib/plugins/approval-floor.d.ts +2 -0
- package/lib/plugins/approval-floor.js +223 -0
- package/lib/plugins/approval.d.ts +16 -0
- package/lib/plugins/approval.js +674 -0
- package/lib/plugins/ask.d.ts +2 -0
- package/lib/plugins/ask.js +245 -0
- package/lib/plugins/browser.d.ts +68 -0
- package/lib/plugins/browser.js +1341 -0
- package/lib/plugins/commands.d.ts +23 -0
- package/lib/plugins/commands.js +498 -0
- package/lib/plugins/compact.d.ts +21 -0
- package/lib/plugins/compact.js +274 -0
- package/lib/plugins/computer-use.d.ts +47 -0
- package/lib/plugins/computer-use.js +539 -0
- package/lib/plugins/desktop-dock.d.ts +2 -0
- package/lib/plugins/desktop-dock.js +626 -0
- package/lib/plugins/dock-browser.d.ts +34 -0
- package/lib/plugins/dock-browser.js +765 -0
- package/lib/plugins/dsh-compat.d.ts +25 -0
- package/lib/plugins/dsh-compat.js +37 -0
- package/lib/plugins/file-review.d.ts +108 -0
- package/lib/plugins/file-review.js +393 -0
- package/lib/plugins/git.d.ts +26 -0
- package/lib/plugins/git.js +132 -0
- package/lib/plugins/goal.d.ts +20 -0
- package/lib/plugins/goal.js +245 -0
- package/lib/plugins/guards.d.ts +14 -0
- package/lib/plugins/guards.js +18 -0
- package/lib/plugins/hooks.d.ts +19 -0
- package/lib/plugins/hooks.js +583 -0
- package/lib/plugins/host-stdio.d.ts +88 -0
- package/lib/plugins/host-stdio.js +317 -0
- package/lib/plugins/image-diet.d.ts +17 -0
- package/lib/plugins/image-diet.js +85 -0
- package/lib/plugins/lifecycle-hooks.d.ts +26 -0
- package/lib/plugins/lifecycle-hooks.js +338 -0
- package/lib/plugins/llm.d.ts +13 -0
- package/lib/plugins/llm.js +215 -0
- package/lib/plugins/lsp.d.ts +54 -0
- package/lib/plugins/lsp.js +608 -0
- package/lib/plugins/mcp.d.ts +16 -0
- package/lib/plugins/mcp.js +673 -0
- package/lib/plugins/memory.d.ts +17 -0
- package/lib/plugins/memory.js +858 -0
- package/lib/plugins/microcompact.d.ts +18 -0
- package/lib/plugins/microcompact.js +119 -0
- package/lib/plugins/mode.d.ts +18 -0
- package/lib/plugins/mode.js +115 -0
- package/lib/plugins/permission-presets.d.ts +18 -0
- package/lib/plugins/permission-presets.js +135 -0
- package/lib/plugins/plan.d.ts +2 -0
- package/lib/plugins/plan.js +192 -0
- package/lib/plugins/plugin-manager.d.ts +2 -0
- package/lib/plugins/plugin-manager.js +165 -0
- package/lib/plugins/presets.d.ts +2 -0
- package/lib/plugins/presets.js +182 -0
- package/lib/plugins/prompt.d.ts +2 -0
- package/lib/plugins/prompt.js +264 -0
- package/lib/plugins/ptc.d.ts +2 -0
- package/lib/plugins/ptc.js +272 -0
- package/lib/plugins/remote/http.d.ts +11 -0
- package/lib/plugins/remote/http.js +11 -0
- package/lib/plugins/remote/routes.d.ts +24 -0
- package/lib/plugins/remote/routes.js +384 -0
- package/lib/plugins/remote/types.d.ts +135 -0
- package/lib/plugins/remote/types.js +87 -0
- package/lib/plugins/remote.d.ts +17 -0
- package/lib/plugins/remote.js +1028 -0
- package/lib/plugins/runtime-api.d.ts +16 -0
- package/lib/plugins/runtime-api.js +99 -0
- package/lib/plugins/runtime.d.ts +10 -0
- package/lib/plugins/runtime.js +451 -0
- package/lib/plugins/sandbox.d.ts +2 -0
- package/lib/plugins/sandbox.js +761 -0
- package/lib/plugins/schedule.d.ts +26 -0
- package/lib/plugins/schedule.js +538 -0
- package/lib/plugins/self-improve.d.ts +61 -0
- package/lib/plugins/self-improve.js +953 -0
- package/lib/plugins/session-search.d.ts +16 -0
- package/lib/plugins/session-search.js +308 -0
- package/lib/plugins/session-title.d.ts +19 -0
- package/lib/plugins/session-title.js +109 -0
- package/lib/plugins/session.d.ts +7 -0
- package/lib/plugins/session.js +296 -0
- package/lib/plugins/settings.d.ts +3 -0
- package/lib/plugins/settings.js +705 -0
- package/lib/plugins/skills.d.ts +2 -0
- package/lib/plugins/skills.js +360 -0
- package/lib/plugins/spill.d.ts +16 -0
- package/lib/plugins/spill.js +200 -0
- package/lib/plugins/subagent.d.ts +2 -0
- package/lib/plugins/subagent.js +862 -0
- package/lib/plugins/surfaces.d.ts +11 -0
- package/lib/plugins/surfaces.js +20 -0
- package/lib/plugins/team.d.ts +16 -0
- package/lib/plugins/team.js +149 -0
- package/lib/plugins/todo.d.ts +19 -0
- package/lib/plugins/todo.js +188 -0
- package/lib/plugins/tool-search.d.ts +2 -0
- package/lib/plugins/tool-search.js +425 -0
- package/lib/plugins/tools-default.d.ts +12 -0
- package/lib/plugins/tools-default.js +275 -0
- package/lib/plugins/tools.d.ts +16 -0
- package/lib/plugins/tools.js +62 -0
- package/lib/plugins/transcript.d.ts +15 -0
- package/lib/plugins/transcript.js +311 -0
- package/lib/plugins/tui.d.ts +2 -0
- package/lib/plugins/tui.js +98 -0
- package/lib/plugins/waiting.d.ts +12 -0
- package/lib/plugins/waiting.js +34 -0
- package/lib/plugins/web-search.d.ts +14 -0
- package/lib/plugins/web-search.js +263 -0
- package/lib/plugins/webserver.d.ts +48 -0
- package/lib/plugins/webserver.js +228 -0
- package/lib/remote/assets/app-DAcUCOEn.js +14 -0
- package/lib/remote/assets/icon-192.png +0 -0
- package/lib/remote/assets/icon-512.png +0 -0
- package/lib/remote/assets/index.html +34 -0
- package/lib/remote/assets/manifest.json +27 -0
- package/lib/remote/assets/style-DIB6729x.css +1 -0
- package/lib/remote/assets/sw.js +66 -0
- package/lib/services/types.d.ts +1228 -0
- package/lib/services/types.js +1 -0
- package/lib/tools/config-cli.d.ts +1 -0
- package/lib/tools/config-cli.js +89 -0
- package/lib/ui/arrival.d.ts +72 -0
- package/lib/ui/arrival.js +105 -0
- package/lib/ui/assets/app-B70nVIKd.js +69 -0
- package/lib/ui/assets/chunk-2UxHyX5q.js +1 -0
- package/lib/ui/assets/chunk-B1dDrJ26.js +1 -0
- package/lib/ui/assets/chunk-B6aJPvgy.js +1 -0
- package/lib/ui/assets/chunk-BEwlwnbL.js +1 -0
- package/lib/ui/assets/chunk-BGcgwlCI.js +12 -0
- package/lib/ui/assets/chunk-BIGW1oBm.js +1 -0
- package/lib/ui/assets/chunk-BPQ3VLAy.js +1 -0
- package/lib/ui/assets/chunk-BRH0KZbk.js +141 -0
- package/lib/ui/assets/chunk-BX6H_ySx.js +1 -0
- package/lib/ui/assets/chunk-B_P3UbPa.js +1 -0
- package/lib/ui/assets/chunk-B_TYUkTI.js +1 -0
- package/lib/ui/assets/chunk-B_vNuMnf.js +1 -0
- package/lib/ui/assets/chunk-BajyYBss.js +10 -0
- package/lib/ui/assets/chunk-BdnUsdx6.js +1 -0
- package/lib/ui/assets/chunk-Buea-lGh.js +1 -0
- package/lib/ui/assets/chunk-C0PvZCCE.js +1 -0
- package/lib/ui/assets/chunk-C0m74zRD.js +1 -0
- package/lib/ui/assets/chunk-C27-OAKa.js +1 -0
- package/lib/ui/assets/chunk-C2oV4EkX.js +1 -0
- package/lib/ui/assets/chunk-C7zT0LnQ.js +1 -0
- package/lib/ui/assets/chunk-C9GSunVT.js +1 -0
- package/lib/ui/assets/chunk-C9tS-k6U.js +1 -0
- package/lib/ui/assets/chunk-CHLpvVh8.js +1 -0
- package/lib/ui/assets/chunk-CLxacb5B.js +1 -0
- package/lib/ui/assets/chunk-COt5Ahok.js +1 -0
- package/lib/ui/assets/chunk-CPTIWCL2.js +1 -0
- package/lib/ui/assets/chunk-CQLl-L-p.js +1 -0
- package/lib/ui/assets/chunk-CUWRgxuq.js +55 -0
- package/lib/ui/assets/chunk-CWanVSXC.js +1 -0
- package/lib/ui/assets/chunk-Cf5RLm7j.js +1 -0
- package/lib/ui/assets/chunk-CickPsom.js +1 -0
- package/lib/ui/assets/chunk-Cj5Yp3dK.js +1 -0
- package/lib/ui/assets/chunk-Cmh6b_Ma.js +1 -0
- package/lib/ui/assets/chunk-Cp-IABpG.js +1 -0
- package/lib/ui/assets/chunk-CqE71os6.js +1 -0
- package/lib/ui/assets/chunk-CtKP3W-j.js +1 -0
- package/lib/ui/assets/chunk-Cvjx9yec.js +1 -0
- package/lib/ui/assets/chunk-CxGP3dtY.js +1 -0
- package/lib/ui/assets/chunk-CylS5w8V.js +1 -0
- package/lib/ui/assets/chunk-D38fbIBN.js +1 -0
- package/lib/ui/assets/chunk-D97Zzqfu.js +1 -0
- package/lib/ui/assets/chunk-DAi9KRSo.js +1 -0
- package/lib/ui/assets/chunk-DHK4yRfg.js +1 -0
- package/lib/ui/assets/chunk-DJeabLx-.js +1 -0
- package/lib/ui/assets/chunk-DNRxmY5a.js +1 -0
- package/lib/ui/assets/chunk-DNSN1z5r.js +1 -0
- package/lib/ui/assets/chunk-DNasK_ZG.js +1 -0
- package/lib/ui/assets/chunk-DPvrY3IQ.js +1 -0
- package/lib/ui/assets/chunk-DXmwc3jG.js +1 -0
- package/lib/ui/assets/chunk-Dd0kTxKP.js +1 -0
- package/lib/ui/assets/chunk-De0Btta0.js +1 -0
- package/lib/ui/assets/chunk-Des-eS-w.js +1 -0
- package/lib/ui/assets/chunk-Df6bDoY_.js +1 -0
- package/lib/ui/assets/chunk-DijEV5ha.js +1 -0
- package/lib/ui/assets/chunk-DkvoVjww.js +1 -0
- package/lib/ui/assets/chunk-DlC-OlHO.js +1 -0
- package/lib/ui/assets/chunk-DnzrXTVv.js +1 -0
- package/lib/ui/assets/chunk-Dualf0xi.js +1 -0
- package/lib/ui/assets/chunk-EXT84bkm.js +1 -0
- package/lib/ui/assets/chunk-GwR3hkPK.js +1 -0
- package/lib/ui/assets/chunk-P80f7IUj.js +1 -0
- package/lib/ui/assets/chunk-VOosw3JB.js +1 -0
- package/lib/ui/assets/chunk-YQ4lDkt_.js +140 -0
- package/lib/ui/assets/chunk-Yzrsuije.js +1 -0
- package/lib/ui/assets/chunk-Z72CF80r.js +1 -0
- package/lib/ui/assets/chunk-g9-lgVsj.js +1 -0
- package/lib/ui/assets/chunk-vGWfd6FD.js +1 -0
- package/lib/ui/assets/chunk-wDzz0qaB.js +1 -0
- package/lib/ui/assets/chunk-ySF7HWmS.js +1 -0
- package/lib/ui/assets/icon.png +0 -0
- package/lib/ui/assets/index.html +14 -0
- package/lib/ui/assets/pdf.worker.min-CjEcRF4W.mjs +28 -0
- package/lib/ui/assets/style-Bq01xH_b.css +1 -0
- package/lib/ui/frames.d.ts +67 -0
- package/lib/ui/frames.js +146 -0
- package/lib/ui/index.d.ts +59 -0
- package/lib/ui/index.js +50 -0
- package/lib/ui/manifest.d.ts +172 -0
- package/lib/ui/manifest.js +158 -0
- package/lib/ui/plugin.d.ts +161 -0
- package/lib/ui/plugin.js +7 -0
- package/lib/ui/plugins/browser/client.js +290 -0
- package/lib/ui/plugins/browser/icon.png +0 -0
- package/lib/ui/plugins/composer/client.js +1843 -0
- package/lib/ui/plugins/composer/icon.png +0 -0
- package/lib/ui/plugins/files/client.js +646 -0
- package/lib/ui/plugins/files/icon.png +0 -0
- package/lib/ui/plugins/manifest.json +107 -0
- package/lib/ui/plugins/plugins/client.js +682 -0
- package/lib/ui/plugins/plugins/icon.png +0 -0
- package/lib/ui/plugins/preview/client.js +197 -0
- package/lib/ui/plugins/preview/icon.png +0 -0
- package/lib/ui/plugins/prompt/client.js +653 -0
- package/lib/ui/plugins/prompt/icon.png +0 -0
- package/lib/ui/plugins/sessions/client.js +1577 -0
- package/lib/ui/plugins/sessions/icon.png +0 -0
- package/lib/ui/plugins/settings/client.js +5470 -0
- package/lib/ui/plugins/settings/icon.png +0 -0
- package/lib/ui/plugins/skills/client.js +485 -0
- package/lib/ui/plugins/skills/icon.png +0 -0
- package/lib/ui/plugins/status/client.js +1420 -0
- package/lib/ui/plugins/status/icon.png +0 -0
- package/lib/ui/plugins/statusbar/client.js +504 -0
- package/lib/ui/plugins/statusbar/icon.png +0 -0
- package/lib/ui/plugins/tasks/client.js +52 -0
- package/lib/ui/plugins/tasks/icon.png +0 -0
- package/lib/ui/plugins/team/client.js +527 -0
- package/lib/ui/plugins/team/icon.png +0 -0
- package/lib/ui/plugins/terminal/client.js +232 -0
- package/lib/ui/plugins/terminal/icon.png +0 -0
- package/lib/ui/plugins/trace/client.js +1376 -0
- package/lib/ui/plugins/trace/icon.png +0 -0
- package/lib/ui/plugins/transcript/client.js +2661 -0
- package/lib/ui/plugins/transcript/icon.png +0 -0
- package/lib/ui/scoped-slots.d.ts +20 -0
- package/lib/ui/scoped-slots.js +21 -0
- package/lib/ui/sdk/@dsc/ui/index.d.ts +242 -0
- package/lib/ui/sdk/@dsc/ui/package.json +16 -0
- package/lib/ui/sdk/@dsc/ui-core/contract.d.ts +1973 -0
- package/lib/ui/sdk/@dsc/ui-core/core/diff-text.d.ts +74 -0
- package/lib/ui/sdk/@dsc/ui-core/core/dock-browser.d.ts +137 -0
- package/lib/ui/sdk/@dsc/ui-core/core/dock-ops.d.ts +59 -0
- package/lib/ui/sdk/@dsc/ui-core/core/model-caps.d.ts +83 -0
- package/lib/ui/sdk/@dsc/ui-core/core/native-bridge.d.ts +135 -0
- package/lib/ui/sdk/@dsc/ui-core/core/token-estimate.d.ts +55 -0
- package/lib/ui/sdk/@dsc/ui-core/package.json +16 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/arrival.d.ts +72 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/frames.d.ts +67 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/index.d.ts +59 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/manifest.d.ts +172 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/plugin.d.ts +161 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/scoped-slots.d.ts +20 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/services-react.d.ts +27 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/services.d.ts +63 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/slots-react.d.ts +28 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/slots.d.ts +481 -0
- package/lib/ui/sdk/@dsc/ui-core/ui/store.d.ts +89 -0
- package/lib/ui/sdk/@dsc/ui-dockkit/dock-model.d.ts +238 -0
- package/lib/ui/sdk/@dsc/ui-dockkit/dock-state.d.ts +163 -0
- package/lib/ui/sdk/@dsc/ui-dockkit/dock.d.ts +24 -0
- package/lib/ui/sdk/@dsc/ui-dockkit/index.d.ts +15 -0
- package/lib/ui/sdk/@dsc/ui-dockkit/package.json +16 -0
- package/lib/ui/sdk/@dsc/ui-layout/app.d.ts +74 -0
- package/lib/ui/sdk/@dsc/ui-layout/appearance.d.ts +70 -0
- package/lib/ui/sdk/@dsc/ui-layout/host-channel.d.ts +112 -0
- package/lib/ui/sdk/@dsc/ui-layout/index.d.ts +34 -0
- package/lib/ui/sdk/@dsc/ui-layout/package.json +16 -0
- package/lib/ui/sdk/@dsc/ui-layout/page-state.d.ts +14 -0
- package/lib/ui/sdk/@dsc/ui-layout/review-pane.d.ts +21 -0
- package/lib/ui/sdk/@dsc/ui-layout/runtime-context.d.ts +44 -0
- package/lib/ui/sdk/@dsc/ui-layout/thread-view.d.ts +28 -0
- package/lib/ui/sdk/@dsc/ui-layout/update-button.d.ts +21 -0
- package/lib/ui/sdk/@dsc/ui-primitives/code-file-icon-artwork.d.ts +53 -0
- package/lib/ui/sdk/@dsc/ui-primitives/code-view.d.ts +15 -0
- package/lib/ui/sdk/@dsc/ui-primitives/compat.d.ts +27 -0
- package/lib/ui/sdk/@dsc/ui-primitives/completion-sound.d.ts +24 -0
- package/lib/ui/sdk/@dsc/ui-primitives/confirm.d.ts +33 -0
- package/lib/ui/sdk/@dsc/ui-primitives/css.d.ts +47 -0
- package/lib/ui/sdk/@dsc/ui-primitives/diff-rows.d.ts +20 -0
- package/lib/ui/sdk/@dsc/ui-primitives/file-icons.d.ts +38 -0
- package/lib/ui/sdk/@dsc/ui-primitives/file-type-classify.d.ts +66 -0
- package/lib/ui/sdk/@dsc/ui-primitives/file-type-icon.d.ts +57 -0
- package/lib/ui/sdk/@dsc/ui-primitives/highlight.d.ts +24 -0
- package/lib/ui/sdk/@dsc/ui-primitives/hover-delay.d.ts +17 -0
- package/lib/ui/sdk/@dsc/ui-primitives/icons.d.ts +109 -0
- package/lib/ui/sdk/@dsc/ui-primitives/index.d.ts +55 -0
- package/lib/ui/sdk/@dsc/ui-primitives/lang-table.d.ts +10 -0
- package/lib/ui/sdk/@dsc/ui-primitives/markdown.d.ts +20 -0
- package/lib/ui/sdk/@dsc/ui-primitives/native.d.ts +41 -0
- package/lib/ui/sdk/@dsc/ui-primitives/package.json +16 -0
- package/lib/ui/sdk/@dsc/ui-primitives/panels.d.ts +86 -0
- package/lib/ui/sdk/@dsc/ui-primitives/paths.d.ts +11 -0
- package/lib/ui/sdk/@dsc/ui-primitives/pdf-view.d.ts +13 -0
- package/lib/ui/sdk/@dsc/ui-primitives/plugin-text.d.ts +15 -0
- package/lib/ui/sdk/@dsc/ui-primitives/select.d.ts +35 -0
- package/lib/ui/sdk/@dsc/ui-primitives/table-view.d.ts +23 -0
- package/lib/ui/sdk/@dsc/ui-primitives/thread-resizer.d.ts +22 -0
- package/lib/ui/sdk/@dsc/ui-primitives/time.d.ts +33 -0
- package/lib/ui/sdk/@dsc/ui-primitives/toast.d.ts +16 -0
- package/lib/ui/sdk/@dsc/ui-primitives/tokens.d.ts +14 -0
- package/lib/ui/sdk/@dsc/ui-primitives/tool-args.d.ts +49 -0
- package/lib/ui/sdk/@dsc/ui-primitives/tool-summary.d.ts +64 -0
- package/lib/ui/sdk/@dsc/ui-renderer/boot.d.ts +121 -0
- package/lib/ui/sdk/@dsc/ui-renderer/hmr.d.ts +43 -0
- package/lib/ui/sdk/@dsc/ui-renderer/index.d.ts +20 -0
- package/lib/ui/sdk/@dsc/ui-renderer/package.json +16 -0
- package/lib/ui/sdk/@dsc/ui-slots/index.d.ts +19 -0
- package/lib/ui/sdk/@dsc/ui-slots/package.json +16 -0
- package/lib/ui/sdk/@dsc/ui-slots/slots-react.d.ts +48 -0
- package/lib/ui/sdk/@dsc/ui-slots/slots.d.ts +102 -0
- package/lib/ui/services-react.d.ts +27 -0
- package/lib/ui/services-react.js +32 -0
- package/lib/ui/services.d.ts +63 -0
- package/lib/ui/services.js +121 -0
- package/lib/ui/slots-ink.d.ts +26 -0
- package/lib/ui/slots-ink.js +78 -0
- package/lib/ui/slots-react.d.ts +28 -0
- package/lib/ui/slots-react.js +88 -0
- package/lib/ui/slots.d.ts +481 -0
- package/lib/ui/slots.js +550 -0
- package/lib/ui/store.d.ts +89 -0
- package/lib/ui/store.js +62 -0
- package/lib/ui/tui-plugin-api.d.ts +23 -0
- package/lib/ui/tui-plugin-api.js +23 -0
- package/lib/ui/tui-plugins.d.ts +47 -0
- package/lib/ui/tui-plugins.js +220 -0
- package/package.json +58 -6
|
@@ -0,0 +1,1311 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 桌面操作实现(Windows):截屏、点鼠标、敲键盘、拖拽、列窗口/应用/显示器,
|
|
3
|
+
* 以及无障碍(UIA)元素观察与定向动作。全部通过一次 PowerShell 子进程 +
|
|
4
|
+
* user32 P/Invoke + System.Windows.Automation + System.Drawing 完成,零 npm 依赖。
|
|
5
|
+
*
|
|
6
|
+
* 坐标系说明(重要):一律用**虚拟屏幕物理像素**。每次动作前进程内先
|
|
7
|
+
* `SetProcessDPIAware()`,于是截屏像素与屏幕像素一比一对齐;截图被缩放过时,
|
|
8
|
+
* 调用方传进来的仍是图上的像素,本模块自己按缩放系数换算回去。
|
|
9
|
+
*
|
|
10
|
+
* 无障碍观察(0.6.68 parity 补齐):新增 `get_app_state` 观察路径,用 UIA
|
|
11
|
+
* 枚举窗口控件树,产出结构化 elements + 文本树 + state_id。
|
|
12
|
+
* - 元素索引作用域 = 该窗口**最近一次观察**;元素消失一律 fail-closed(报错,
|
|
13
|
+
* 绝不退回坐标去乱点)。
|
|
14
|
+
* - 目标两种(判别字段 `type`):`{type:'element', index}` 或
|
|
15
|
+
* `{type:'coordinate', x, y}`(与 ZCode 的 target 语义一致)。
|
|
16
|
+
*
|
|
17
|
+
* 两个自研坑(dsh 把桌面操作外包给 Rust driver,所以它没有这两个坑):
|
|
18
|
+
* - `Graphics.CopyFromScreen` 在 DPI 缩放下会拍到缩放过的图,所以必须先声明 DPI 感知;
|
|
19
|
+
* - 文本输入走剪贴板粘贴,绕开 SendKeys 对中文和 `{}` 之类的限制,代价是覆盖剪贴板,
|
|
20
|
+
* 因此这里会先把原来的**文本**剪贴板存回去(图片/文件类剪贴板内容保不住)。
|
|
21
|
+
*
|
|
22
|
+
* @module dsc/core/desktop-control
|
|
23
|
+
*/
|
|
24
|
+
import { spawn } from 'node:child_process';
|
|
25
|
+
import { appendFileSync, mkdirSync } from 'node:fs';
|
|
26
|
+
import { dirname, join } from 'node:path';
|
|
27
|
+
/** 桌面动作(工具 action 枚举;名字尽量照 ZCode 语义)。 */
|
|
28
|
+
export const DESKTOP_ACTIONS = [
|
|
29
|
+
'screenshot',
|
|
30
|
+
'click',
|
|
31
|
+
'type',
|
|
32
|
+
'key',
|
|
33
|
+
'scroll',
|
|
34
|
+
'drag',
|
|
35
|
+
'paste',
|
|
36
|
+
'set_value',
|
|
37
|
+
'select_text',
|
|
38
|
+
'perform_action',
|
|
39
|
+
'cursor',
|
|
40
|
+
'list_windows',
|
|
41
|
+
'list_apps',
|
|
42
|
+
'screens',
|
|
43
|
+
'get_app_state',
|
|
44
|
+
'stop',
|
|
45
|
+
];
|
|
46
|
+
/** 旧名兼容:`window_list` 是 `list_windows` 的历史叫法(老配置里可能还写着它)。 */
|
|
47
|
+
export const ACTION_ALIASES = { window_list: 'list_windows' };
|
|
48
|
+
/** 把(可能是旧名的)动作名归一成规范名;不认识返回 null。 */
|
|
49
|
+
export function canonicalAction(name) {
|
|
50
|
+
if (DESKTOP_ACTIONS.includes(name))
|
|
51
|
+
return name;
|
|
52
|
+
return ACTION_ALIASES[name] ?? null;
|
|
53
|
+
}
|
|
54
|
+
/** 只读观察动作(看,不动手):免审批的那一档就是它们。 */
|
|
55
|
+
export const READ_ACTIONS = [
|
|
56
|
+
'screenshot',
|
|
57
|
+
'cursor',
|
|
58
|
+
'list_windows',
|
|
59
|
+
'list_apps',
|
|
60
|
+
'screens',
|
|
61
|
+
'get_app_state',
|
|
62
|
+
];
|
|
63
|
+
const DPI_A = "using System.Runtime.InteropServices; public class DscDpi { [DllImport(\"user32.dll\")] public static extern bool SetProcessDPIAware(); }";
|
|
64
|
+
const MOUSE_TYPE = `using System; using System.Runtime.InteropServices; public class DscMouse {
|
|
65
|
+
[DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y);
|
|
66
|
+
[DllImport("user32.dll")] public static extern void mouse_event(uint f, uint dx, uint dy, uint d, UIntPtr e);
|
|
67
|
+
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT p); public struct POINT { public int X; public int Y; } }`;
|
|
68
|
+
/** 窗口查询/激活的 P/Invoke(前台判断、白名单、窗口绑定、激活都用它)。 */
|
|
69
|
+
const WINDOW_TYPE = `using System; using System.Diagnostics; using System.Runtime.InteropServices; public class DscWin {
|
|
70
|
+
[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow();
|
|
71
|
+
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr h);
|
|
72
|
+
[DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr h, int cmd);
|
|
73
|
+
[DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr h, out uint pid);
|
|
74
|
+
[DllImport("user32.dll", CharSet=CharSet.Auto, SetLastError=true)] public static extern int GetWindowText(IntPtr h, System.Text.StringBuilder s, int n);
|
|
75
|
+
[DllImport("user32.dll")] public static extern bool GetWindowRect(IntPtr h, out RECT r);
|
|
76
|
+
[DllImport("user32.dll")] public static extern bool IsWindowVisible(IntPtr h);
|
|
77
|
+
[DllImport("user32.dll")] public static extern bool IsIconic(IntPtr h);
|
|
78
|
+
public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
|
79
|
+
static string Proc(IntPtr h) { uint pid = 0; GetWindowThreadProcessId(h, out pid); try { return Process.GetProcessById((int)pid).ProcessName; } catch { return "?"; } }
|
|
80
|
+
public static string Describe() { IntPtr h = GetForegroundWindow(); var sb = new System.Text.StringBuilder(512); GetWindowText(h, sb, sb.Capacity); return Proc(h) + "|" + sb.ToString(); } }`;
|
|
81
|
+
/** UIA 引导(观察与元素动作共用)。这段 PowerShell 里定义了 Walk / Patterns-Of / Value-Of 等小函数。 */
|
|
82
|
+
const UIA_BOOT = `Add-Type -AssemblyName UIAutomationClient
|
|
83
|
+
Add-Type -AssemblyName UIAutomationTypes
|
|
84
|
+
Add-Type -AssemblyName WindowsBase
|
|
85
|
+
Add-Type -TypeDefinition '${DPI_A}'
|
|
86
|
+
[DscDpi]::SetProcessDPIAware() | Out-Null
|
|
87
|
+
Add-Type -TypeDefinition '${WINDOW_TYPE}'
|
|
88
|
+
$scopeChildren = [System.Windows.Automation.TreeScope]::Children
|
|
89
|
+
$cond = [System.Windows.Automation.Condition]::TrueCondition
|
|
90
|
+
$walker = [System.Windows.Automation.TreeWalker]::ControlViewWalker
|
|
91
|
+
function Resolve-Win([int]$winId, [string]$winText) {
|
|
92
|
+
$w = $null
|
|
93
|
+
if ($winId -gt 0) { try { $w = [System.Windows.Automation.AutomationElement]::FromHandle([IntPtr]$winId) } catch { $w = $null } }
|
|
94
|
+
if ($null -eq $w -and $winText -ne '') {
|
|
95
|
+
$all = [System.Windows.Automation.AutomationElement]::RootElement.FindAll($scopeChildren, $cond)
|
|
96
|
+
foreach ($c in $all) { try { if ($c.Current.Name.IndexOf($winText, [System.StringComparison]::OrdinalIgnoreCase) -ge 0) { $w = $c; break } } catch { } }
|
|
97
|
+
}
|
|
98
|
+
if ($null -eq $w) { try { $w = [System.Windows.Automation.AutomationElement]::FromHandle([DscWin]::GetForegroundWindow()) } catch { $w = $null } }
|
|
99
|
+
return $w
|
|
100
|
+
}
|
|
101
|
+
function Proc-Of([int]$h) { $p = 0; [DscWin]::GetWindowThreadProcessId([IntPtr]$h, [ref]$p) | Out-Null; try { return [System.Diagnostics.Process]::GetProcessById($p).ProcessName } catch { return '?' } }
|
|
102
|
+
function Rect-Of($el) { try { $r = $el.Current.BoundingRectangle; if ($r.IsEmpty) { return $null }; return [pscustomobject]@{ x=[int]$r.X; y=[int]$r.Y; width=[int]$r.Width; height=[int]$r.Height } } catch { return $null } }
|
|
103
|
+
function Patterns-Of($el) {
|
|
104
|
+
$p = New-Object System.Collections.ArrayList
|
|
105
|
+
$o = $null
|
|
106
|
+
try { if ($el.TryGetCurrentPattern([System.Windows.Automation.InvokePattern]::Pattern, [ref]$o)) { [void]$p.Add('Invoke') } } catch { }
|
|
107
|
+
$o = $null
|
|
108
|
+
try { if ($el.TryGetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern, [ref]$o)) { [void]$p.Add('Toggle') } } catch { }
|
|
109
|
+
$o = $null
|
|
110
|
+
try { if ($el.TryGetCurrentPattern([System.Windows.Automation.SelectionItemPattern]::Pattern, [ref]$o)) { [void]$p.Add('SelectionItem') } } catch { }
|
|
111
|
+
$o = $null
|
|
112
|
+
try { if ($el.TryGetCurrentPattern([System.Windows.Automation.ExpandCollapsePattern]::Pattern, [ref]$o)) { [void]$p.Add('ExpandCollapse') } } catch { }
|
|
113
|
+
$o = $null
|
|
114
|
+
try { if ($el.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$o)) { [void]$p.Add('Value') } } catch { }
|
|
115
|
+
$o = $null
|
|
116
|
+
try { if ($el.TryGetCurrentPattern([System.Windows.Automation.ScrollPattern]::Pattern, [ref]$o)) { [void]$p.Add('Scroll') } } catch { }
|
|
117
|
+
$o = $null
|
|
118
|
+
try { if ($el.TryGetCurrentPattern([System.Windows.Automation.RangeValuePattern]::Pattern, [ref]$o)) { [void]$p.Add('RangeValue') } } catch { }
|
|
119
|
+
$o = $null
|
|
120
|
+
try { if ($el.TryGetCurrentPattern([System.Windows.Automation.TextPattern]::Pattern, [ref]$o)) { [void]$p.Add('Text') } } catch { }
|
|
121
|
+
return $p
|
|
122
|
+
}
|
|
123
|
+
function Value-Of($el) { try { $vp = $el.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern); return [string]$vp.Current.Value } catch { return '' } }
|
|
124
|
+
function Read-El($el, [int]$i) {
|
|
125
|
+
$ct = ''; $nm = ''; $aid = ''; $val = ''; $en = $true; $off = $false
|
|
126
|
+
try { $ct = [string]$el.Current.ControlType.ProgrammaticName; if ($ct.StartsWith('ControlType.')) { $ct = $ct.Substring(12) } } catch { }
|
|
127
|
+
try { $nm = [string]$el.Current.Name } catch { }
|
|
128
|
+
try { $aid = [string]$el.Current.AutomationId } catch { }
|
|
129
|
+
try { $en = [bool]$el.Current.IsEnabled } catch { }
|
|
130
|
+
try { $off = [bool]$el.Current.IsOffscreen } catch { }
|
|
131
|
+
if (-not $off) { $val = Value-Of $el }
|
|
132
|
+
return [pscustomobject]@{ index=$i; controlType=$ct; title=$nm; automaid=$aid; value=$val; bounds=(Rect-Of $el); patterns=(Patterns-Of $el); enabled=$en; offscreen=$off }
|
|
133
|
+
}`;
|
|
134
|
+
/** 鼠标键 → 按下/抬起事件位。 */
|
|
135
|
+
const BUTTON_EVENT = {
|
|
136
|
+
left: [2, 4],
|
|
137
|
+
right: [8, 16],
|
|
138
|
+
middle: [32, 64],
|
|
139
|
+
};
|
|
140
|
+
/** SendKeys 键名白名单(不认识的键名一律拒绝,避免把任意文本当语法喂进去)。 */
|
|
141
|
+
const KEY_MAP = {
|
|
142
|
+
enter: '{ENTER}',
|
|
143
|
+
tab: '{TAB}',
|
|
144
|
+
esc: '{ESC}',
|
|
145
|
+
escape: '{ESC}',
|
|
146
|
+
space: ' ',
|
|
147
|
+
backspace: '{BACKSPACE}',
|
|
148
|
+
delete: '{DELETE}',
|
|
149
|
+
insert: '{INSERT}',
|
|
150
|
+
home: '{HOME}',
|
|
151
|
+
end: '{END}',
|
|
152
|
+
pageup: '{PGUP}',
|
|
153
|
+
pagedown: '{PGDN}',
|
|
154
|
+
up: '{UP}',
|
|
155
|
+
down: '{DOWN}',
|
|
156
|
+
left: '{LEFT}',
|
|
157
|
+
right: '{RIGHT}',
|
|
158
|
+
f1: '{F1}',
|
|
159
|
+
f2: '{F2}',
|
|
160
|
+
f3: '{F3}',
|
|
161
|
+
f4: '{F4}',
|
|
162
|
+
f5: '{F5}',
|
|
163
|
+
f6: '{F6}',
|
|
164
|
+
f7: '{F7}',
|
|
165
|
+
f8: '{F8}',
|
|
166
|
+
f9: '{F9}',
|
|
167
|
+
f10: '{F10}',
|
|
168
|
+
f11: '{F11}',
|
|
169
|
+
f12: '{F12}',
|
|
170
|
+
};
|
|
171
|
+
/**
|
|
172
|
+
* 明确禁止的按键组合。SendKeys 本来就发不出 Win 键,这里再挡一层
|
|
173
|
+
* 那些「一发出去就把人锁在门外」的组合。
|
|
174
|
+
*/
|
|
175
|
+
const BANNED_KEYS = ['win', 'cmd', 'super', 'meta'];
|
|
176
|
+
/** 观察/遍历上限(防止一棵病态树把上下文撑爆;超了在文本里明说被截断)。 */
|
|
177
|
+
export const OBSERVE_MAX_ELEMENTS = 400;
|
|
178
|
+
function errText(error) {
|
|
179
|
+
return error instanceof Error ? error.message : String(error);
|
|
180
|
+
}
|
|
181
|
+
/** PowerShell 单引号字符串转义。 */
|
|
182
|
+
function psQuote(text) {
|
|
183
|
+
return `'${text.replace(/'/g, "''")}'`;
|
|
184
|
+
}
|
|
185
|
+
/** 正在跑的 PowerShell 子进程(急停时一次性杀掉)。 */
|
|
186
|
+
const activeChildren = new Set();
|
|
187
|
+
/**
|
|
188
|
+
* 急停:杀掉所有在跑的 PowerShell 子进程。
|
|
189
|
+
* @returns 实际尝试杀掉的进程数(0 = 当时没有在跑的)。
|
|
190
|
+
*/
|
|
191
|
+
export function killAllPowerShell() {
|
|
192
|
+
const count = activeChildren.size;
|
|
193
|
+
for (const child of activeChildren) {
|
|
194
|
+
try {
|
|
195
|
+
child.kill();
|
|
196
|
+
}
|
|
197
|
+
catch {
|
|
198
|
+
// 已经退出/杀不掉都无所谓,急停只求尽快
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
activeChildren.clear();
|
|
202
|
+
return count;
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* 跑一段 PowerShell 并取 stdout。
|
|
206
|
+
* @param signal - 外部取消时立刻杀掉子进程(用户中断这一轮,不该留一个正在点鼠标的进程)。
|
|
207
|
+
*/
|
|
208
|
+
export function runPowerShell(script, timeoutMs = 20000, signal) {
|
|
209
|
+
return new Promise((resolve, reject) => {
|
|
210
|
+
const child = spawn('powershell.exe', ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], { windowsHide: true });
|
|
211
|
+
activeChildren.add(child);
|
|
212
|
+
let out = '';
|
|
213
|
+
let err = '';
|
|
214
|
+
const timer = setTimeout(() => {
|
|
215
|
+
child.kill();
|
|
216
|
+
reject(new Error(`桌面操作超时(${Math.round(timeoutMs / 1000)} 秒)`));
|
|
217
|
+
}, timeoutMs);
|
|
218
|
+
const abort = () => {
|
|
219
|
+
child.kill();
|
|
220
|
+
reject(new Error('桌面操作被取消'));
|
|
221
|
+
};
|
|
222
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
223
|
+
const clean = () => {
|
|
224
|
+
activeChildren.delete(child);
|
|
225
|
+
clearTimeout(timer);
|
|
226
|
+
signal?.removeEventListener('abort', abort);
|
|
227
|
+
};
|
|
228
|
+
child.stdout.on('data', (chunk) => {
|
|
229
|
+
out += chunk.toString();
|
|
230
|
+
});
|
|
231
|
+
child.stderr.on('data', (chunk) => {
|
|
232
|
+
err += chunk.toString();
|
|
233
|
+
});
|
|
234
|
+
child.on('close', (code) => {
|
|
235
|
+
clean();
|
|
236
|
+
if (code === 0)
|
|
237
|
+
resolve(out);
|
|
238
|
+
else
|
|
239
|
+
reject(new Error(err.trim() !== '' ? err.trim() : `powershell 退出码 ${String(code)}`));
|
|
240
|
+
});
|
|
241
|
+
child.on('error', (error) => {
|
|
242
|
+
clean();
|
|
243
|
+
reject(error);
|
|
244
|
+
});
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
/** 把 `ctrl+shift+s` 这类键名翻成 SendKeys 语法。 */
|
|
248
|
+
export function toSendKeys(key) {
|
|
249
|
+
const parts = String(key)
|
|
250
|
+
.toLowerCase()
|
|
251
|
+
.split('+')
|
|
252
|
+
.map((part) => part.trim())
|
|
253
|
+
.filter((part) => part !== '');
|
|
254
|
+
let modifiers = '';
|
|
255
|
+
const singles = [];
|
|
256
|
+
for (const part of parts) {
|
|
257
|
+
if (part === 'ctrl')
|
|
258
|
+
modifiers += '^';
|
|
259
|
+
else if (part === 'alt')
|
|
260
|
+
modifiers += '%';
|
|
261
|
+
else if (part === 'shift')
|
|
262
|
+
modifiers += '+';
|
|
263
|
+
else if (BANNED_KEYS.includes(part)) {
|
|
264
|
+
throw new Error(`不允许按 Win 键(${part}):这类按键会把桌面切走,请用 click 点你要的地方`);
|
|
265
|
+
}
|
|
266
|
+
else if (KEY_MAP[part] !== undefined)
|
|
267
|
+
singles.push(KEY_MAP[part]);
|
|
268
|
+
else if (part.length === 1)
|
|
269
|
+
singles.push(part);
|
|
270
|
+
else
|
|
271
|
+
throw new Error(`不认识的按键「${part}」。认得的:${Object.keys(KEY_MAP).join(' / ')},或单字符`);
|
|
272
|
+
}
|
|
273
|
+
if (singles.length === 0)
|
|
274
|
+
throw new Error(`按键 ${key} 里没有可执行的键`);
|
|
275
|
+
return modifiers + singles.join('');
|
|
276
|
+
}
|
|
277
|
+
// ── 脚本生成(全部导出,自检直接对文本断言) ─────────────────────────────
|
|
278
|
+
/** 截屏脚本:DPI 感知 → 抓虚拟屏(或裁切块)→ 需要时缩放 → 按质量存 JPEG/PNG → base64。 */
|
|
279
|
+
export function screenshotScript(maxEdge, format, quality, clip) {
|
|
280
|
+
const encode = format === 'jpeg'
|
|
281
|
+
? `$codec = [System.Drawing.Imaging.ImageCodecInfo]::GetImageEncoders() | Where-Object { $_.MimeType -eq 'image/jpeg' }
|
|
282
|
+
$ep = New-Object System.Drawing.Imaging.EncoderParameters 1
|
|
283
|
+
$ep.Param[0] = New-Object System.Drawing.Imaging.EncoderParameter([System.Drawing.Imaging.Encoder]::Quality, [long]${quality})
|
|
284
|
+
$bmp.Save($ms, $codec, $ep)`
|
|
285
|
+
: '$bmp.Save($ms, [System.Drawing.Imaging.ImageFormat]::Png)';
|
|
286
|
+
const hasClip = clip !== undefined && clip.width > 0 && clip.height > 0;
|
|
287
|
+
const region = hasClip
|
|
288
|
+
? `$srcRect = New-Object System.Drawing.Rectangle ([int]($b.Left + ${Math.round(clip.x)})), ([int]($b.Top + ${Math.round(clip.y)})), ${Math.round(clip.width)}, ${Math.round(clip.height)}
|
|
289
|
+
$ox = ${Math.round(clip.x)}; $oy = ${Math.round(clip.y)}`
|
|
290
|
+
: `$srcRect = New-Object System.Drawing.Rectangle $b.Left, $b.Top, $b.Width, $b.Height
|
|
291
|
+
$ox = 0; $oy = 0`;
|
|
292
|
+
return `
|
|
293
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
294
|
+
Add-Type -AssemblyName System.Drawing
|
|
295
|
+
Add-Type -TypeDefinition '${DPI_A}'
|
|
296
|
+
[DscDpi]::SetProcessDPIAware() | Out-Null
|
|
297
|
+
$b = [System.Windows.Forms.SystemInformation]::VirtualScreen
|
|
298
|
+
${region}
|
|
299
|
+
$src = New-Object System.Drawing.Bitmap $srcRect.Width, $srcRect.Height
|
|
300
|
+
$g = [System.Drawing.Graphics]::FromImage($src)
|
|
301
|
+
$g.CopyFromScreen($srcRect.Left, $srcRect.Top, 0, 0, $src.Size)
|
|
302
|
+
$max = ${maxEdge}
|
|
303
|
+
$w = $src.Width; $h = $src.Height
|
|
304
|
+
if ($w -gt $max -or $h -gt $max) {
|
|
305
|
+
if ($w -ge $h) { $nh = [int]([double]$max * $h / $w); $nw = $max } else { $nw = [int]([double]$max * $w / $h); $nh = $max }
|
|
306
|
+
$bmp = New-Object System.Drawing.Bitmap $nw, $nh
|
|
307
|
+
$g2 = [System.Drawing.Graphics]::FromImage($bmp)
|
|
308
|
+
$g2.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic
|
|
309
|
+
$g2.DrawImage($src, 0, 0, $w, $h)
|
|
310
|
+
} else { $bmp = $src; $nw = $w; $nh = $h }
|
|
311
|
+
$ms = New-Object System.IO.MemoryStream
|
|
312
|
+
${encode}
|
|
313
|
+
"$nw,$nh,$srcRect.Width,$srcRect.Height,$ox,$oy,$($b.Left),$($b.Top),$($b.Width),$($b.Height)|" + [Convert]::ToBase64String($ms.ToArray())
|
|
314
|
+
`;
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* 无障碍观察脚本:枚举目标窗口(或前台窗口)的控件树,输出一行 JSON。
|
|
318
|
+
* UIA 不可用时输出 `{ok:false,error:"…"}`(Node 侧抛成明确中文),绝不静默。
|
|
319
|
+
*/
|
|
320
|
+
export function observeScript(options = {}) {
|
|
321
|
+
const winId = Math.max(0, Math.round(options.windowId ?? 0));
|
|
322
|
+
const winText = psQuote(String(options.windowText ?? ''));
|
|
323
|
+
const max = Math.max(10, Math.round(options.maxElements ?? OBSERVE_MAX_ELEMENTS));
|
|
324
|
+
return `
|
|
325
|
+
$ErrorActionPreference = 'Stop'
|
|
326
|
+
try {
|
|
327
|
+
${UIA_BOOT}
|
|
328
|
+
$win = Resolve-Win ${winId} ${winText}
|
|
329
|
+
if ($null -eq $win) { throw '拿不到目标窗口(前台窗口句柄无效,或标题没匹配上)' }
|
|
330
|
+
$wh = [int]$win.Current.NativeWindowHandle
|
|
331
|
+
$wtitle = ''; try { $wtitle = [string]$win.Current.Name } catch { }
|
|
332
|
+
$proc = Proc-Of $wh
|
|
333
|
+
$wb = Rect-Of $win
|
|
334
|
+
$max = ${max}
|
|
335
|
+
$script:idx = 0
|
|
336
|
+
$items = New-Object System.Collections.ArrayList
|
|
337
|
+
$lines = New-Object System.Collections.ArrayList
|
|
338
|
+
function Walk($el, [int]$depth) {
|
|
339
|
+
if ($script:idx -ge $max) { return }
|
|
340
|
+
$i = $script:idx; $script:idx = $i + 1
|
|
341
|
+
$o = Read-El $el $i
|
|
342
|
+
[void]$items.Add($o)
|
|
343
|
+
$line = (' ' * $depth) + '[' + $i + '] ' + $o.controlType
|
|
344
|
+
if ($o.title -ne '') { $line += ' "' + $o.title + '"' }
|
|
345
|
+
if ($o.value -ne '') { $line += ' = "' + $o.value + '"' }
|
|
346
|
+
if ($o.offscreen) { $line += ' (屏外)' }
|
|
347
|
+
[void]$lines.Add($line)
|
|
348
|
+
$child = $walker.GetFirstChild($el)
|
|
349
|
+
while ($null -ne $child) {
|
|
350
|
+
if ($script:idx -ge $max) { break }
|
|
351
|
+
Walk $child ($depth + 1)
|
|
352
|
+
$child = $walker.GetNextSibling($child)
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
Walk $win 0
|
|
356
|
+
if ($script:idx -ge $max) { [void]$lines.Add('…(控件数超过 ' + $max + ',已截断)') }
|
|
357
|
+
$obj = [pscustomobject]@{ ok=$true; error=''; app=$proc; window=[pscustomobject]@{ id=$wh; title=$wtitle; bounds=$wb }; elements=$items; text=($lines -join "\`n") }
|
|
358
|
+
$obj | ConvertTo-Json -Depth 8 -Compress
|
|
359
|
+
} catch {
|
|
360
|
+
[pscustomobject]@{ ok=$false; error=[string]$_.Exception.Message; app=''; window=$null; elements=@(); text='' } | ConvertTo-Json -Depth 8 -Compress
|
|
361
|
+
}
|
|
362
|
+
`;
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* 元素动作脚本:重新遍历到第 index 个控件,按 pattern 派发(set_value / select_text / perform_action)。
|
|
366
|
+
* 输出 JSON `{ok, error, method}`;method='none' 表示该控件没有对应模式(Node 侧据此回退)。
|
|
367
|
+
*/
|
|
368
|
+
export function elementActionScript(kind, options) {
|
|
369
|
+
const winId = Math.max(0, Math.round(options.windowId ?? 0));
|
|
370
|
+
const winText = psQuote(String(options.windowText ?? ''));
|
|
371
|
+
const index = Math.max(0, Math.round(options.index));
|
|
372
|
+
const dispatch = buildElementDispatch(kind, options.pattern, options.value);
|
|
373
|
+
return `
|
|
374
|
+
$ErrorActionPreference = 'Stop'
|
|
375
|
+
try {
|
|
376
|
+
${UIA_BOOT}
|
|
377
|
+
$win = Resolve-Win ${winId} ${winText}
|
|
378
|
+
if ($null -eq $win) { throw '拿不到目标窗口,元素索引无法解析' }
|
|
379
|
+
$target = ${index}
|
|
380
|
+
$script:idx = 0
|
|
381
|
+
$script:found = $null
|
|
382
|
+
function Find($el) {
|
|
383
|
+
if ($null -ne $script:found) { return }
|
|
384
|
+
if ($script:idx -eq $target) { $script:found = $el; return }
|
|
385
|
+
$script:idx = $script:idx + 1
|
|
386
|
+
$child = $walker.GetFirstChild($el)
|
|
387
|
+
while ($null -ne $child) {
|
|
388
|
+
if ($null -ne $script:found) { return }
|
|
389
|
+
Find $child
|
|
390
|
+
$child = $walker.GetNextSibling($child)
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
Find $win
|
|
394
|
+
if ($null -eq $script:found) { throw '元素 #' + $target + ' 已经不在了(界面可能变了,请重新 get_app_state)' }
|
|
395
|
+
$method = ${dispatch}
|
|
396
|
+
[pscustomobject]@{ ok=$true; error=''; method=$method } | ConvertTo-Json -Compress
|
|
397
|
+
} catch {
|
|
398
|
+
[pscustomobject]@{ ok=$false; error=[string]$_.Exception.Message; method='error' } | ConvertTo-Json -Compress
|
|
399
|
+
}
|
|
400
|
+
`;
|
|
401
|
+
}
|
|
402
|
+
/** 生成元素动作的 PowerShell 派发片段(返回值:'value' | 'invoke' | 'toggle' | 'expand' | 'collapse' | 'select' | 'selecttext' | 'scrollinto' | 'none')。 */
|
|
403
|
+
function buildElementDispatch(kind, pattern, value) {
|
|
404
|
+
if (kind === 'set_value') {
|
|
405
|
+
return `$vp = $null
|
|
406
|
+
if ($script:found.TryGetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern, [ref]$vp)) { $vp.SetValue(${psQuote(String(value ?? ''))}); 'value' } else { 'none' }`;
|
|
407
|
+
}
|
|
408
|
+
if (kind === 'select_text') {
|
|
409
|
+
return `$tp = $null
|
|
410
|
+
if ($script:found.TryGetCurrentPattern([System.Windows.Automation.TextPattern]::Pattern, [ref]$tp)) { $tp.DocumentRange.Select(); 'selecttext' }
|
|
411
|
+
else { $si = $null; if ($script:found.TryGetCurrentPattern([System.Windows.Automation.SelectionItemPattern]::Pattern, [ref]$si)) { $si.Select(); 'select' } else { 'none' } }`;
|
|
412
|
+
}
|
|
413
|
+
const name = String(pattern ?? 'invoke').toLowerCase();
|
|
414
|
+
const map = {
|
|
415
|
+
invoke: `$p = $null; if ($script:found.TryGetCurrentPattern([System.Windows.Automation.InvokePattern]::Pattern, [ref]$p)) { $p.Invoke(); 'invoke' } else { 'none' }`,
|
|
416
|
+
toggle: `$p = $null; if ($script:found.TryGetCurrentPattern([System.Windows.Automation.TogglePattern]::Pattern, [ref]$p)) { $p.Toggle(); 'toggle' } else { 'none' }`,
|
|
417
|
+
expand: `$p = $null; if ($script:found.TryGetCurrentPattern([System.Windows.Automation.ExpandCollapsePattern]::Pattern, [ref]$p)) { $p.Expand(); 'expand' } else { 'none' }`,
|
|
418
|
+
collapse: `$p = $null; if ($script:found.TryGetCurrentPattern([System.Windows.Automation.ExpandCollapsePattern]::Pattern, [ref]$p)) { $p.Collapse(); 'collapse' } else { 'none' }`,
|
|
419
|
+
select: `$p = $null; if ($script:found.TryGetCurrentPattern([System.Windows.Automation.SelectionItemPattern]::Pattern, [ref]$p)) { $p.Select(); 'select' } else { 'none' }`,
|
|
420
|
+
scrollintoview: `$p = $null; if ($script:found.TryGetCurrentPattern([System.Windows.Automation.ScrollItemPattern]::Pattern, [ref]$p)) { $p.ScrollIntoView(); 'scrollinto' } else { 'none' }`,
|
|
421
|
+
};
|
|
422
|
+
const body = map[name];
|
|
423
|
+
if (body === undefined) {
|
|
424
|
+
return `throw '不认识的 action「${name}」。认得的:${Object.keys(map).join(' / ')}'`;
|
|
425
|
+
}
|
|
426
|
+
return body;
|
|
427
|
+
}
|
|
428
|
+
/** 拖拽脚本:起点按下 → 分步插值移动到终点 → **finally 里保证抬起**。 */
|
|
429
|
+
export function dragScript(from, to, steps, stepDelayMs) {
|
|
430
|
+
const n = Math.max(2, Math.round(steps));
|
|
431
|
+
const delay = Math.max(0, Math.round(stepDelayMs));
|
|
432
|
+
return `
|
|
433
|
+
Add-Type -TypeDefinition '${MOUSE_TYPE}'
|
|
434
|
+
[DscMouse]::SetCursorPos(${Math.round(from.x)}, ${Math.round(from.y)}) | Out-Null
|
|
435
|
+
Start-Sleep -Milliseconds 60
|
|
436
|
+
[DscMouse]::mouse_event(2,0,0,0,[UIntPtr]::Zero)
|
|
437
|
+
try {
|
|
438
|
+
for ($i = 1; $i -le ${n}; $i++) {
|
|
439
|
+
$px = [int](${Math.round(from.x)} + (${Math.round(to.x)} - ${Math.round(from.x)}) * $i / ${n})
|
|
440
|
+
$py = [int](${Math.round(from.y)} + (${Math.round(to.y)} - ${Math.round(from.y)}) * $i / ${n})
|
|
441
|
+
[DscMouse]::SetCursorPos($px, $py) | Out-Null
|
|
442
|
+
Start-Sleep -Milliseconds ${delay}
|
|
443
|
+
}
|
|
444
|
+
} finally {
|
|
445
|
+
[DscMouse]::mouse_event(4,0,0,0,[UIntPtr]::Zero)
|
|
446
|
+
}
|
|
447
|
+
'Done'
|
|
448
|
+
`;
|
|
449
|
+
}
|
|
450
|
+
/** 点击脚本:移动到点 → 单击/双击(可选先激活窗口)。 */
|
|
451
|
+
export function clickScript(point, button, double, activateLine, pauseMs) {
|
|
452
|
+
const events = BUTTON_EVENT[button] ?? BUTTON_EVENT.left;
|
|
453
|
+
const pause = pauseMs > 0 ? `Start-Sleep -Milliseconds ${pauseMs}` : '';
|
|
454
|
+
const clicks = double ? 2 : 1;
|
|
455
|
+
const clickLines = Array.from({ length: clicks }, () => `[DscMouse]::mouse_event(${events[0]},0,0,0,[UIntPtr]::Zero); Start-Sleep -Milliseconds 30; [DscMouse]::mouse_event(${events[1]},0,0,0,[UIntPtr]::Zero)`).join(`; ${pause}; `);
|
|
456
|
+
return `
|
|
457
|
+
Add-Type -TypeDefinition '${MOUSE_TYPE}'
|
|
458
|
+
Add-Type -TypeDefinition '${WINDOW_TYPE}'
|
|
459
|
+
${activateLine}
|
|
460
|
+
[DscMouse]::SetCursorPos(${Math.round(point.x)}, ${Math.round(point.y)}) | Out-Null
|
|
461
|
+
${pause}
|
|
462
|
+
${clickLines}
|
|
463
|
+
'Done'
|
|
464
|
+
`;
|
|
465
|
+
}
|
|
466
|
+
/** 滚轮脚本:可选先把鼠标移到某点,再滚 amount 档。 */
|
|
467
|
+
export function scrollScript(amount, move, pauseMs) {
|
|
468
|
+
const pause = pauseMs > 0 ? `Start-Sleep -Milliseconds ${pauseMs}` : '';
|
|
469
|
+
const step = amount > 0 ? '4294967176' : '120';
|
|
470
|
+
return `
|
|
471
|
+
Add-Type -TypeDefinition '${MOUSE_TYPE}'
|
|
472
|
+
${move}
|
|
473
|
+
for ($i = 0; $i -lt ${Math.abs(amount)}; $i++) { [DscMouse]::mouse_event(0x0800, 0, 0, ${step}, [UIntPtr]::Zero); ${pause} }
|
|
474
|
+
'Done'
|
|
475
|
+
`;
|
|
476
|
+
}
|
|
477
|
+
/** 按键脚本:按住 hold_seconds(可选)再松开,repeat 次。 */
|
|
478
|
+
export function keyScript(keys, repeat, holdSeconds) {
|
|
479
|
+
const hold = holdSeconds > 0 ? `Start-Sleep -Milliseconds ${Math.round(holdSeconds * 1000)}` : '';
|
|
480
|
+
return `
|
|
481
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
482
|
+
for ($i = 0; $i -lt ${Math.max(1, Math.round(repeat))}; $i++) {
|
|
483
|
+
[System.Windows.Forms.SendKeys]::SendWait(${psQuote(keys)})
|
|
484
|
+
${hold}
|
|
485
|
+
}
|
|
486
|
+
'Done'
|
|
487
|
+
`;
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* 粘贴脚本:把 text(可选)写进系统剪贴板,Ctrl+V,然后**还原用户原来的文本剪贴板**。
|
|
491
|
+
* `text` 省略时直接用当前剪贴板(不动它,也不还原)。
|
|
492
|
+
*/
|
|
493
|
+
export function pasteScript(text) {
|
|
494
|
+
if (text === undefined) {
|
|
495
|
+
return `
|
|
496
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
497
|
+
[System.Windows.Forms.SendKeys]::SendWait('^v')
|
|
498
|
+
'Done'
|
|
499
|
+
`;
|
|
500
|
+
}
|
|
501
|
+
return `
|
|
502
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
503
|
+
$old = $null
|
|
504
|
+
try { $old = Get-Clipboard -TextFormatType Text } catch { }
|
|
505
|
+
Set-Clipboard -Value ${psQuote(text)}
|
|
506
|
+
[System.Windows.Forms.SendKeys]::SendWait('^v')
|
|
507
|
+
Start-Sleep -Milliseconds 80
|
|
508
|
+
if ($null -ne $old) { Set-Clipboard -Value $old }
|
|
509
|
+
'Done'
|
|
510
|
+
`;
|
|
511
|
+
}
|
|
512
|
+
/** 文本输入脚本:写剪贴板 → 粘贴 → 还原原剪贴板。 */
|
|
513
|
+
export function typeScript(text) {
|
|
514
|
+
return `
|
|
515
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
516
|
+
$old = $null
|
|
517
|
+
try { $old = Get-Clipboard -TextFormatType Text } catch { }
|
|
518
|
+
Set-Clipboard -Value ${psQuote(text)}
|
|
519
|
+
[System.Windows.Forms.SendKeys]::SendWait('^v')
|
|
520
|
+
Start-Sleep -Milliseconds 80
|
|
521
|
+
if ($null -ne $old) { Set-Clipboard -Value $old }
|
|
522
|
+
'Done'
|
|
523
|
+
`;
|
|
524
|
+
}
|
|
525
|
+
/** 列窗口脚本(增强:window_id/title/app/bounds/main/focused/onscreen)。 */
|
|
526
|
+
export function listWindowsScript() {
|
|
527
|
+
return `
|
|
528
|
+
$ErrorActionPreference = 'Stop'
|
|
529
|
+
try {
|
|
530
|
+
${UIA_BOOT}
|
|
531
|
+
$root = [System.Windows.Automation.AutomationElement]::RootElement
|
|
532
|
+
$fg = [int][DscWin]::GetForegroundWindow()
|
|
533
|
+
$wins = $root.FindAll($scopeChildren, $cond)
|
|
534
|
+
$out = New-Object System.Collections.ArrayList
|
|
535
|
+
foreach ($w in $wins) {
|
|
536
|
+
try {
|
|
537
|
+
$h = [int]$w.Current.NativeWindowHandle
|
|
538
|
+
if ($h -le 0) { continue }
|
|
539
|
+
$mainH = 0
|
|
540
|
+
try { $p = 0; [DscWin]::GetWindowThreadProcessId([IntPtr]$h, [ref]$p) | Out-Null; $mainH = [int]([System.Diagnostics.Process]::GetProcessById($p)).MainWindowHandle } catch { $mainH = 0 }
|
|
541
|
+
$r = Rect-Of $w
|
|
542
|
+
[void]$out.Add([pscustomobject]@{ window_id=$h; title=[string]$w.Current.Name; app=(Proc-Of $h); bounds=$r; main=($h -eq $mainH); focused=($h -eq $fg); onscreen=(-not [bool]$w.Current.IsOffscreen) })
|
|
543
|
+
} catch { }
|
|
544
|
+
}
|
|
545
|
+
[pscustomobject]@{ ok=$true; error=''; windows=$out } | ConvertTo-Json -Depth 6 -Compress
|
|
546
|
+
} catch {
|
|
547
|
+
[pscustomobject]@{ ok=$false; error=[string]$_.Exception.Message; windows=@() } | ConvertTo-Json -Depth 6 -Compress
|
|
548
|
+
}
|
|
549
|
+
`;
|
|
550
|
+
}
|
|
551
|
+
/** 列应用脚本:进程名 + 主窗口 + 是否前台。 */
|
|
552
|
+
export function listAppsScript() {
|
|
553
|
+
return `
|
|
554
|
+
$ErrorActionPreference = 'Stop'
|
|
555
|
+
try {
|
|
556
|
+
Add-Type -TypeDefinition '${DPI_A}'
|
|
557
|
+
[DscDpi]::SetProcessDPIAware() | Out-Null
|
|
558
|
+
Add-Type -TypeDefinition '${WINDOW_TYPE}'
|
|
559
|
+
$fgh = [DscWin]::GetForegroundWindow()
|
|
560
|
+
$fgp = 0
|
|
561
|
+
[DscWin]::GetWindowThreadProcessId($fgh, [ref]$fgp) | Out-Null
|
|
562
|
+
$out = New-Object System.Collections.ArrayList
|
|
563
|
+
foreach ($p in (Get-Process | Where-Object { $_.MainWindowHandle -ne 0 } | Sort-Object ProcessName -Unique)) {
|
|
564
|
+
[void]$out.Add([pscustomobject]@{ pid=$p.Id; app=$p.ProcessName; title=$p.MainWindowTitle; foreground=($p.Id -eq [int]$fgp) })
|
|
565
|
+
}
|
|
566
|
+
[pscustomobject]@{ ok=$true; error=''; apps=$out } | ConvertTo-Json -Depth 6 -Compress
|
|
567
|
+
} catch {
|
|
568
|
+
[pscustomobject]@{ ok=$false; error=[string]$_.Exception.Message; apps=@() } | ConvertTo-Json -Depth 6 -Compress
|
|
569
|
+
}
|
|
570
|
+
`;
|
|
571
|
+
}
|
|
572
|
+
/** 显示器枚举脚本(含负坐标)。 */
|
|
573
|
+
export function screensScript() {
|
|
574
|
+
return `
|
|
575
|
+
$ErrorActionPreference = 'Stop'
|
|
576
|
+
try {
|
|
577
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
578
|
+
Add-Type -TypeDefinition '${DPI_A}'
|
|
579
|
+
[DscDpi]::SetProcessDPIAware() | Out-Null
|
|
580
|
+
$out = New-Object System.Collections.ArrayList
|
|
581
|
+
foreach ($s in [System.Windows.Forms.Screen]::AllScreens) {
|
|
582
|
+
[void]$out.Add([pscustomobject]@{ device=$s.DeviceName; primary=$s.Primary; bounds=[pscustomobject]@{ x=[int]$s.Bounds.X; y=[int]$s.Bounds.Y; width=[int]$s.Bounds.Width; height=[int]$s.Bounds.Height } })
|
|
583
|
+
}
|
|
584
|
+
[pscustomobject]@{ ok=$true; error=''; screens=$out } | ConvertTo-Json -Depth 6 -Compress
|
|
585
|
+
} catch {
|
|
586
|
+
[pscustomobject]@{ ok=$false; error=[string]$_.Exception.Message; screens=@() } | ConvertTo-Json -Depth 6 -Compress
|
|
587
|
+
}
|
|
588
|
+
`;
|
|
589
|
+
}
|
|
590
|
+
/** 窗口探测脚本:解析 window 选择器,返回句柄 + 标题 + 矩形(绑定坐标与激活用)。 */
|
|
591
|
+
export function probeWindowScript(selector) {
|
|
592
|
+
const isHandle = typeof selector === 'number' || /^\d+$/.test(String(selector).trim());
|
|
593
|
+
const winId = isHandle ? Math.max(0, Math.round(Number(selector))) : 0;
|
|
594
|
+
const winText = isHandle ? '' : String(selector);
|
|
595
|
+
return `
|
|
596
|
+
$ErrorActionPreference = 'Stop'
|
|
597
|
+
try {
|
|
598
|
+
Add-Type -TypeDefinition '${DPI_A}'
|
|
599
|
+
[DscDpi]::SetProcessDPIAware() | Out-Null
|
|
600
|
+
Add-Type -TypeDefinition '${WINDOW_TYPE}'
|
|
601
|
+
$h = [IntPtr]::Zero
|
|
602
|
+
$winId = ${winId}
|
|
603
|
+
$winText = ${psQuote(winText)}
|
|
604
|
+
if ($winId -gt 0) { $h = [IntPtr]$winId }
|
|
605
|
+
elseif ($winText -ne '') {
|
|
606
|
+
foreach ($p in (Get-Process | Where-Object { $_.MainWindowHandle -ne 0 })) {
|
|
607
|
+
try { if ($p.MainWindowTitle.IndexOf($winText, [System.StringComparison]::OrdinalIgnoreCase) -ge 0 -or $p.ProcessName.IndexOf($winText, [System.StringComparison]::OrdinalIgnoreCase) -ge 0) { $h = $p.MainWindowHandle; break } } catch { }
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
if ($h -eq [IntPtr]::Zero) { throw '找不到窗口「' + $winText + '」' }
|
|
611
|
+
$r = New-Object DscWin+RECT
|
|
612
|
+
[DscWin]::GetWindowRect($h, [ref]$r) | Out-Null
|
|
613
|
+
$sb = New-Object System.Text.StringBuilder 512
|
|
614
|
+
[DscWin]::GetWindowText($h, $sb, $sb.Capacity) | Out-Null
|
|
615
|
+
[pscustomobject]@{ ok=$true; error=''; id=[int]$h; title=$sb.ToString(); bounds=[pscustomobject]@{ x=$r.Left; y=$r.Top; width=($r.Right - $r.Left); height=($r.Bottom - $r.Top) } } | ConvertTo-Json -Compress
|
|
616
|
+
} catch {
|
|
617
|
+
[pscustomobject]@{ ok=$false; error=[string]$_.Exception.Message; id=0; title=''; bounds=$null } | ConvertTo-Json -Compress
|
|
618
|
+
}
|
|
619
|
+
`;
|
|
620
|
+
}
|
|
621
|
+
// ── 纯函数:目标解析 / 帧解析 / 差分 / settle ─────────────────────────────
|
|
622
|
+
function assertInt(value, label) {
|
|
623
|
+
const num = Number(value);
|
|
624
|
+
if (!Number.isInteger(num))
|
|
625
|
+
throw new Error(`${label} 必须是整数,收到 ${String(value)}`);
|
|
626
|
+
return num;
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* 把截图像素坐标换算成屏幕物理坐标。
|
|
630
|
+
* @throws 小数坐标,或换算后落在屏幕外(这里不靠 SetCursorPos 静默夹紧,越界就是要告诉模型)。
|
|
631
|
+
*/
|
|
632
|
+
export function resolveScreenPoint(args, bounds, scale) {
|
|
633
|
+
const rawX = args.x;
|
|
634
|
+
const rawY = args.y;
|
|
635
|
+
if (rawX === undefined || rawY === undefined)
|
|
636
|
+
throw new Error('这个动作要带 x 与 y(截图上的像素坐标)');
|
|
637
|
+
if (!Number.isInteger(rawX) || !Number.isInteger(rawY)) {
|
|
638
|
+
throw new Error(`x 与 y 必须是整数,收到 (${String(rawX)}, ${String(rawY)})`);
|
|
639
|
+
}
|
|
640
|
+
return clampToScreen(Math.round(rawX / scale), Math.round(rawY / scale), bounds);
|
|
641
|
+
}
|
|
642
|
+
function clampToScreen(x, y, bounds) {
|
|
643
|
+
if (x < bounds.left || x >= bounds.left + bounds.width || y < bounds.top || y >= bounds.top + bounds.height) {
|
|
644
|
+
throw new Error(`坐标 (${x}, ${y}) 在屏幕外(有效范围 ${bounds.left},${bounds.top} 到 ${bounds.left + bounds.width - 1},${bounds.top + bounds.height - 1})。先 screenshot 看清界面再点`);
|
|
645
|
+
}
|
|
646
|
+
return { x, y };
|
|
647
|
+
}
|
|
648
|
+
/**
|
|
649
|
+
* 解析一个动作目标成屏幕物理坐标(纯函数,自检直接调)。
|
|
650
|
+
*
|
|
651
|
+
* - coordinate:图上的整数像素(按 scale 换算);若绑定了窗口,则相对该窗口左上角。
|
|
652
|
+
* - element:走该窗口**最近一次观察**(frame)里的 index;**元素不存在一律 fail-closed**
|
|
653
|
+
* (抛 ELEMENT_UNAVAILABLE,绝不退回坐标);元素在屏外/无矩形同样报错。
|
|
654
|
+
*
|
|
655
|
+
* @param target - 元素或坐标目标。缺省时用 legacy 的扁平 x/y(兼容旧参数形状)。
|
|
656
|
+
* @param frame - 该窗口最近一次观察(element 目标必须有)。
|
|
657
|
+
* @param bounds - 虚拟屏幕矩形。
|
|
658
|
+
* @param scale - 最近一张截图的缩放系数。
|
|
659
|
+
* @param windowBounds - 绑定窗口的矩形(给了则坐标按窗口左上角解析)。
|
|
660
|
+
*/
|
|
661
|
+
export function resolveTarget(target, legacy, frame, bounds, scale, windowBounds) {
|
|
662
|
+
const actual = target ?? (legacy?.x !== undefined && legacy?.y !== undefined ? { type: 'coordinate', x: legacy.x, y: legacy.y } : undefined);
|
|
663
|
+
if (actual === undefined) {
|
|
664
|
+
throw new Error('这个动作要带 target({type:\'element\', index} 或 {type:\'coordinate\', x, y})');
|
|
665
|
+
}
|
|
666
|
+
if (actual.type === 'coordinate') {
|
|
667
|
+
const x = assertInt(actual.x, 'coordinate.x');
|
|
668
|
+
const y = assertInt(actual.y, 'coordinate.y');
|
|
669
|
+
const ox = windowBounds?.x ?? 0;
|
|
670
|
+
const oy = windowBounds?.y ?? 0;
|
|
671
|
+
const point = clampToScreen(Math.round(x / scale) + ox, Math.round(y / scale) + oy, bounds);
|
|
672
|
+
const label = windowBounds === undefined || windowBounds === null
|
|
673
|
+
? `坐标 (${point.x}, ${point.y})`
|
|
674
|
+
: `窗口内坐标 (${x}, ${y}) → 屏幕 (${point.x}, ${point.y})`;
|
|
675
|
+
return { x: point.x, y: point.y, label };
|
|
676
|
+
}
|
|
677
|
+
// element
|
|
678
|
+
const index = assertInt(actual.index, 'element.index');
|
|
679
|
+
if (frame === undefined) {
|
|
680
|
+
throw new Error(`元素目标要有该窗口的观察结果,先 get_app_state 观察一次再点(当前没有任何观察)`);
|
|
681
|
+
}
|
|
682
|
+
if (actual.state_id !== undefined && actual.state_id !== frame.state_id) {
|
|
683
|
+
throw new Error(`ELEMENT_UNAVAILABLE:state_id「${actual.state_id}」不是该窗口最近的观察「${frame.state_id}」,索引已失效,请重新 get_app_state`);
|
|
684
|
+
}
|
|
685
|
+
if (index < 0 || index >= frame.elements.length) {
|
|
686
|
+
throw new Error(`ELEMENT_UNAVAILABLE:元素 #${index} 不存在(该窗口最近一次观察有 ${frame.elements.length} 个控件),请重新 get_app_state`);
|
|
687
|
+
}
|
|
688
|
+
const element = frame.elements[index];
|
|
689
|
+
if (element.offscreen || element.bounds === null || element.bounds.width <= 0 || element.bounds.height <= 0) {
|
|
690
|
+
throw new Error(`ELEMENT_UNAVAILABLE:元素 #${index}(${element.controlType}「${element.title}」)在屏外或没有矩形,无法定位`);
|
|
691
|
+
}
|
|
692
|
+
const cx = element.bounds.x + Math.floor(element.bounds.width / 2);
|
|
693
|
+
const cy = element.bounds.y + Math.floor(element.bounds.height / 2);
|
|
694
|
+
clampToScreen(cx, cy, bounds);
|
|
695
|
+
return { x: cx, y: cy, label: `元素 #${index}(${element.controlType}「${element.title}」)`, element };
|
|
696
|
+
}
|
|
697
|
+
/** 把一次的 JSON 控件对象规整成 DesktopElement。 */
|
|
698
|
+
function normalizeElement(raw, fallbackIndex) {
|
|
699
|
+
const item = (raw ?? {});
|
|
700
|
+
const boundsRaw = item.bounds;
|
|
701
|
+
const bounds = boundsRaw === null || boundsRaw === undefined
|
|
702
|
+
? null
|
|
703
|
+
: {
|
|
704
|
+
x: Number(boundsRaw.x ?? 0),
|
|
705
|
+
y: Number(boundsRaw.y ?? 0),
|
|
706
|
+
width: Number(boundsRaw.width ?? 0),
|
|
707
|
+
height: Number(boundsRaw.height ?? 0),
|
|
708
|
+
};
|
|
709
|
+
const patternsRaw = item.patterns;
|
|
710
|
+
const patterns = Array.isArray(patternsRaw)
|
|
711
|
+
? patternsRaw.map((p) => String(p))
|
|
712
|
+
: patternsRaw === undefined || patternsRaw === null
|
|
713
|
+
? []
|
|
714
|
+
: [String(patternsRaw)];
|
|
715
|
+
return {
|
|
716
|
+
index: Number.isInteger(item.index) ? item.index : fallbackIndex,
|
|
717
|
+
controlType: String(item.controlType ?? ''),
|
|
718
|
+
title: String(item.title ?? ''),
|
|
719
|
+
automaid: String(item.automaid ?? ''),
|
|
720
|
+
value: String(item.value ?? ''),
|
|
721
|
+
bounds,
|
|
722
|
+
patterns,
|
|
723
|
+
enabled: item.enabled !== false,
|
|
724
|
+
offscreen: item.offscreen === true,
|
|
725
|
+
};
|
|
726
|
+
}
|
|
727
|
+
/** 观察编号计数器(进程内单调递增)。 */
|
|
728
|
+
let stateCounter = 0;
|
|
729
|
+
/** 观察结果缓存:窗口键 → 最近一次观察。 */
|
|
730
|
+
const framesByWindow = new Map();
|
|
731
|
+
/** 最近一次观察(不指定窗口时的元素作用域与差分基线)。 */
|
|
732
|
+
let latestFrame;
|
|
733
|
+
/** 自检用:清空观察缓存(生产路径不调用)。 */
|
|
734
|
+
export function resetFrames() {
|
|
735
|
+
framesByWindow.clear();
|
|
736
|
+
latestFrame = undefined;
|
|
737
|
+
stateCounter = 0;
|
|
738
|
+
}
|
|
739
|
+
/** 该窗口最近一次观察。 */
|
|
740
|
+
export function frameForWindow(windowKey) {
|
|
741
|
+
if (windowKey === undefined)
|
|
742
|
+
return latestFrame;
|
|
743
|
+
return framesByWindow.get(String(windowKey));
|
|
744
|
+
}
|
|
745
|
+
/** 解析观察脚本的 JSON,产出 DesktopFrame 并写入缓存。 */
|
|
746
|
+
export function parseFrame(raw) {
|
|
747
|
+
const trimmed = raw.trim();
|
|
748
|
+
let data;
|
|
749
|
+
try {
|
|
750
|
+
data = JSON.parse(trimmed);
|
|
751
|
+
}
|
|
752
|
+
catch {
|
|
753
|
+
throw new Error(`无障碍观察返回的不是 JSON:${trimmed.slice(0, 120)}`);
|
|
754
|
+
}
|
|
755
|
+
if (data.ok !== true) {
|
|
756
|
+
throw new Error(`无障碍观察失败:${String(data.error ?? '未知原因')}(这台机器可能不支持 UIA,或目标窗口已关闭)`);
|
|
757
|
+
}
|
|
758
|
+
const winRaw = (data.window ?? {});
|
|
759
|
+
const boundsRaw = winRaw.bounds;
|
|
760
|
+
const window = {
|
|
761
|
+
id: Number(winRaw.id ?? 0),
|
|
762
|
+
title: String(winRaw.title ?? ''),
|
|
763
|
+
bounds: boundsRaw === null || boundsRaw === undefined
|
|
764
|
+
? null
|
|
765
|
+
: {
|
|
766
|
+
x: Number(boundsRaw.x ?? 0),
|
|
767
|
+
y: Number(boundsRaw.y ?? 0),
|
|
768
|
+
width: Number(boundsRaw.width ?? 0),
|
|
769
|
+
height: Number(boundsRaw.height ?? 0),
|
|
770
|
+
},
|
|
771
|
+
};
|
|
772
|
+
const rawElements = data.elements;
|
|
773
|
+
const list = Array.isArray(rawElements) ? rawElements : rawElements === undefined || rawElements === null ? [] : [rawElements];
|
|
774
|
+
const elements = list.map((item, i) => normalizeElement(item, i));
|
|
775
|
+
stateCounter += 1;
|
|
776
|
+
const frame = {
|
|
777
|
+
state_id: `st-${stateCounter}`,
|
|
778
|
+
app: String(data.app ?? ''),
|
|
779
|
+
window,
|
|
780
|
+
elements,
|
|
781
|
+
text: String(data.text ?? ''),
|
|
782
|
+
snapshot_mode: 'full',
|
|
783
|
+
};
|
|
784
|
+
framesByWindow.set(String(window.id), frame);
|
|
785
|
+
latestFrame = frame;
|
|
786
|
+
return frame;
|
|
787
|
+
}
|
|
788
|
+
/** 帧差分:按 index 对齐(未变的索引仍有效),得出 added/removed/changed。 */
|
|
789
|
+
export function diffFrames(base, next) {
|
|
790
|
+
const removed = [];
|
|
791
|
+
const added = [];
|
|
792
|
+
const changed = [];
|
|
793
|
+
const max = Math.max(base.elements.length, next.elements.length);
|
|
794
|
+
for (let i = 0; i < max; i += 1) {
|
|
795
|
+
const a = base.elements[i];
|
|
796
|
+
const b = next.elements[i];
|
|
797
|
+
if (a === undefined && b !== undefined)
|
|
798
|
+
added.push(i);
|
|
799
|
+
else if (a !== undefined && b === undefined)
|
|
800
|
+
removed.push(i);
|
|
801
|
+
else if (a !== undefined && b !== undefined) {
|
|
802
|
+
const same = a.controlType === b.controlType &&
|
|
803
|
+
a.title === b.title &&
|
|
804
|
+
a.value === b.value &&
|
|
805
|
+
a.automaid === b.automaid &&
|
|
806
|
+
JSON.stringify(a.bounds) === JSON.stringify(b.bounds);
|
|
807
|
+
if (!same)
|
|
808
|
+
changed.push(i);
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
return { removed, added, changed };
|
|
812
|
+
}
|
|
813
|
+
/** 把差分结论写成给人/模型看的文本。 */
|
|
814
|
+
export function diffText(base, next, diff) {
|
|
815
|
+
const head = `观察 ${next.state_id}(相对 ${base.state_id} 的变化;未列出的索引按上次的树不变):` +
|
|
816
|
+
`新增 [${diff.added.join(', ')}],移除 [${diff.removed.join(', ')}],变化 [${diff.changed.join(', ')}]`;
|
|
817
|
+
const details = [];
|
|
818
|
+
for (const i of diff.changed) {
|
|
819
|
+
const b = next.elements[i];
|
|
820
|
+
if (b === undefined)
|
|
821
|
+
continue;
|
|
822
|
+
details.push(` [${i}] ${b.controlType} "${b.title}" = "${b.value}"`);
|
|
823
|
+
}
|
|
824
|
+
return details.length === 0 ? head : `${head}\n${details.join('\n')}`;
|
|
825
|
+
}
|
|
826
|
+
/** 默认 sleep(注入点之外的真实实现)。 */
|
|
827
|
+
function defaultSleep(ms) {
|
|
828
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
829
|
+
}
|
|
830
|
+
/**
|
|
831
|
+
* 观察稳定性判定:连续两次指纹相同即算稳定;超时上限到了就如实返回 stable:false。
|
|
832
|
+
*
|
|
833
|
+
* 为什么要 settle:动作发出去到界面真正变化之间有延迟,直接观察容易拍到中间态。
|
|
834
|
+
* 禁止模型自己轮询/setTimeout,所以等待发生在这里,且必须有上限。
|
|
835
|
+
*/
|
|
836
|
+
export async function settleObserve(snapshot, fingerprint, options = {}) {
|
|
837
|
+
const intervalMs = Math.max(0, Math.round(options.intervalMs ?? 120));
|
|
838
|
+
const timeoutMs = Math.max(intervalMs, Math.round(options.timeoutMs ?? 1500));
|
|
839
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
840
|
+
const now = options.now ?? Date.now;
|
|
841
|
+
const started = now();
|
|
842
|
+
let previous = fingerprint(await snapshot());
|
|
843
|
+
let attempts = 1;
|
|
844
|
+
while (now() - started < timeoutMs) {
|
|
845
|
+
await sleep(intervalMs);
|
|
846
|
+
const current = fingerprint(await snapshot());
|
|
847
|
+
attempts += 1;
|
|
848
|
+
if (current === previous) {
|
|
849
|
+
return { stable: true, attempts, waitedMs: now() - started, fingerprint: current };
|
|
850
|
+
}
|
|
851
|
+
previous = current;
|
|
852
|
+
if (now() - started >= timeoutMs)
|
|
853
|
+
break;
|
|
854
|
+
}
|
|
855
|
+
return { stable: false, attempts, waitedMs: now() - started, fingerprint: previous };
|
|
856
|
+
}
|
|
857
|
+
// ── 进程内动作 ────────────────────────────────────────────────────────────
|
|
858
|
+
/** 屏幕边界 + 前台窗口,一次进程问清楚(越界判断与白名单判断都要用)。 */
|
|
859
|
+
async function probeScreen(signal) {
|
|
860
|
+
const out = await runPowerShell(`
|
|
861
|
+
Add-Type -AssemblyName System.Windows.Forms
|
|
862
|
+
Add-Type -TypeDefinition '${DPI_A}'
|
|
863
|
+
Add-Type -TypeDefinition '${WINDOW_TYPE}'
|
|
864
|
+
[DscDpi]::SetProcessDPIAware() | Out-Null
|
|
865
|
+
$b = [System.Windows.Forms.SystemInformation]::VirtualScreen
|
|
866
|
+
"$($b.Left),$($b.Top),$($b.Width),$($b.Height)|" + [DscWin]::Describe()
|
|
867
|
+
`, 12000, signal);
|
|
868
|
+
// 前台窗口那一段自己可能带竖线(进程|标题),所以只按第一个竖线切
|
|
869
|
+
const parts = out.trim().split('|');
|
|
870
|
+
const geom = parts.shift() ?? '';
|
|
871
|
+
const foreground = parts.join('|');
|
|
872
|
+
const [left, top, width, height] = geom.split(',').map((part) => Number.parseInt(part, 10));
|
|
873
|
+
if ([left, top, width, height].some((value) => !Number.isFinite(value))) {
|
|
874
|
+
throw new Error(`读屏幕边界失败,收到「${out.trim()}」`);
|
|
875
|
+
}
|
|
876
|
+
return { left: left, top: top, width: width, height: height, foreground };
|
|
877
|
+
}
|
|
878
|
+
/** 应用白名单:前台窗口的进程名或标题命中白名单才放行。 */
|
|
879
|
+
function assertAllowedApp(foreground, allowed) {
|
|
880
|
+
if (allowed.length === 0)
|
|
881
|
+
return;
|
|
882
|
+
const [processName = '', ...titleParts] = foreground.split('|');
|
|
883
|
+
const title = titleParts.join('|');
|
|
884
|
+
const hit = allowed.some((entry) => {
|
|
885
|
+
const needle = entry.trim().toLowerCase();
|
|
886
|
+
if (needle === '')
|
|
887
|
+
return false;
|
|
888
|
+
return processName.toLowerCase().includes(needle) || title.toLowerCase().includes(needle);
|
|
889
|
+
});
|
|
890
|
+
if (!hit) {
|
|
891
|
+
throw new Error(`当前前台窗口是「${title || processName}」,不在你设的应用白名单(${allowed.join('、')})里,这一步被拦下了`);
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
/** 观察目标窗口的无障碍树(新动作 get_app_state 的实现)。 */
|
|
895
|
+
export async function observeAppState(options = {}, signal) {
|
|
896
|
+
const sel = options.window;
|
|
897
|
+
const isHandle = typeof sel === 'number' || (sel !== undefined && /^\d+$/.test(String(sel).trim()));
|
|
898
|
+
const script = observeScript({
|
|
899
|
+
windowId: isHandle ? Number(sel) : 0,
|
|
900
|
+
windowText: isHandle || sel === undefined ? '' : String(sel),
|
|
901
|
+
maxElements: options.maxElements,
|
|
902
|
+
});
|
|
903
|
+
const raw = await runPowerShell(script, 25000, signal).catch((error) => {
|
|
904
|
+
throw new Error(`无障碍观察失败:${errText(error)}`);
|
|
905
|
+
});
|
|
906
|
+
// parseFrame 会把新帧写进缓存;基线要在写入之前快照下来(引用拷贝,parseFrame 只 set 不 mutate)。
|
|
907
|
+
const before = new Map(framesByWindow);
|
|
908
|
+
const beforeLatest = latestFrame;
|
|
909
|
+
const frame = parseFrame(raw);
|
|
910
|
+
const base = options.disableDiffing === true ? undefined : before.get(String(frame.window.id)) ?? beforeLatest;
|
|
911
|
+
let text;
|
|
912
|
+
if (base !== undefined && base.state_id !== frame.state_id && base.window.id === frame.window.id) {
|
|
913
|
+
const diff = diffFrames(base, frame);
|
|
914
|
+
frame.snapshot_mode = 'diff';
|
|
915
|
+
frame.base_state_id = base.state_id;
|
|
916
|
+
frame.diff = diff;
|
|
917
|
+
const quiet = diff.added.length === 0 && diff.removed.length === 0 && diff.changed.length === 0;
|
|
918
|
+
text =
|
|
919
|
+
`窗口「${frame.window.title}」(进程 ${frame.app},句柄 ${frame.window.id}),无障碍观察 ${frame.state_id},共 ${frame.elements.length} 个控件。\n` +
|
|
920
|
+
(quiet ? '与上次观察相比:**没有变化**(界面稳定)。' : diffText(base, frame, diff)) +
|
|
921
|
+
(frame.snapshot_mode === 'diff' ? `\n(增量模式;需要全树就再调一次 get_app_state 带 disable_diffing:true)` : '');
|
|
922
|
+
}
|
|
923
|
+
else {
|
|
924
|
+
text =
|
|
925
|
+
`窗口「${frame.window.title}」(进程 ${frame.app},句柄 ${frame.window.id}),无障碍观察 ${frame.state_id},共 ${frame.elements.length} 个控件。\n` +
|
|
926
|
+
`元素索引只对「这次观察」有效;界面一变就要重新观察。\n\n${frame.text}`;
|
|
927
|
+
}
|
|
928
|
+
return { text, frame };
|
|
929
|
+
}
|
|
930
|
+
/** 解析窗口选择器,得到句柄 + 标题 + 矩形。 */
|
|
931
|
+
async function probeWindow(selector, signal) {
|
|
932
|
+
const raw = await runPowerShell(probeWindowScript(selector), 12000, signal);
|
|
933
|
+
const data = JSON.parse(raw.trim());
|
|
934
|
+
if (data.ok !== true)
|
|
935
|
+
throw new Error(String(data.error ?? '窗口解析失败'));
|
|
936
|
+
const b = (data.bounds ?? {});
|
|
937
|
+
return {
|
|
938
|
+
id: Number(data.id ?? 0),
|
|
939
|
+
title: String(data.title ?? ''),
|
|
940
|
+
bounds: { x: Number(b.x ?? 0), y: Number(b.y ?? 0), width: Number(b.width ?? 0), height: Number(b.height ?? 0) },
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
/** 绑定窗口:解析 `args.window`,返回该窗口的观察帧、矩形与激活脚本片段。 */
|
|
944
|
+
async function bindWindow(args, signal) {
|
|
945
|
+
if (args.window === undefined)
|
|
946
|
+
return { frame: latestFrame, activateLine: '' };
|
|
947
|
+
const info = await probeWindow(args.window, signal);
|
|
948
|
+
const activateLine = `[DscWin]::SetForegroundWindow([IntPtr]${info.id}) | Out-Null; Start-Sleep -Milliseconds 80`;
|
|
949
|
+
return {
|
|
950
|
+
frame: frameForWindow(String(info.id)) ?? latestFrame,
|
|
951
|
+
windowBounds: info.bounds,
|
|
952
|
+
windowKey: String(info.id),
|
|
953
|
+
activateLine,
|
|
954
|
+
};
|
|
955
|
+
}
|
|
956
|
+
/** 执行点击(内部复用:set_value 回退也要用)。 */
|
|
957
|
+
async function clickAt(point, button, double, activateLine, pauseMs, signal) {
|
|
958
|
+
await runPowerShell(clickScript(point, button, double, activateLine, pauseMs), 15000, signal);
|
|
959
|
+
}
|
|
960
|
+
/** 执行文本输入(内部复用)。 */
|
|
961
|
+
async function typeText(text, signal) {
|
|
962
|
+
await runPowerShell(typeScript(text), 15000, signal);
|
|
963
|
+
}
|
|
964
|
+
/**
|
|
965
|
+
* 执行一个桌面动作。
|
|
966
|
+
* @param scale - 上一次截屏的缩放系数(点/滚动的坐标按它换算回物理像素)。
|
|
967
|
+
* @param allowedApps - 应用白名单(进程名或窗口标题的子串);空数组 = 不限。
|
|
968
|
+
*/
|
|
969
|
+
export async function runDesktopAction(action, args, options, signal) {
|
|
970
|
+
const canonical = canonicalAction(String(action));
|
|
971
|
+
if (canonical === null) {
|
|
972
|
+
throw new Error(`不认识的 action「${String(action)}」。要用的值:${DESKTOP_ACTIONS.join(' / ')}`);
|
|
973
|
+
}
|
|
974
|
+
const pauseMs = options.actionDelayMs;
|
|
975
|
+
switch (canonical) {
|
|
976
|
+
case 'screenshot': {
|
|
977
|
+
throw new Error('内部错误:screenshot 要走 takeScreenshot()');
|
|
978
|
+
}
|
|
979
|
+
case 'get_app_state': {
|
|
980
|
+
return await observeAppState({ window: args.window, disableDiffing: args.disableDiffing === true }, signal);
|
|
981
|
+
}
|
|
982
|
+
case 'click': {
|
|
983
|
+
const bounds = await probeScreen(signal);
|
|
984
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
985
|
+
const bind = await bindWindow(args, signal);
|
|
986
|
+
const point = resolveTarget(args.target, args, bind.frame, bounds, options.scale, bind.windowBounds);
|
|
987
|
+
const button = String(args.button ?? 'left');
|
|
988
|
+
if (BUTTON_EVENT[button] === undefined)
|
|
989
|
+
throw new Error(`不认识的鼠标键「${button}」`);
|
|
990
|
+
await clickAt(point, button, args.double === true, bind.activateLine, pauseMs, signal);
|
|
991
|
+
return { text: `已在 ${point.label} ${args.double === true ? '双击' : '单击'}${button}。再截一张确认点中了什么。` };
|
|
992
|
+
}
|
|
993
|
+
case 'drag': {
|
|
994
|
+
const steps = args.amount !== undefined ? Math.min(Math.max(Math.round(args.amount), 2), 200) : 24;
|
|
995
|
+
const bounds = await probeScreen(signal);
|
|
996
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
997
|
+
const bind = await bindWindow(args, signal);
|
|
998
|
+
const from = resolveTarget(args.from, undefined, bind.frame, bounds, options.scale, bind.windowBounds);
|
|
999
|
+
const to = resolveTarget(args.to, undefined, bind.frame, bounds, options.scale, bind.windowBounds);
|
|
1000
|
+
await runPowerShell(dragScript(from, to, steps, Math.max(8, Math.round(pauseMs / 4))), 20000, signal);
|
|
1001
|
+
return { text: `已从 ${from.label} 拖到 ${to.label}(分 ${steps} 步,鼠标已保证抬起)。` };
|
|
1002
|
+
}
|
|
1003
|
+
case 'type': {
|
|
1004
|
+
const text = String(args.text ?? '');
|
|
1005
|
+
if (text === '')
|
|
1006
|
+
throw new Error('type 要带 text');
|
|
1007
|
+
if (text.length > 4000)
|
|
1008
|
+
throw new Error(`一次最多输 4000 字,这次有 ${text.length} 字,拆开输`);
|
|
1009
|
+
const bounds = await probeScreen(signal);
|
|
1010
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
1011
|
+
const bind = await bindWindow(args, signal);
|
|
1012
|
+
if (bind.activateLine !== '')
|
|
1013
|
+
await runPowerShell(`Add-Type -TypeDefinition '${WINDOW_TYPE}'\n${bind.activateLine}\n'Done'`, 12000, signal);
|
|
1014
|
+
await typeText(text, signal);
|
|
1015
|
+
return { text: `已输入 ${text.length} 个字符(走剪贴板粘贴;原来的文本剪贴板已还原)。` };
|
|
1016
|
+
}
|
|
1017
|
+
case 'paste': {
|
|
1018
|
+
const bounds = await probeScreen(signal);
|
|
1019
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
1020
|
+
const bind = await bindWindow(args, signal);
|
|
1021
|
+
if (bind.activateLine !== '')
|
|
1022
|
+
await runPowerShell(`Add-Type -TypeDefinition '${WINDOW_TYPE}'\n${bind.activateLine}\n'Done'`, 12000, signal);
|
|
1023
|
+
const text = args.text === undefined ? undefined : String(args.text);
|
|
1024
|
+
await runPowerShell(pasteScript(text), 15000, signal);
|
|
1025
|
+
return {
|
|
1026
|
+
text: text === undefined
|
|
1027
|
+
? '已按 Ctrl+V 粘贴当前剪贴板内容(没有改动剪贴板)。'
|
|
1028
|
+
: `已把 ${text.length} 个字符经剪贴板粘贴进去,并还原了你原来的文本剪贴板。`,
|
|
1029
|
+
};
|
|
1030
|
+
}
|
|
1031
|
+
case 'set_value': {
|
|
1032
|
+
const value = String(args.value ?? '');
|
|
1033
|
+
const index = args.target?.type === 'element' ? args.target.index : undefined;
|
|
1034
|
+
if (index === undefined)
|
|
1035
|
+
throw new Error('set_value 要带 {type:\'element\', index} 目标');
|
|
1036
|
+
const bounds = await probeScreen(signal);
|
|
1037
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
1038
|
+
const bind = await bindWindow(args, signal);
|
|
1039
|
+
const frame = bind.frame;
|
|
1040
|
+
// 先确认元素还在(fail-closed)
|
|
1041
|
+
resolveTarget(args.target, undefined, frame, bounds, options.scale, bind.windowBounds);
|
|
1042
|
+
const winSel = args.window;
|
|
1043
|
+
const info = await elementAction('set_value', { window: winSel, frame, index, value, pattern: args.pattern }, signal);
|
|
1044
|
+
if (info.method === 'none') {
|
|
1045
|
+
// 落不到 ValuePattern:回退成「点一下元素再 type」
|
|
1046
|
+
const point = resolveTarget(args.target, undefined, frame, bounds, options.scale, bind.windowBounds);
|
|
1047
|
+
await clickAt(point, 'left', false, bind.activateLine, pauseMs, signal);
|
|
1048
|
+
await typeText(value, signal);
|
|
1049
|
+
return { text: `元素 #${index} 不支持直接设值,已回退为「点击后输入」${value.length} 个字符。` };
|
|
1050
|
+
}
|
|
1051
|
+
return { text: `已把元素 #${index} 的值设为「${value}」(ValuePattern)。` };
|
|
1052
|
+
}
|
|
1053
|
+
case 'select_text': {
|
|
1054
|
+
const index = args.target?.type === 'element' ? args.target.index : undefined;
|
|
1055
|
+
if (index === undefined)
|
|
1056
|
+
throw new Error('select_text 要带 {type:\'element\', index} 目标');
|
|
1057
|
+
const bounds = await probeScreen(signal);
|
|
1058
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
1059
|
+
const bind = await bindWindow(args, signal);
|
|
1060
|
+
resolveTarget(args.target, undefined, bind.frame, bounds, options.scale, bind.windowBounds);
|
|
1061
|
+
const info = await elementAction('select_text', { window: args.window, frame: bind.frame, index }, signal);
|
|
1062
|
+
if (info.method === 'none')
|
|
1063
|
+
throw new Error(`元素 #${index} 不支持选择文本(没有 TextPattern / SelectionItemPattern)`);
|
|
1064
|
+
return { text: `已选中元素 #${index} 的文本(${info.method})。` };
|
|
1065
|
+
}
|
|
1066
|
+
case 'perform_action': {
|
|
1067
|
+
const index = args.target?.type === 'element' ? args.target.index : undefined;
|
|
1068
|
+
if (index === undefined)
|
|
1069
|
+
throw new Error('perform_action 要带 {type:\'element\', index} 目标');
|
|
1070
|
+
if (args.pattern === undefined || String(args.pattern).trim() === '')
|
|
1071
|
+
throw new Error('perform_action 要带 pattern,例如 invoke / toggle / expand / collapse / select');
|
|
1072
|
+
const bounds = await probeScreen(signal);
|
|
1073
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
1074
|
+
const bind = await bindWindow(args, signal);
|
|
1075
|
+
resolveTarget(args.target, undefined, bind.frame, bounds, options.scale, bind.windowBounds);
|
|
1076
|
+
const info = await elementAction('perform_action', { window: args.window, frame: bind.frame, index, pattern: args.pattern }, signal);
|
|
1077
|
+
if (info.method === 'none')
|
|
1078
|
+
throw new Error(`元素 #${index} 不支持 pattern「${String(args.pattern)}」对应的 UIA 模式`);
|
|
1079
|
+
return { text: `已对元素 #${index} 执行 ${String(args.pattern)}(${info.method})。` };
|
|
1080
|
+
}
|
|
1081
|
+
case 'key': {
|
|
1082
|
+
const keys = toSendKeys(String(args.key ?? ''));
|
|
1083
|
+
const bounds = await probeScreen(signal);
|
|
1084
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
1085
|
+
const bind = await bindWindow(args, signal);
|
|
1086
|
+
if (bind.activateLine !== '')
|
|
1087
|
+
await runPowerShell(`Add-Type -TypeDefinition '${WINDOW_TYPE}'\n${bind.activateLine}\n'Done'`, 12000, signal);
|
|
1088
|
+
const repeat = args.repeat === undefined ? 1 : Math.min(Math.max(Math.round(args.repeat), 1), 50);
|
|
1089
|
+
const hold = args.holdSeconds === undefined ? 0 : Math.min(Math.max(Number(args.holdSeconds), 0), 10);
|
|
1090
|
+
await runPowerShell(keyScript(keys, repeat, hold), 15000, signal);
|
|
1091
|
+
return { text: `已按键 ${String(args.key)}${repeat > 1 ? ` × ${repeat}` : ''}。` };
|
|
1092
|
+
}
|
|
1093
|
+
case 'scroll': {
|
|
1094
|
+
const amount = assertInt(args.amount ?? 3, 'amount');
|
|
1095
|
+
if (Math.abs(amount) > 30)
|
|
1096
|
+
throw new Error('一次最多滚 30 档');
|
|
1097
|
+
const bounds = await probeScreen(signal);
|
|
1098
|
+
assertAllowedApp(bounds.foreground, options.allowedApps);
|
|
1099
|
+
const bind = await bindWindow(args, signal);
|
|
1100
|
+
let move = '';
|
|
1101
|
+
if (args.target !== undefined || (args.x !== undefined && args.y !== undefined)) {
|
|
1102
|
+
const point = resolveTarget(args.target, args, bind.frame, bounds, options.scale, bind.windowBounds);
|
|
1103
|
+
move = `[DscMouse]::SetCursorPos(${point.x}, ${point.y}) | Out-Null`;
|
|
1104
|
+
}
|
|
1105
|
+
await runPowerShell(scrollScript(amount, move, pauseMs), 15000, signal);
|
|
1106
|
+
return { text: `已滚动 ${amount} 档(正数向上,负数向下)。` };
|
|
1107
|
+
}
|
|
1108
|
+
case 'cursor': {
|
|
1109
|
+
const out = await runPowerShell(`
|
|
1110
|
+
Add-Type -TypeDefinition '${MOUSE_TYPE}'
|
|
1111
|
+
$p = New-Object DscMouse+POINT
|
|
1112
|
+
[DscMouse]::GetCursorPos([ref]$p) | Out-Null
|
|
1113
|
+
"$($p.X),$($p.Y)"
|
|
1114
|
+
`, 12000, signal);
|
|
1115
|
+
const [x = '?', y = '?'] = out.trim().split(',');
|
|
1116
|
+
return { text: `鼠标当前位置:屏幕 (${x}, ${y})。` };
|
|
1117
|
+
}
|
|
1118
|
+
case 'list_windows': {
|
|
1119
|
+
const raw = await runPowerShell(listWindowsScript(), 15000, signal);
|
|
1120
|
+
const data = JSON.parse(raw.trim());
|
|
1121
|
+
if (data.ok !== true)
|
|
1122
|
+
throw new Error(`列窗口失败:${String(data.error ?? '未知原因')}`);
|
|
1123
|
+
const rawList = data.windows;
|
|
1124
|
+
const list = Array.isArray(rawList) ? rawList : rawList === undefined || rawList === null ? [] : [rawList];
|
|
1125
|
+
if (list.length === 0)
|
|
1126
|
+
return { text: '(没找到带标题的窗口)' };
|
|
1127
|
+
const lines = list.map((item) => {
|
|
1128
|
+
const w = item;
|
|
1129
|
+
const b = (w.bounds ?? {});
|
|
1130
|
+
const flags = [w.focused === true ? '前台' : '', w.main === true ? '主窗口' : '', w.onscreen === false ? '最小化/屏外' : ''].filter((f) => f !== '');
|
|
1131
|
+
return ` id=${String(w.window_id)} [${String(w.app)}] ${w.title === '' ? '(无标题)' : String(w.title)} @ ${b.x},${b.y} ${b.width}×${b.height}${flags.length > 0 ? `(${flags.join('、')})` : ''}`;
|
|
1132
|
+
});
|
|
1133
|
+
return { text: `窗口清单(id 用作用户 window 参数):\n${lines.join('\n')}` };
|
|
1134
|
+
}
|
|
1135
|
+
case 'list_apps': {
|
|
1136
|
+
const raw = await runPowerShell(listAppsScript(), 15000, signal);
|
|
1137
|
+
const data = JSON.parse(raw.trim());
|
|
1138
|
+
if (data.ok !== true)
|
|
1139
|
+
throw new Error(`列应用失败:${String(data.error ?? '未知原因')}`);
|
|
1140
|
+
const rawList = data.apps;
|
|
1141
|
+
const list = Array.isArray(rawList) ? rawList : rawList === undefined || rawList === null ? [] : [rawList];
|
|
1142
|
+
const lines = list.map((item) => {
|
|
1143
|
+
const a = item;
|
|
1144
|
+
return ` ${String(a.app)}(pid ${String(a.pid)})${a.title === '' ? '' : ` — ${String(a.title)}`}${a.foreground === true ? ' 【前台】' : ''}`;
|
|
1145
|
+
});
|
|
1146
|
+
return { text: `有主窗口的应用(进程名 + 主窗口 + 是否前台):\n${lines.join('\n')}` };
|
|
1147
|
+
}
|
|
1148
|
+
case 'screens': {
|
|
1149
|
+
const raw = await runPowerShell(screensScript(), 12000, signal);
|
|
1150
|
+
const data = JSON.parse(raw.trim());
|
|
1151
|
+
if (data.ok !== true)
|
|
1152
|
+
throw new Error(`枚举显示器失败:${String(data.error ?? '未知原因')}`);
|
|
1153
|
+
const rawList = data.screens;
|
|
1154
|
+
const list = Array.isArray(rawList) ? rawList : rawList === undefined || rawList === null ? [] : [rawList];
|
|
1155
|
+
const lines = list.map((item) => {
|
|
1156
|
+
const s = item;
|
|
1157
|
+
const b = (s.bounds ?? {});
|
|
1158
|
+
return ` ${String(s.device)}${s.primary === true ? '(主)' : ''} @ ${b.x},${b.y} ${b.width}×${b.height}`;
|
|
1159
|
+
});
|
|
1160
|
+
return { text: `显示器(虚拟屏坐标,可含负坐标):\n${lines.join('\n')}` };
|
|
1161
|
+
}
|
|
1162
|
+
case 'stop': {
|
|
1163
|
+
throw new Error('内部错误:stop 由插件处理(要联动取消信号与动作面)');
|
|
1164
|
+
}
|
|
1165
|
+
default: {
|
|
1166
|
+
throw new Error(`不认识的 action「${canonical}」。要用的值:${DESKTOP_ACTIONS.join(' / ')}`);
|
|
1167
|
+
}
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
/** 跑一次元素动作脚本(set_value / select_text / perform_action),返回 {method}。 */
|
|
1171
|
+
async function elementAction(kind, input, signal) {
|
|
1172
|
+
const sel = input.window;
|
|
1173
|
+
const isHandle = typeof sel === 'number' || (sel !== undefined && /^\d+$/.test(String(sel).trim()));
|
|
1174
|
+
const script = elementActionScript(kind, {
|
|
1175
|
+
windowId: isHandle ? Number(sel) : input.frame?.window.id ?? 0,
|
|
1176
|
+
windowText: isHandle || sel === undefined ? '' : String(sel),
|
|
1177
|
+
index: input.index,
|
|
1178
|
+
value: input.value,
|
|
1179
|
+
pattern: input.pattern,
|
|
1180
|
+
});
|
|
1181
|
+
const raw = await runPowerShell(script, 20000, signal);
|
|
1182
|
+
const data = JSON.parse(raw.trim());
|
|
1183
|
+
if (data.ok !== true)
|
|
1184
|
+
throw new Error(String(data.error ?? '元素动作失败'));
|
|
1185
|
+
return { method: String(data.method ?? 'none') };
|
|
1186
|
+
}
|
|
1187
|
+
/**
|
|
1188
|
+
* 抓一张全屏图(或裁切块)。
|
|
1189
|
+
* @returns 图像 data URL + 缩放系数(点坐标要按它换算)。
|
|
1190
|
+
*/
|
|
1191
|
+
export async function takeScreenshot(options, signal) {
|
|
1192
|
+
const clip = options.clip;
|
|
1193
|
+
if (clip !== undefined) {
|
|
1194
|
+
for (const [key, value] of Object.entries(clip)) {
|
|
1195
|
+
if (!Number.isFinite(Number(value)))
|
|
1196
|
+
throw new Error(`截图的 clip.${key} 要是数字,收到 ${String(value)}`);
|
|
1197
|
+
}
|
|
1198
|
+
if (clip.width <= 0 || clip.height <= 0)
|
|
1199
|
+
throw new Error('截图的 clip.width / clip.height 要大于 0');
|
|
1200
|
+
}
|
|
1201
|
+
const out = await runPowerShell(screenshotScript(Math.min(Math.max(Math.round(options.maxEdge), 640), 3840), options.format === 'png' ? 'png' : 'jpeg', Math.min(Math.max(Math.round(options.quality), 30), 100), clip), 30000, signal).catch((error) => {
|
|
1202
|
+
throw new Error(`截屏失败:${errText(error)}`);
|
|
1203
|
+
});
|
|
1204
|
+
const trimmed = out.trim();
|
|
1205
|
+
const at = trimmed.indexOf('|');
|
|
1206
|
+
if (at < 0)
|
|
1207
|
+
throw new Error(`截屏返回看不懂的格式:${trimmed.slice(0, 60)}`);
|
|
1208
|
+
const head = trimmed.slice(0, at);
|
|
1209
|
+
const base64 = trimmed.slice(at + 1);
|
|
1210
|
+
const nums = head.split(',').map((part) => Number.parseInt(part, 10));
|
|
1211
|
+
const [imageWidth = 0, imageHeight = 0, clipWidth = 0, clipHeight = 0, originX = 0, originY = 0, vLeft = 0, vTop = 0, vWidth = 0, vHeight = 0] = nums;
|
|
1212
|
+
if (base64 === '')
|
|
1213
|
+
throw new Error('截屏是空的');
|
|
1214
|
+
const sourceWidth = clipWidth || imageWidth || vWidth;
|
|
1215
|
+
const scale = sourceWidth > 0 ? sourceWidth / (imageWidth || sourceWidth) : 1;
|
|
1216
|
+
const mime = options.format === 'png' ? 'image/png' : 'image/jpeg';
|
|
1217
|
+
const regionText = clip === undefined
|
|
1218
|
+
? `整个桌面`
|
|
1219
|
+
: `裁切区域(相对虚拟屏左上角 ${vLeft},${vTop})起点 (${originX}, ${originY}) 大小 ${clipWidth}×${clipHeight}`;
|
|
1220
|
+
return {
|
|
1221
|
+
text: `这是${regionText}的截图,图片尺寸 ${String(imageWidth)}×${String(imageHeight)};` +
|
|
1222
|
+
`虚拟屏分辨率 ${String(vWidth)}×${String(vHeight)},起点 (${String(vLeft)}, ${String(vTop)})。` +
|
|
1223
|
+
`要点的坐标用这张图上的像素值(坐标相对裁切区域左上角),Pneuma 会换算(当前换算系数 ${scale.toFixed(2)})。` +
|
|
1224
|
+
`鼠标现在指的位置可用 cursor 查。`,
|
|
1225
|
+
images: [`data:${mime};base64,${base64}`],
|
|
1226
|
+
scale,
|
|
1227
|
+
};
|
|
1228
|
+
}
|
|
1229
|
+
/** 前台窗口是谁(白名单提示与自查用)。 */
|
|
1230
|
+
export async function foregroundWindow(signal) {
|
|
1231
|
+
const bounds = await probeScreen(signal);
|
|
1232
|
+
return bounds.foreground;
|
|
1233
|
+
}
|
|
1234
|
+
/** 枚举显示器(screens 动作的独立入口,自检/诊断用)。 */
|
|
1235
|
+
export async function listScreens(signal) {
|
|
1236
|
+
const raw = await runPowerShell(screensScript(), 12000, signal);
|
|
1237
|
+
const data = JSON.parse(raw.trim());
|
|
1238
|
+
if (data.ok !== true)
|
|
1239
|
+
throw new Error(String(data.error ?? '枚举显示器失败'));
|
|
1240
|
+
const rawList = data.screens;
|
|
1241
|
+
const list = Array.isArray(rawList) ? rawList : rawList === undefined || rawList === null ? [] : [rawList];
|
|
1242
|
+
return list.map((item) => {
|
|
1243
|
+
const s = item;
|
|
1244
|
+
const b = (s.bounds ?? {});
|
|
1245
|
+
return {
|
|
1246
|
+
device: String(s.device ?? ''),
|
|
1247
|
+
primary: s.primary === true,
|
|
1248
|
+
bounds: { x: Number(b.x ?? 0), y: Number(b.y ?? 0), width: Number(b.width ?? 0), height: Number(b.height ?? 0) },
|
|
1249
|
+
};
|
|
1250
|
+
});
|
|
1251
|
+
}
|
|
1252
|
+
// ── 危险分级与独立审计 ────────────────────────────────────────────────────
|
|
1253
|
+
/** 「系统级」动作的命中形状:任务管理器、注册表、命令解释器、系统配置工具这类。 */
|
|
1254
|
+
const SYSTEM_HINTS = /(taskmgr|任务管理器|regedit|注册表|reg\.exe|reg\.exe|taskkill|cmd\.exe|powershell|pwsh|wscript|cscript|mshta|rundll32|control\.exe|services\.msc|gpedit|secpol|diskmgmt|bcdedit|diskpart|net\.exe|net\s+user|shutdown|format\.com|lusrmgr|userpasswords)/i;
|
|
1255
|
+
/**
|
|
1256
|
+
* 危险分级:
|
|
1257
|
+
* - `read`:只读观察(截图/查鼠标/列窗口/列应用/列显示器/get_app_state),免审批那一档;
|
|
1258
|
+
* - `input`:普通输入与点击(要审批);
|
|
1259
|
+
* - `system`:系统级(急停 stop,或目标/文本命中系统工具形状),最严。
|
|
1260
|
+
*/
|
|
1261
|
+
export function gradeDesktopAction(action, args = {}) {
|
|
1262
|
+
if (action === 'stop')
|
|
1263
|
+
return 'system';
|
|
1264
|
+
if (READ_ACTIONS.includes(action))
|
|
1265
|
+
return 'read';
|
|
1266
|
+
let blob = '';
|
|
1267
|
+
try {
|
|
1268
|
+
blob = JSON.stringify(args ?? {});
|
|
1269
|
+
}
|
|
1270
|
+
catch {
|
|
1271
|
+
blob = String(args);
|
|
1272
|
+
}
|
|
1273
|
+
if (SYSTEM_HINTS.test(blob))
|
|
1274
|
+
return 'system';
|
|
1275
|
+
return 'input';
|
|
1276
|
+
}
|
|
1277
|
+
/** 电脑操作审计文件:`<home>/audit/computer-use.jsonl`(home 来自 ctx.settings.about().home,不硬编码)。 */
|
|
1278
|
+
export function computerAuditPath(home) {
|
|
1279
|
+
return join(home, 'audit', 'computer-use.jsonl');
|
|
1280
|
+
}
|
|
1281
|
+
/** 把一条审计记录压成一行 JSON(含换行;自检直接断言这一行的形状)。 */
|
|
1282
|
+
export function auditLine(record) {
|
|
1283
|
+
return `${JSON.stringify(record)}\n`;
|
|
1284
|
+
}
|
|
1285
|
+
/** 追加一行审计(写失败吞掉:审计是辅助面,不该卡住动作)。 */
|
|
1286
|
+
export function appendComputerAudit(home, record) {
|
|
1287
|
+
try {
|
|
1288
|
+
const path = computerAuditPath(home);
|
|
1289
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
1290
|
+
appendFileSync(path, auditLine(record), 'utf8');
|
|
1291
|
+
}
|
|
1292
|
+
catch {
|
|
1293
|
+
// 磁盘问题不影响动作本身
|
|
1294
|
+
}
|
|
1295
|
+
}
|
|
1296
|
+
/**
|
|
1297
|
+
* 沙箱接入结论(P2-8,如实记录):
|
|
1298
|
+
*
|
|
1299
|
+
* 现有的沙箱执行缝是 `src/core/tools/command-runner.ts` 的 `planCommand`,
|
|
1300
|
+
* 它是为「命令执行工具」换执行体设计的。电脑操作的 PowerShell **不能**走这条缝,原因是
|
|
1301
|
+
* 结构性的、不是偷懒:
|
|
1302
|
+
* 1. 隔离后端(docker 容器 / Windows 受限令牌)拿不到**交互式用户桌面**——
|
|
1303
|
+
* 容器里没有 user32 的桌面对象,受限令牌换了会话也就没有输入注入能力,
|
|
1304
|
+
* 真接上去的结果是电脑操作直接不可用;
|
|
1305
|
+
* 2. `command-runner` 的 base 计划是「一条 shell 命令」,而电脑操作是一串
|
|
1306
|
+
* P/Invoke 脚本,语义对不上(容器后端会把 `powershell -Command <脚本>` 再套一层 shell);
|
|
1307
|
+
* 3. 沙箱守卫链按 `input.command` 判 exec 类,`computer` 工具没有 command 参数,
|
|
1308
|
+
* 策略层天然看不到它——它不是「漏判」,而是这类动作本就不该由命令沙箱兜。
|
|
1309
|
+
* 因此保持现状:电脑操作由审批卡 + 应用白名单 + 危险分级 + 独立审计兜底。
|
|
1310
|
+
*/
|
|
1311
|
+
export const SANDBOX_NOTE = '桌面控制的 PowerShell 不走命令沙箱执行缝(隔离后端拿不到交互式桌面);由审批 + 白名单 + 危险分级 + 独立审计兜底。';
|