@moretea-labs/forge 1.6.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/CHANGELOG.md +39 -0
- package/CLAUDE.md +4 -0
- package/README.md +2 -2
- package/README.zh-CN.md +1 -1
- package/SKILL.md +2 -2
- package/THIRD_PARTY_NOTICES.md +1 -1
- package/agents/openai.yaml +1 -1
- package/assets/hooks/hook-input.sh +11 -8
- package/assets/hooks/lib/session-state.sh +10 -7
- package/assets/hooks/lib/workflow-state.sh +13 -177
- package/assets/hooks/post-tool-observer.sh +8 -2
- package/assets/hooks/prompt-guard.sh +5 -81
- package/assets/hooks/session-start-context.sh +56 -1
- package/assets/partials/05-workflow.partial.md +1 -1
- package/assets/partials-agents/02-operating-mode.partial.md +2 -0
- package/assets/partials-agents/03-orchestration.partial.md +1 -1
- package/assets/plugin-registry.v1.json +3 -3
- package/assets/reference-configs/document-generation.md +1 -1
- package/assets/reference-configs/harness-overview.md +1 -1
- package/assets/reference-configs/sprint-contracts.md +3 -3
- package/assets/skill-version.json +2 -2
- package/assets/templates/contract.template.md +2 -7
- package/assets/templates/helpers/capture-plan.sh +2 -2
- package/assets/templates/helpers/check-architecture-sync.sh +17 -0
- package/assets/templates/helpers/check-task-workflow.sh +4 -4
- package/assets/templates/helpers/contract-worktree.sh +1 -21
- package/assets/templates/helpers/ensure-task-workflow.sh +11 -36
- package/assets/templates/helpers/inspect-project-state.ts +8 -6
- package/assets/templates/helpers/migrate-project-template.sh +22 -45
- package/assets/templates/helpers/migrate-workflow-docs.ts +9 -7
- package/assets/templates/helpers/new-plan.sh +1 -1
- package/assets/templates/helpers/plan-to-todo.sh +12 -37
- package/assets/templates/helpers/ship-worktrees.sh +2 -29
- package/assets/templates/helpers/verify-contract.sh +1 -111
- package/assets/templates/helpers/verify-sprint.sh +3 -180
- package/assets/templates/helpers/workflow-contract.ts +5 -1
- package/assets/templates/plan.template.md +1 -1
- package/assets/templates/review.template.md +9 -34
- package/assets/workflow-contract.v1.json +10 -19
- package/docs/README.md +10 -10
- package/docs/architecture/CURRENT.md +178 -0
- package/docs/architecture/EVOLUTION.md +42 -0
- package/docs/architecture/current/README.md +4 -156
- package/docs/architecture/decisions/20260803-release-identity-binding-and-exit-policy.md +2 -2
- package/docs/architecture/decisions/20260823-immediate-controller-relay.md +53 -0
- package/docs/architecture/decisions/20260826-browser-runtime-v2.md +73 -0
- package/docs/architecture/history/README.md +7 -0
- package/docs/architecture/{chatgpt-handoff-facade.md → history/chatgpt-handoff-facade.md} +4 -0
- package/docs/architecture/{global-hook-runtime.md → history/global-hook-runtime.md} +7 -3
- package/docs/architecture/{ios-semantic-automation-provider-v2.md → history/ios-semantic-automation-provider-v2.md} +6 -2
- package/docs/architecture/history.md +3 -34
- package/docs/architecture/index.md +15 -98
- package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +1 -1
- package/docs/architecture/reviews/2026-08-22-semantic-harness-convergence-acceptance.md +39 -0
- package/docs/architecture/versions/1.6.md +38 -0
- package/docs/architecture/versions/1.7.md +41 -0
- package/docs/forge-chatgpt-browser-engine.md +2 -2
- package/docs/forge-chatgpt-mcp-setup.md +4 -28
- package/docs/forge-chatgpt-supervised-automation.md +3 -3
- package/docs/forge-github-issue-launcher.md +1 -1
- package/docs/forge-plugin-management.md +1 -1
- package/docs/forge-runtime-self-healing-loop.md +1 -1
- package/docs/operations/controller-browser-plugin.md +26 -6
- package/docs/operations/controller-performance-and-502.md +143 -0
- package/docs/operations/controller-reliability-runbook.md +1 -1
- package/docs/operations/features.md +2 -2
- package/docs/operations/github-repository.md +1 -1
- package/docs/operations/homebrew.md +2 -2
- package/docs/operations/mcp-tool-exposure.md +5 -3
- package/docs/wiki/Architecture.md +6 -6
- package/docs/wiki/Implementation.md +2 -2
- package/docs/wiki/Integrations.md +2 -2
- package/docs/wiki/Work-Lifecycle.md +1 -1
- package/package.json +33 -11
- package/scripts/activate-source-baseline.ts +17 -13
- package/scripts/architecture-event.ts +2 -820
- package/scripts/architecture-queue.sh +2 -645
- package/scripts/archive-architecture-request.sh +2 -242
- package/scripts/archive-workflow.sh +1 -291
- package/scripts/benchmark-background-check-overlap.ts +228 -0
- package/scripts/benchmark-mcp-host.sh +117 -0
- package/scripts/benchmark-mcp-transport.ts +170 -0
- package/scripts/benchmark-quality-harness.ts +146 -0
- package/scripts/benchmark-route-session-concurrency.ts +207 -6
- package/scripts/benchmark-thin-harness-mcp-e2e.ts +7 -1
- package/scripts/benchmark-typescript-navigation.ts +186 -0
- package/scripts/benchmark-work-efficiency.ts +334 -0
- package/scripts/capability-config.ts +2 -392
- package/scripts/capability-resolver.ts +2 -503
- package/scripts/capture-plan.sh +1 -332
- package/scripts/check-agent-tooling.sh +2 -1558
- package/scripts/check-architecture-sync.sh +2 -371
- package/scripts/check-brain-manifest.sh +2 -253
- package/scripts/check-context-files.sh +2 -47
- package/scripts/check-deploy-sql-order.sh +2 -115
- package/scripts/check-mcp-compatibility.ts +14 -0
- package/scripts/check-runtime-architecture.mjs +349 -79
- package/scripts/check-skill-version.ts +3 -191
- package/scripts/check-source-duplication.mjs +61 -0
- package/scripts/check-stable-baseline.ts +92 -0
- package/scripts/check-task-sync.sh +2 -63
- package/scripts/check-task-workflow.sh +2 -1000
- package/scripts/codex-handoff-resume.sh +2 -267
- package/scripts/context-contract-sync.sh +2 -514
- package/scripts/contract-run.ts +2 -463
- package/scripts/contract-worktree.sh +1 -722
- package/scripts/ensure-task-workflow.sh +1 -1225
- package/scripts/host-temporary-cloud-mcp-e2e.sh +265 -0
- package/scripts/inspect-project-state.ts +4 -279
- package/scripts/install-standalone-recovery.ts +24 -0
- package/scripts/lib/project-init-lib.sh +22 -46
- package/scripts/lib/run-forge-helper.sh +19 -0
- package/scripts/lib/run-forge-helper.ts +26 -0
- package/scripts/migrate-project-template.sh +1 -1189
- package/scripts/migrate-workflow-docs.ts +4 -412
- package/scripts/new-plan.sh +1 -180
- package/scripts/new-spec.sh +1 -47
- package/scripts/new-sprint.sh +1 -46
- package/scripts/plan-to-todo.sh +1 -796
- package/scripts/prepare-codex-handoff.sh +2 -201
- package/scripts/prepare-handoff.sh +1 -78
- package/scripts/refresh-current-status.sh +2 -467
- package/scripts/run-governed-gate.ts +5 -1
- package/scripts/select-agent-context-blocks.sh +2 -64
- package/scripts/ship-worktrees.sh +1 -558
- package/scripts/smoke-ios-physical-device.ts +4 -2
- package/scripts/smoke-runtime-recovery.ts +1 -35
- package/scripts/smoke-schedule-engine.ts +1 -20
- package/scripts/sprint-backlog.sh +1 -834
- package/scripts/stage-runtime-release.ts +2 -1
- package/scripts/summarize-failures.sh +2 -123
- package/scripts/switch-plan.sh +1 -150
- package/scripts/sync-brain-docs.sh +2 -379
- package/scripts/sync-generated-authority.mjs +175 -0
- package/scripts/verify-contract.sh +2 -703
- package/scripts/verify-forge-runtime.sh +0 -1
- package/scripts/verify-sprint.sh +1 -677
- package/scripts/workflow-contract.ts +4 -205
- package/scripts/workstream-sync.sh +2 -261
- package/src/cli/agent-jobs/executor-health.ts +1 -1
- package/src/cli/commands/init-hook.ts +3 -3
- package/src/cli/commands/mcp.ts +9 -5
- package/src/cli/commands/plugin.ts +12 -3
- package/src/cli/commands/recovery.ts +25 -3
- package/src/cli/commands/runtime.ts +55 -3
- package/src/cli/commands/security.ts +12 -1
- package/src/cli/commands/setup-profile.ts +31 -21
- package/src/cli/controller/check-runner.ts +113 -5
- package/src/cli/controller/composite-result.ts +1 -1
- package/src/cli/controller/context/focus.ts +51 -0
- package/src/cli/controller/context/instruction-context.ts +114 -0
- package/src/cli/controller/context/known-paths.ts +76 -0
- package/src/cli/controller/context/query-planning.ts +110 -0
- package/src/cli/controller/context/rule-resolution.ts +532 -0
- package/src/cli/controller/context/source-materializer.ts +278 -0
- package/src/cli/controller/context/structural.ts +16 -0
- package/src/cli/controller/context/types.ts +286 -0
- package/src/cli/controller/context-pack.ts +828 -487
- package/src/cli/controller/launch-agents.ts +77 -2
- package/src/cli/controller/operational-plan.ts +2 -2
- package/src/cli/controller/work-mode.ts +6 -4
- package/src/cli/editing/edit-session.ts +60 -7
- package/src/cli/github/contracts.ts +16 -0
- package/src/cli/github/github.ts +135 -16
- package/src/cli/github/session.ts +2 -1
- package/src/cli/installer/managed-entries.ts +15 -1
- package/src/cli/local-bridge/console-automations.ts +54 -78
- package/src/cli/local-bridge/console-view-models.ts +1 -35
- package/src/cli/local-bridge/facade-api.ts +37 -264
- package/src/cli/local-bridge/frontend/src/App.tsx +3 -3
- package/src/cli/local-bridge/frontend/src/features/automations/AutomationsView.tsx +23 -13
- package/src/cli/local-bridge/frontend/src/features/capabilities/CapabilitiesView.tsx +37 -4
- package/src/cli/local-bridge/frontend/src/features/overview/OverviewView.tsx +6 -4
- package/src/cli/local-bridge/frontend/src/lib/api.ts +1 -6
- package/src/cli/local-bridge/frontend/src/lib/routes.ts +1 -1
- package/src/cli/local-bridge/frontend/src/shell/Sidebar.tsx +2 -2
- package/src/cli/local-bridge/frontend/src/styles.css +1 -1
- package/src/cli/local-bridge/frontend/src/types.ts +5 -8
- package/src/cli/local-bridge/job-store.ts +1 -490
- package/src/cli/local-bridge/server.ts +2 -515
- package/src/cli/local-bridge/ui-dist/app.css +1 -1
- package/src/cli/local-bridge/ui-dist/app.js +3703 -3757
- package/src/cli/mcp/auth.ts +5 -5
- package/src/cli/mcp/instructions.ts +3 -2
- package/src/cli/mcp/legacy-tool-service.ts +10 -222
- package/src/cli/mcp/multi-repository.ts +16 -5
- package/src/cli/mcp/oauth.ts +175 -26
- package/src/cli/mcp/post-finalize-work-attribution.ts +150 -0
- package/src/cli/mcp/repository-tools.ts +282 -146
- package/src/cli/mcp/server.ts +586 -116
- package/src/cli/mcp/tool-capability-cache.ts +1 -1
- package/src/cli/mcp/tool-contract.ts +14 -0
- package/src/cli/mcp/toolset-selection.ts +1 -1
- package/src/cli/mcp/toolset.ts +44 -7
- package/src/cli/mcp/transports/http.ts +124 -14
- package/src/cli/mcp/transports/session-registry.ts +43 -5
- package/src/cli/mcp/types.ts +3 -3
- package/src/cli/repositories/command-classifier.ts +103 -21
- package/src/cli/repositories/command-executor.ts +78 -526
- package/src/cli/repositories/command-process.ts +108 -0
- package/src/cli/repositories/command-scope.ts +99 -41
- package/src/cli/repositories/controller-home.ts +7 -8
- package/src/cli/repositories/git-command-executor.ts +2 -0
- package/src/cli/repositories/locks.ts +53 -16
- package/src/cli/repositories/repository-snapshot.ts +124 -0
- package/src/cli/repositories/selected-path-actions.ts +17 -9
- package/src/cli/repositories/structured-git.ts +118 -6
- package/src/cli/repository/inspector.ts +563 -65
- package/src/cli/repository/session-cache.ts +58 -17
- package/src/cli/v81-entry.ts +1 -4
- package/src/effects/fs-transaction.ts +85 -17
- package/src/runtime/context/codegraph-read-provider.ts +233 -2
- package/src/runtime/context/codegraph-sidecar.cjs +74 -40
- package/src/runtime/context/generic-lsp-provider.ts +355 -0
- package/src/runtime/context/lsp-client.ts +283 -0
- package/src/runtime/context/semantic-navigation-contract.ts +61 -0
- package/src/runtime/context/semantic-navigation.ts +223 -0
- package/src/runtime/context/swift-navigation.ts +565 -0
- package/src/runtime/context/typescript-navigation.ts +226 -0
- package/src/runtime/control-plane/dispatch-priority.ts +28 -0
- package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +48 -16
- package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +86 -24
- package/src/runtime/control-plane/execution/repository-work-attribution.ts +106 -0
- package/src/runtime/control-plane/execution/retained-work-resume.ts +203 -0
- package/src/runtime/control-plane/execution/session-store.ts +18 -0
- package/src/runtime/control-plane/execution/verification-evidence.ts +58 -3
- package/src/runtime/control-plane/execution/work-completion-authority.ts +58 -0
- package/src/runtime/control-plane/execution/work-evidence-policy.ts +200 -0
- package/src/runtime/control-plane/execution/work-execution-support.ts +386 -0
- package/src/runtime/control-plane/execution/work-finalization-service.ts +1913 -0
- package/src/runtime/control-plane/execution/work-handle-authority.ts +96 -0
- package/src/runtime/control-plane/execution/work-handle-store.ts +27 -2
- package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +258 -0
- package/src/runtime/control-plane/execution/work-operation-service.ts +468 -0
- package/src/runtime/control-plane/execution/work-path-scope.ts +37 -0
- package/src/runtime/control-plane/execution/work-preparation-service.ts +479 -0
- package/src/runtime/control-plane/execution/work-process-evidence.ts +129 -0
- package/src/runtime/control-plane/execution/work-remote-delivery.ts +215 -0
- package/src/runtime/control-plane/execution/work-task-receipt.ts +22 -18
- package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +156 -11
- package/src/runtime/control-plane/execution/work-validation-reconciler.ts +22 -14
- package/src/runtime/control-plane/execution/work-verification-context.ts +56 -0
- package/src/runtime/control-plane/execution/work-verification-service.ts +388 -0
- package/src/runtime/control-plane/execution/work-verification-snapshot.ts +90 -9
- package/src/runtime/control-plane/facade/capability-registry.ts +129 -25
- package/src/runtime/control-plane/facade/controller-round-relay.ts +1118 -0
- package/src/runtime/control-plane/facade/controller-session-store.ts +302 -12
- package/src/runtime/control-plane/facade/goal-workloop-access.ts +17 -1
- package/src/runtime/control-plane/facade/goal-workloop.ts +928 -190
- package/src/runtime/control-plane/facade/index.ts +3 -0
- package/src/runtime/control-plane/facade/plan-contract-store.ts +275 -42
- package/src/runtime/control-plane/facade/repository-work-admission.ts +147 -0
- package/src/runtime/control-plane/facade/requirement-authority.ts +83 -0
- package/src/runtime/control-plane/facade/semantic-admission.ts +205 -0
- package/src/runtime/control-plane/facade/suggested-actions.ts +5 -1
- package/src/runtime/control-plane/facade/types.ts +90 -10
- package/src/runtime/control-plane/facade/work-contract-store.ts +157 -261
- package/src/runtime/control-plane/facade/work-state-machine.ts +250 -0
- package/src/runtime/control-plane/generated-cache-retention.ts +271 -0
- package/src/runtime/control-plane/global-scheduler/config.ts +44 -0
- package/src/runtime/control-plane/global-scheduler/dispatch-capacity.ts +80 -0
- package/src/runtime/control-plane/global-scheduler/durable-admission.ts +46 -0
- package/src/runtime/control-plane/global-scheduler/maintenance.ts +152 -0
- package/src/runtime/control-plane/global-scheduler/projection-refresh.ts +84 -0
- package/src/runtime/control-plane/global-scheduler/reconciliation.ts +3 -3
- package/src/runtime/control-plane/global-scheduler/scheduler.ts +278 -543
- package/src/runtime/control-plane/global-scheduler/source-scan.ts +59 -0
- package/src/runtime/control-plane/global-scheduler/state.ts +88 -0
- package/src/runtime/control-plane/global-scheduler/worker-attachment.ts +57 -0
- package/src/runtime/control-plane/global-scheduler/worker-exit-decision.ts +38 -0
- package/src/runtime/control-plane/global-scheduler/worker-exit-reconciler.ts +175 -0
- package/src/runtime/control-plane/global-scheduler/worker-launch.ts +111 -0
- package/src/runtime/control-plane/global-scheduler/worker-lifecycle-store.ts +64 -0
- package/src/runtime/control-plane/global-scheduler/worker-lifecycle.ts +150 -0
- package/src/runtime/control-plane/global-scheduler/worker-process.ts +119 -0
- package/src/runtime/control-plane/global-scheduler/worker-stderr.ts +60 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-binding-store.ts +63 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +365 -117
- package/src/runtime/control-plane/launcher/launch-reservation-store.ts +89 -0
- package/src/runtime/control-plane/launcher/thin-launcher.ts +244 -11
- package/src/runtime/control-plane/persistence/controller-home-migration.ts +487 -0
- package/src/runtime/control-plane/persistence/requirement-store.ts +0 -12
- package/src/runtime/control-plane/persistence/sqlite-store.ts +33 -2
- package/src/runtime/control-plane/release-retention.ts +62 -2
- package/src/runtime/control-plane/repo-actor/registry.ts +4 -2
- package/src/runtime/control-plane/routing/route-policy.ts +24 -21
- package/src/runtime/control-plane/routing/workspace-admission.ts +70 -0
- package/src/runtime/control-plane/runtime-status-client.ts +0 -2
- package/src/runtime/diagnostics/mcp-timing.ts +13 -0
- package/src/runtime/diagnostics/mcp-transport-benchmark.ts +129 -0
- package/src/runtime/diagnostics/performance.ts +24 -1
- package/src/runtime/evidence/event-ledger.ts +1 -1
- package/src/runtime/evidence/process-check-execution.ts +18 -0
- package/src/runtime/execution/jobs/child-reference.ts +6 -2
- package/src/runtime/execution/jobs/store.ts +58 -4
- package/src/runtime/execution/jobs/types.ts +1 -1
- package/src/runtime/execution/managed-workspace.ts +203 -1
- package/src/runtime/execution/process-runtime/check-facade.ts +51 -4
- package/src/runtime/execution/process-runtime/check-result.ts +88 -0
- package/src/runtime/execution/process-runtime/check-runner-sidecar.ts +7 -1
- package/src/runtime/execution/process-runtime/check-scheduling.ts +114 -0
- package/src/runtime/execution/process-runtime/command-facade.ts +367 -40
- package/src/runtime/execution/process-runtime/gc.ts +38 -6
- package/src/runtime/execution/process-runtime/lightweight-managed.ts +1201 -0
- package/src/runtime/execution/process-runtime/persisted-check.ts +51 -5
- package/src/runtime/execution/process-runtime/resource-claims.ts +10 -6
- package/src/runtime/execution/process-runtime/runtime.ts +53 -3
- package/src/runtime/execution/process-runtime/store.ts +1 -0
- package/src/runtime/execution/process-runtime/types.ts +4 -12
- package/src/runtime/execution/thin-harness/async-process.ts +12 -2
- package/src/runtime/execution/thin-harness/search-worker.ts +54 -5
- package/src/runtime/execution/work-activity.ts +21 -0
- package/src/runtime/execution/workers/executor.ts +18 -61
- package/src/runtime/gateway/mcp/execution-tools.ts +35 -1852
- package/src/runtime/gateway/mcp/legacy-ios-tool-adapter.ts +61 -0
- package/src/runtime/gateway/mcp/process-tools.ts +83 -37
- package/src/runtime/gateway/mcp/router.ts +186 -84
- package/src/runtime/gateway/mcp/runtime-tool-definitions.ts +565 -0
- package/src/runtime/gateway/mcp/runtime-tools.ts +2378 -1947
- package/src/runtime/plugins/app-store-connect-adapter.ts +216 -47
- package/src/runtime/plugins/browser-adapter.ts +2662 -829
- package/src/runtime/plugins/browser-automation-service.ts +10 -0
- package/src/runtime/plugins/browser-handoff-host.ts +2 -25
- package/src/runtime/plugins/browser-handoff.ts +110 -5
- package/src/runtime/plugins/browser-macos-bridge.ts +701 -63
- package/src/runtime/plugins/browser-manifest-surface.ts +603 -0
- package/src/runtime/plugins/browser-native-session-service.ts +222 -0
- package/src/runtime/plugins/browser-node-bridge.ts +6 -34
- package/src/runtime/plugins/browser-provider-registry.ts +109 -0
- package/src/runtime/plugins/browser-runtime-contract.ts +154 -0
- package/src/runtime/plugins/browser-runtime.ts +193 -0
- package/src/runtime/plugins/browser-session-authority.ts +332 -0
- package/src/runtime/plugins/browser-session-store.ts +121 -0
- package/src/runtime/plugins/browser-session-types.ts +98 -0
- package/src/runtime/plugins/capability-authorization-grants.ts +354 -0
- package/src/runtime/plugins/desktop-operator-registration.ts +10 -4
- package/src/runtime/plugins/external-adapter.ts +358 -3
- package/src/runtime/plugins/external-unix-socket.ts +7 -1
- package/src/runtime/plugins/first-party-registry.ts +2 -0
- package/src/runtime/plugins/github-adapter.ts +125 -0
- package/src/runtime/plugins/ios/agent-device-provider.ts +1 -1
- package/src/runtime/plugins/ios/agent-device-typed-provider.ts +52 -9
- package/src/runtime/plugins/ios/remote-xpc-hid.ts +1171 -0
- package/src/runtime/plugins/ios-adapter.ts +106 -16
- package/src/runtime/plugins/ios-agent-device.ts +234 -64
- package/src/runtime/plugins/ios-physical-device.ts +1143 -392
- package/src/runtime/plugins/lightweight-action.ts +153 -0
- package/src/runtime/plugins/local-system-adapter.ts +123 -6
- package/src/runtime/plugins/macos-capability-broker.ts +47 -9
- package/src/runtime/plugins/plugin-action-sidecar.ts +59 -0
- package/src/runtime/plugins/plugin-management-adapter.ts +49 -8
- package/src/runtime/plugins/resend-adapter.ts +1 -1
- package/src/runtime/plugins/store.ts +330 -33
- package/src/runtime/plugins/types.ts +38 -3
- package/src/runtime/plugins/xiaohongshu-publish.ts +515 -0
- package/src/runtime/projections/git-status-sampler.ts +16 -16
- package/src/runtime/projections/materialized-view.ts +28 -10
- package/src/runtime/recovery/maintenance-executor.ts +426 -25
- package/src/runtime/root/mcp-transport.ts +180 -10
- package/src/runtime/root/package-connector-service.ts +230 -10
- package/src/runtime/root/package-runtime-release.ts +143 -17
- package/src/runtime/root/package-runtime-service.ts +461 -22
- package/src/runtime/root/release-manifest.ts +144 -18
- package/src/runtime/root/release-materialize.ts +125 -32
- package/src/runtime/root/release-store.ts +48 -11
- package/src/runtime/root/runtime.ts +23 -5
- package/src/runtime/root/service.ts +100 -29
- package/src/runtime/root/types.ts +12 -0
- package/src/runtime/safe-tooling/ios-development.ts +321 -63
- package/src/runtime/shared/trusted-executable.ts +62 -0
- package/src/runtime/shared/trusted-node-executable.ts +48 -0
- package/src/runtime/standalone-recovery/core.ts +1079 -240
- package/src/runtime/standalone-recovery/entry.ts +100 -24
- package/src/runtime/standalone-recovery/installer.ts +20 -4
- package/src/runtime/standalone-recovery/service-labels.ts +2 -0
- package/src/runtime/standalone-recovery/watchdog-heartbeat.ts +160 -0
- package/src/runtime/workflow/schedules/browser-probe.ts +81 -3
- package/src/runtime/workflow/schedules/engine.ts +364 -202
- package/src/runtime/workflow/schedules/github-issue-watch.ts +158 -0
- package/src/runtime/workflow/schedules/settlement.ts +64 -17
- package/src/runtime/workflow/schedules/store.ts +209 -19
- package/src/runtime/workflow/schedules/types.ts +2 -4
- package/src/runtime/workflow/schedules/work-continuation.ts +26 -13
- package/src/testing/test-governance.ts +8 -40
- package/assets/reference-configs/heartbeat-triage.md +0 -57
- package/assets/templates/helpers/harness-trace-grade.sh +0 -150
- package/assets/templates/helpers/heartbeat-triage.sh +0 -361
- package/assets/templates/helpers/maintenance-triage.sh +0 -167
- package/docs/architecture/current/approved-target-architecture.zh-CN.md +0 -1501
- package/docs/architecture/current/architecture-invariants.md +0 -331
- package/docs/architecture/current/automation-and-schedule-engine.md +0 -459
- package/docs/architecture/current/codegraph-context-provider.md +0 -238
- package/docs/architecture/current/control-plane-authority-inventory.md +0 -94
- package/docs/architecture/current/controller-repository-decoupling.md +0 -148
- package/docs/architecture/current/dispatch-and-agent-strategy.md +0 -373
- package/docs/architecture/current/entity-model.md +0 -612
- package/docs/architecture/current/external-plugin-governance.md +0 -396
- package/docs/architecture/current/failure-recovery.md +0 -447
- package/docs/architecture/current/governance.md +0 -213
- package/docs/architecture/current/human-interaction-plane.md +0 -87
- package/docs/architecture/current/implementation-status.md +0 -128
- package/docs/architecture/current/job-and-run-lifecycle.md +0 -413
- package/docs/architecture/current/legacy-governance.md +0 -58
- package/docs/architecture/current/local-workspace-target-grants.md +0 -70
- package/docs/architecture/current/migration-roadmap.md +0 -105
- package/docs/architecture/current/multi-repository-execution.md +0 -371
- package/docs/architecture/current/operations-runbook.md +0 -45
- package/docs/architecture/current/peer-supercontroller-runtime.md +0 -32
- package/docs/architecture/current/personal-assistant-plugin-baseline.md +0 -206
- package/docs/architecture/current/repository-access-modes.md +0 -102
- package/docs/architecture/current/route-policy.md +0 -15
- package/docs/architecture/current/route-session-concurrency.md +0 -85
- package/docs/architecture/current/runtime-architecture-simplification.md +0 -214
- package/docs/architecture/current/runtime-directory-map.md +0 -49
- package/docs/architecture/current/runtime-health-and-resource-lifecycle.md +0 -85
- package/docs/architecture/current/scheduler-and-resource-claims.md +0 -422
- package/docs/architecture/current/session-aware-execution-and-authorization.md +0 -77
- package/docs/architecture/current/system-overview.md +0 -169
- package/docs/architecture/current/target-requirements-traceability.md +0 -202
- package/docs/architecture/current/thin-harness-governance-policy.md +0 -32
- package/docs/architecture/current/thin-harness-v1.md +0 -321
- package/docs/architecture/current/verification-and-release-gates.md +0 -391
- package/docs/forge-autonomous-goal-loop.md +0 -9
- package/docs/forge-provider-configuration.md +0 -7
- package/docs/operations/chatgpt-personal-assistant-runtime.md +0 -93
- package/docs/operations/gmail-assistant-routines.md +0 -186
- package/docs/operations/personal-assistant-operator-guide.md +0 -61
- package/docs/operations/provider-configuration.md +0 -31
- package/scripts/harness-trace-grade.sh +0 -150
- package/scripts/heartbeat-triage.sh +0 -361
- package/scripts/maintenance-triage.sh +0 -167
- package/src/cli/local-bridge/frontend/src/features/settings/SettingsView.tsx +0 -2
- package/src/cli/repositories/goal-registry.ts +0 -239
- package/src/runtime/assistant/action-proposals.ts +0 -279
- package/src/runtime/assistant/intent.ts +0 -395
- package/src/runtime/assistant/model-provider.ts +0 -460
- package/src/runtime/assistant/openapi.ts +0 -223
- package/src/runtime/assistant/policy.ts +0 -89
- package/src/runtime/assistant/readiness.ts +0 -258
- package/src/runtime/assistant/routine-runtime.ts +0 -477
- package/src/runtime/assistant/schedule-binding.ts +0 -171
- package/src/runtime/assistant/standing-grants.ts +0 -375
- package/src/runtime/assistant/store.ts +0 -243
- package/src/runtime/assistant/types.ts +0 -135
- package/src/runtime/control-plane/goal-loop/config-facade.ts +0 -700
- package/src/runtime/control-plane/goal-loop/config-store.ts +0 -570
- package/src/runtime/control-plane/goal-loop/config-types.ts +0 -213
- package/src/runtime/control-plane/goal-loop/executor-router.ts +0 -109
- package/src/runtime/control-plane/goal-loop/failure-taxonomy.ts +0 -149
- package/src/runtime/control-plane/goal-loop/goal-contract-store.ts +0 -304
- package/src/runtime/control-plane/goal-loop/goal-loop-engine.ts +0 -1001
- package/src/runtime/control-plane/goal-loop/handoff-packet.ts +0 -160
- package/src/runtime/control-plane/goal-loop/index.ts +0 -13
- package/src/runtime/control-plane/goal-loop/local-tools.ts +0 -290
- package/src/runtime/control-plane/goal-loop/policy-gates.ts +0 -203
- package/src/runtime/control-plane/goal-loop/provider-dispatch.ts +0 -93
- package/src/runtime/control-plane/goal-loop/provider-registry.ts +0 -545
- package/src/runtime/control-plane/goal-loop/types.ts +0 -378
- package/src/runtime/control-plane/startup-recovery.ts +0 -198
- package/src/runtime/gateway/mcp/resource-policy.ts +0 -186
- package/src/runtime/personal-assistant/gmail-triage-manager.ts +0 -231
- package/src/runtime/personal-assistant/reporting-runtime.ts +0 -344
- package/src/runtime/personal-assistant/triage-runtime.ts +0 -748
- package/src/runtime/workflow/findings/store.ts +0 -204
- package/src/runtime/workflow/findings/types.ts +0 -58
- package/src/runtime/workflow/portfolio/engine.ts +0 -20
- package/src/runtime/workflow/portfolio/store.ts +0 -185
- package/src/runtime/workflow/portfolio/types.ts +0 -29
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { randomUUID } from 'crypto';
|
|
2
|
-
import { existsSync, mkdirSync } from 'fs';
|
|
1
|
+
import { createHash, randomUUID } from 'crypto';
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync } from 'fs';
|
|
3
3
|
import { join } from 'path';
|
|
4
4
|
import { spawnSync } from 'child_process';
|
|
5
|
+
import { performance } from 'perf_hooks';
|
|
5
6
|
import { repositoryControllerRoot } from '../../cli/repositories/controller-home';
|
|
7
|
+
import { runBoundedProcess } from '../execution/thin-harness/async-process';
|
|
6
8
|
import { readJsonFile, writeJsonAtomic } from '../shared/json-files';
|
|
7
9
|
import { AssistantPluginError, toAssistantPluginError } from './errors';
|
|
8
10
|
import {
|
|
@@ -19,15 +21,22 @@ import {
|
|
|
19
21
|
import type {
|
|
20
22
|
AssistantPluginActionDescriptor,
|
|
21
23
|
AssistantPluginActionExecutionInput,
|
|
24
|
+
AssistantPluginAuthorizationContext,
|
|
25
|
+
AssistantPluginAuthorizationTarget,
|
|
22
26
|
AssistantPluginCapability,
|
|
23
27
|
} from './types';
|
|
28
|
+
import { executeRemoteXpcHidInput, prewarmRemoteXpcHid, remoteXpcHidStatus, stopRemoteXpcHidForDevice } from './ios/remote-xpc-hid';
|
|
24
29
|
|
|
25
30
|
const PROVIDER = 'ios-device' as const;
|
|
26
31
|
const SESSION_EXPIRY_MS = 2 * 60 * 60_000;
|
|
32
|
+
const CORE_DEVICE_READY_CACHE_MS = 5 * 60_000;
|
|
33
|
+
const INPUT_UNLOCK_CACHE_MS = 60_000;
|
|
34
|
+
const INPUT_DISPLAY_CACHE_MS = 5 * 60_000;
|
|
35
|
+
const INPUT_FOREGROUND_OBSERVATION_TTL_MS = 30_000;
|
|
27
36
|
const MAX_JSON_BYTES = 64 * 1024;
|
|
28
37
|
const MAX_EVENTS = 200;
|
|
29
|
-
const
|
|
30
|
-
const
|
|
38
|
+
const MAX_BATCH_STEPS = 20;
|
|
39
|
+
const MAX_BATCH_WAIT_MS = 5_000;
|
|
31
40
|
|
|
32
41
|
interface CommandResult {
|
|
33
42
|
ok: boolean;
|
|
@@ -41,69 +50,69 @@ interface CommandOptions {
|
|
|
41
50
|
cwd?: string;
|
|
42
51
|
timeoutMs?: number;
|
|
43
52
|
env?: NodeJS.ProcessEnv;
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
export interface RunnerHttpResult {
|
|
47
|
-
ok: boolean;
|
|
48
|
-
status: number;
|
|
49
|
-
body?: unknown;
|
|
50
|
-
text: string;
|
|
53
|
+
signal?: AbortSignal;
|
|
51
54
|
}
|
|
52
55
|
|
|
53
56
|
export interface IosPhysicalDeviceRuntimeHooks {
|
|
54
57
|
platform(): NodeJS.Platform;
|
|
55
58
|
now(): Date;
|
|
56
59
|
runCommand(command: string, args: string[], options?: CommandOptions): CommandResult;
|
|
57
|
-
|
|
60
|
+
runCommandAsync(command: string, args: string[], options?: CommandOptions): Promise<CommandResult>;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function runCommandSync(command: string, args: string[], options: CommandOptions = {}): CommandResult {
|
|
64
|
+
const result = spawnSync(command, args, {
|
|
65
|
+
cwd: options.cwd,
|
|
66
|
+
env: options.env,
|
|
67
|
+
encoding: 'utf8',
|
|
68
|
+
timeout: options.timeoutMs ?? 30_000,
|
|
69
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
70
|
+
});
|
|
71
|
+
return {
|
|
72
|
+
ok: result.status === 0,
|
|
73
|
+
status: result.status,
|
|
74
|
+
stdout: String(result.stdout ?? ''),
|
|
75
|
+
stderr: String(result.stderr ?? result.error?.message ?? ''),
|
|
76
|
+
command: [command, ...args],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
async function runCommandAsync(command: string, args: string[], options: CommandOptions = {}): Promise<CommandResult> {
|
|
81
|
+
const result = await runBoundedProcess(command, args, {
|
|
82
|
+
cwd: options.cwd ?? process.cwd(),
|
|
83
|
+
env: options.env,
|
|
84
|
+
timeoutMs: options.timeoutMs ?? 30_000,
|
|
85
|
+
maxOutputBytes: 4 * 1024 * 1024,
|
|
86
|
+
signal: options.signal,
|
|
87
|
+
});
|
|
88
|
+
return {
|
|
89
|
+
ok: result.ok,
|
|
90
|
+
status: result.exitCode,
|
|
91
|
+
stdout: result.stdout,
|
|
92
|
+
stderr: result.stderr,
|
|
93
|
+
command: [command, ...args],
|
|
94
|
+
};
|
|
58
95
|
}
|
|
59
96
|
|
|
60
97
|
const defaultHooks: IosPhysicalDeviceRuntimeHooks = {
|
|
61
98
|
platform: () => process.platform,
|
|
62
99
|
now: () => new Date(),
|
|
63
|
-
runCommand:
|
|
64
|
-
|
|
65
|
-
cwd: options.cwd,
|
|
66
|
-
env: options.env,
|
|
67
|
-
encoding: 'utf8',
|
|
68
|
-
timeout: options.timeoutMs ?? 30_000,
|
|
69
|
-
maxBuffer: 4 * 1024 * 1024,
|
|
70
|
-
});
|
|
71
|
-
return {
|
|
72
|
-
ok: result.status === 0,
|
|
73
|
-
status: result.status,
|
|
74
|
-
stdout: String(result.stdout ?? ''),
|
|
75
|
-
stderr: String(result.stderr ?? result.error?.message ?? ''),
|
|
76
|
-
command: [command, ...args],
|
|
77
|
-
};
|
|
78
|
-
},
|
|
79
|
-
requestJson: async (method, url, body, timeoutMs = 30_000) => {
|
|
80
|
-
const controller = new AbortController();
|
|
81
|
-
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
82
|
-
try {
|
|
83
|
-
const response = await fetch(url, {
|
|
84
|
-
method,
|
|
85
|
-
headers: body === undefined ? undefined : { 'content-type': 'application/json' },
|
|
86
|
-
body: body === undefined ? undefined : JSON.stringify(body),
|
|
87
|
-
signal: controller.signal,
|
|
88
|
-
});
|
|
89
|
-
const text = await response.text();
|
|
90
|
-
let parsed: unknown;
|
|
91
|
-
try { parsed = text ? JSON.parse(text) : undefined; } catch { parsed = undefined; }
|
|
92
|
-
return { ok: response.ok, status: response.status, body: parsed, text };
|
|
93
|
-
} finally {
|
|
94
|
-
clearTimeout(timer);
|
|
95
|
-
}
|
|
96
|
-
},
|
|
100
|
+
runCommand: runCommandSync,
|
|
101
|
+
runCommandAsync,
|
|
97
102
|
};
|
|
98
103
|
|
|
99
104
|
let hooks: IosPhysicalDeviceRuntimeHooks = { ...defaultHooks };
|
|
100
105
|
|
|
101
106
|
export function setIosPhysicalDeviceRuntimeHooksForTest(overrides: Partial<IosPhysicalDeviceRuntimeHooks>): void {
|
|
102
107
|
hooks = { ...defaultHooks, ...overrides };
|
|
108
|
+
if (overrides.runCommand && !overrides.runCommandAsync) {
|
|
109
|
+
hooks.runCommandAsync = async (command, args, options = {}) => overrides.runCommand!(command, args, options);
|
|
110
|
+
}
|
|
103
111
|
}
|
|
104
112
|
|
|
105
113
|
export function resetIosPhysicalDeviceRuntimeHooksForTest(): void {
|
|
106
114
|
hooks = { ...defaultHooks };
|
|
115
|
+
coreDeviceReadyCache = undefined;
|
|
107
116
|
}
|
|
108
117
|
|
|
109
118
|
interface PhysicalDevice {
|
|
@@ -124,6 +133,19 @@ interface PhysicalDevice {
|
|
|
124
133
|
connected: boolean;
|
|
125
134
|
}
|
|
126
135
|
|
|
136
|
+
interface PhysicalDeviceCapabilitySummary {
|
|
137
|
+
applicationControl: boolean;
|
|
138
|
+
screenshot: boolean;
|
|
139
|
+
displayInfo: boolean;
|
|
140
|
+
lockState: boolean;
|
|
141
|
+
viewDeviceScreen: boolean;
|
|
142
|
+
hidDigitizer: boolean;
|
|
143
|
+
hidKeyboard: boolean;
|
|
144
|
+
hidScroll: boolean;
|
|
145
|
+
hidButton: boolean;
|
|
146
|
+
universalHid: boolean;
|
|
147
|
+
}
|
|
148
|
+
|
|
127
149
|
interface InstalledApp {
|
|
128
150
|
name: string;
|
|
129
151
|
bundleIdentifier: string;
|
|
@@ -138,25 +160,85 @@ interface PhysicalEvent {
|
|
|
138
160
|
details?: unknown;
|
|
139
161
|
}
|
|
140
162
|
|
|
163
|
+
interface DisplayGeometry {
|
|
164
|
+
width: number;
|
|
165
|
+
height: number;
|
|
166
|
+
pointScale?: number;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
interface CachedDisplayGeometry extends DisplayGeometry {
|
|
170
|
+
observedAt: string;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
interface PhysicalScreenshotObservation {
|
|
174
|
+
observationId: string;
|
|
175
|
+
observedAt: string;
|
|
176
|
+
label: string;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
interface PhysicalForegroundObservation {
|
|
180
|
+
observationId: string;
|
|
181
|
+
screenshotObservedAt: string;
|
|
182
|
+
confirmedAt: string;
|
|
183
|
+
bundleId: string;
|
|
184
|
+
}
|
|
185
|
+
|
|
141
186
|
interface PhysicalSessionState {
|
|
142
187
|
schemaVersion: 1;
|
|
143
188
|
interactionId: string;
|
|
144
189
|
device: PhysicalDevice;
|
|
145
190
|
bundleId: string;
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
sessionId?: string;
|
|
151
|
-
error?: string;
|
|
152
|
-
};
|
|
191
|
+
display?: CachedDisplayGeometry;
|
|
192
|
+
unlockVerifiedAt?: string;
|
|
193
|
+
lastScreenshot?: PhysicalScreenshotObservation;
|
|
194
|
+
foregroundObservation?: PhysicalForegroundObservation;
|
|
153
195
|
events: PhysicalEvent[];
|
|
154
196
|
}
|
|
155
197
|
|
|
198
|
+
interface CoreDeviceStatus {
|
|
199
|
+
available: boolean;
|
|
200
|
+
platform: NodeJS.Platform;
|
|
201
|
+
coreDeviceReady: boolean;
|
|
202
|
+
devicectlVersion?: string;
|
|
203
|
+
reason?: string;
|
|
204
|
+
probed?: boolean;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
type TimingStages = Record<string, { ms: number; cached?: boolean }>;
|
|
208
|
+
let coreDeviceReadyCache: { checkedAtMs: number; status: CoreDeviceStatus } | undefined;
|
|
209
|
+
|
|
156
210
|
function timestamp(): string {
|
|
157
211
|
return hooks.now().toISOString();
|
|
158
212
|
}
|
|
159
213
|
|
|
214
|
+
function elapsedMs(startedAt: number): number {
|
|
215
|
+
return Math.round((performance.now() - startedAt) * 100) / 100;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function recordTiming(stages: TimingStages, name: string, startedAt: number, cached?: boolean): void {
|
|
219
|
+
stages[name] = { ms: elapsedMs(startedAt), ...(cached === undefined ? {} : { cached }) };
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
function finishTimedResult(
|
|
223
|
+
actionStartedAt: number,
|
|
224
|
+
stages: TimingStages,
|
|
225
|
+
result: Record<string, unknown>,
|
|
226
|
+
): Record<string, unknown> {
|
|
227
|
+
return {
|
|
228
|
+
...result,
|
|
229
|
+
timing: {
|
|
230
|
+
totalMs: elapsedMs(actionStartedAt),
|
|
231
|
+
stages,
|
|
232
|
+
},
|
|
233
|
+
};
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function cacheAgeMs(observedAt: string | undefined): number | undefined {
|
|
237
|
+
if (!observedAt) return undefined;
|
|
238
|
+
const parsed = Date.parse(observedAt);
|
|
239
|
+
return Number.isFinite(parsed) ? Math.max(0, hooks.now().getTime() - parsed) : undefined;
|
|
240
|
+
}
|
|
241
|
+
|
|
160
242
|
function sanitize(value: string): string {
|
|
161
243
|
return value.replace(/[^a-zA-Z0-9._-]+/g, '-').replace(/^-+|-+$/g, '') || 'ios-device';
|
|
162
244
|
}
|
|
@@ -235,6 +317,11 @@ function optionalString(value: unknown): string | undefined {
|
|
|
235
317
|
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
236
318
|
}
|
|
237
319
|
|
|
320
|
+
function requireFiniteNumber(value: unknown, name: string): number {
|
|
321
|
+
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
|
322
|
+
throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `${name} must be a finite number.`, { retryable: false });
|
|
323
|
+
}
|
|
324
|
+
|
|
238
325
|
function requireBundleId(value: unknown): string {
|
|
239
326
|
const bundleId = requireString(value, 'bundle_id');
|
|
240
327
|
if (!/^[A-Za-z0-9][A-Za-z0-9.-]+$/.test(bundleId) || !bundleId.includes('.') || /^[a-z][a-z0-9+.-]*:\/\//i.test(bundleId)) {
|
|
@@ -255,18 +342,19 @@ function commandFailure(result: CommandResult, code: string, fallback: string):
|
|
|
255
342
|
});
|
|
256
343
|
}
|
|
257
344
|
|
|
258
|
-
function runCoreJson(
|
|
345
|
+
async function runCoreJson(
|
|
259
346
|
input: AssistantPluginActionExecutionInput,
|
|
260
347
|
args: string[],
|
|
261
348
|
code: string,
|
|
262
349
|
timeoutMs = 30_000,
|
|
263
|
-
): Record<string, unknown
|
|
350
|
+
): Promise<Record<string, unknown>> {
|
|
264
351
|
if (hooks.platform() !== 'darwin') {
|
|
265
352
|
throw new AssistantPluginError('PLUGIN_DEPENDENCY_MISSING', 'Physical iOS device support requires macOS and Xcode CoreDevice.', { retryable: false });
|
|
266
353
|
}
|
|
267
|
-
const result = hooks.
|
|
354
|
+
const result = await hooks.runCommandAsync('xcrun', ['devicectl', ...args, '--json-output', '-'], {
|
|
268
355
|
cwd: input.repoRoot,
|
|
269
356
|
timeoutMs,
|
|
357
|
+
signal: input.signal,
|
|
270
358
|
});
|
|
271
359
|
if (!result.ok) return commandFailure(result, code, 'CoreDevice command failed.');
|
|
272
360
|
let parsed: Record<string, unknown> | undefined;
|
|
@@ -289,8 +377,30 @@ function stringField(value: Record<string, unknown>, key: string): string | unde
|
|
|
289
377
|
return typeof value[key] === 'string' && String(value[key]).trim() ? String(value[key]).trim() : undefined;
|
|
290
378
|
}
|
|
291
379
|
|
|
292
|
-
function
|
|
293
|
-
|
|
380
|
+
function capabilityNames(value: unknown): string[] {
|
|
381
|
+
return (Array.isArray(value) ? value : [])
|
|
382
|
+
.map((entry) => stringField(objectValue(entry), 'name'))
|
|
383
|
+
.filter((entry): entry is string => Boolean(entry));
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
function capabilitySummary(names: string[]): PhysicalDeviceCapabilitySummary {
|
|
387
|
+
const features = new Set(names);
|
|
388
|
+
return {
|
|
389
|
+
applicationControl: features.has('Application Control') || features.has('Launch Application'),
|
|
390
|
+
screenshot: features.has('Capture Screenshot'),
|
|
391
|
+
displayInfo: features.has('Get Display Information'),
|
|
392
|
+
lockState: features.has('Get Lock State'),
|
|
393
|
+
viewDeviceScreen: features.has('View Device Screen'),
|
|
394
|
+
hidDigitizer: features.has('HID Digitizer'),
|
|
395
|
+
hidKeyboard: features.has('HID Keyboard'),
|
|
396
|
+
hidScroll: features.has('HID Scroll'),
|
|
397
|
+
hidButton: features.has('HID Button'),
|
|
398
|
+
universalHid: features.has('Universal HID Service Pool') || features.has('UniversalHIDService'),
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
async function physicalDevices(input: AssistantPluginActionExecutionInput): Promise<PhysicalDevice[]> {
|
|
403
|
+
const response = await runCoreJson(input, ['list', 'devices'], 'IOS_DEVICE_LIST_FAILED');
|
|
294
404
|
const result = objectValue(response.result);
|
|
295
405
|
const entries = Array.isArray(result.devices) ? result.devices : [];
|
|
296
406
|
return entries
|
|
@@ -299,8 +409,7 @@ function physicalDevices(input: AssistantPluginActionExecutionInput): PhysicalDe
|
|
|
299
409
|
const hardware = objectValue(entry.hardwareProperties);
|
|
300
410
|
const device = objectValue(entry.deviceProperties);
|
|
301
411
|
const connection = objectValue(entry.connectionProperties);
|
|
302
|
-
const capabilities =
|
|
303
|
-
const capabilityNames = capabilities.map((item) => stringField(objectValue(item), 'name')).filter(Boolean);
|
|
412
|
+
const capabilities = capabilityNames(entry.capabilities);
|
|
304
413
|
const pairingState = stringField(connection, 'pairingState');
|
|
305
414
|
const tunnelState = stringField(connection, 'tunnelState');
|
|
306
415
|
const bootState = stringField(device, 'bootState');
|
|
@@ -318,13 +427,13 @@ function physicalDevices(input: AssistantPluginActionExecutionInput): PhysicalDe
|
|
|
318
427
|
bootState,
|
|
319
428
|
developerMode: stringField(device, 'developerModeStatus'),
|
|
320
429
|
ddiServicesAvailable: device.ddiServicesAvailable === true,
|
|
321
|
-
screenshotAvailable:
|
|
430
|
+
screenshotAvailable: capabilities.includes('Capture Screenshot'),
|
|
322
431
|
connected: pairingState === 'paired'
|
|
323
432
|
&& bootState !== 'shutdown'
|
|
324
433
|
&& (tunnelState === 'connected'
|
|
325
434
|
|| device.ddiServicesAvailable === true
|
|
326
|
-
||
|
|
327
|
-
||
|
|
435
|
+
|| capabilities.includes('Application Control')
|
|
436
|
+
|| capabilities.includes('Launch Application')
|
|
328
437
|
|| connection.transportType === 'usb'
|
|
329
438
|
|| connection.transportType === 'wired'),
|
|
330
439
|
reality: stringField(hardware, 'reality'),
|
|
@@ -337,9 +446,9 @@ function physicalDevices(input: AssistantPluginActionExecutionInput): PhysicalDe
|
|
|
337
446
|
.map(({ reality: _reality, platform: _platform, ...entry }) => entry);
|
|
338
447
|
}
|
|
339
448
|
|
|
340
|
-
function selectDevice(input: AssistantPluginActionExecutionInput, selectorValue: unknown): PhysicalDevice {
|
|
449
|
+
async function selectDevice(input: AssistantPluginActionExecutionInput, selectorValue: unknown): Promise<PhysicalDevice> {
|
|
341
450
|
const selector = requireString(selectorValue, 'device');
|
|
342
|
-
const inventory = physicalDevices(input);
|
|
451
|
+
const inventory = await physicalDevices(input);
|
|
343
452
|
const matches = inventory.filter((entry) => entry.identifier === selector || entry.udid === selector || entry.name === selector);
|
|
344
453
|
if (matches.length === 0) {
|
|
345
454
|
throw new AssistantPluginError('IOS_DEVICE_NOT_FOUND', 'No paired physical iPhone matches the exact device selector.', {
|
|
@@ -360,14 +469,130 @@ function selectDevice(input: AssistantPluginActionExecutionInput, selectorValue:
|
|
|
360
469
|
return selected;
|
|
361
470
|
}
|
|
362
471
|
|
|
363
|
-
function
|
|
472
|
+
async function physicalDeviceLockState(input: AssistantPluginActionExecutionInput, device: PhysicalDevice): Promise<Record<string, unknown>> {
|
|
473
|
+
const response = await runCoreJson(input, [
|
|
474
|
+
'device', 'info', 'lockState', '--device', device.identifier,
|
|
475
|
+
], 'IOS_DEVICE_LOCK_STATE_FAILED', 30_000);
|
|
476
|
+
return objectValue(response.result);
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
async function requirePhysicalDeviceUnlocked(input: AssistantPluginActionExecutionInput, device: PhysicalDevice): Promise<void> {
|
|
480
|
+
const lockState = await physicalDeviceLockState(input, device);
|
|
481
|
+
if (lockState.passcodeRequired !== true) return;
|
|
482
|
+
throw new AssistantPluginError(
|
|
483
|
+
'IOS_DEVICE_LOCKED',
|
|
484
|
+
`The selected iPhone ${device.name} is locked. Unlock it before launching an app through CoreDevice.`,
|
|
485
|
+
{
|
|
486
|
+
retryable: true,
|
|
487
|
+
details: {
|
|
488
|
+
device: { identifier: device.identifier, name: device.name },
|
|
489
|
+
lockState: bounded(lockState),
|
|
490
|
+
},
|
|
491
|
+
},
|
|
492
|
+
);
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
async function physicalDisplayGeometry(input: AssistantPluginActionExecutionInput, device: PhysicalDevice): Promise<DisplayGeometry> {
|
|
496
|
+
const displayResponse = await runCoreJson(input, [
|
|
497
|
+
'device', 'info', 'displays', '--device', device.identifier,
|
|
498
|
+
], 'IOS_DEVICE_DISPLAY_INFO_FAILED', 30_000);
|
|
499
|
+
const result = objectValue(displayResponse.result);
|
|
500
|
+
const displays = Array.isArray(result.displays) ? result.displays : [];
|
|
501
|
+
const rows = displays.filter((entry): entry is Record<string, unknown> => Boolean(entry) && typeof entry === 'object' && !Array.isArray(entry));
|
|
502
|
+
const primary = rows.find((entry) => entry.primary === true) ?? rows[0];
|
|
503
|
+
if (!primary) {
|
|
504
|
+
throw new AssistantPluginError('IOS_HID_DISPLAY_INVALID', 'CoreDevice did not report a usable iPhone display for input coordinate mapping.', { retryable: true });
|
|
505
|
+
}
|
|
506
|
+
const bounds = Array.isArray(primary.bounds) ? primary.bounds : undefined;
|
|
507
|
+
const size = bounds && Array.isArray(bounds[1]) ? bounds[1] : Array.isArray(primary.nativeSize) ? primary.nativeSize : undefined;
|
|
508
|
+
const width = Number(size?.[0]);
|
|
509
|
+
const height = Number(size?.[1]);
|
|
510
|
+
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 1 || height <= 1) {
|
|
511
|
+
throw new AssistantPluginError('IOS_HID_DISPLAY_INVALID', 'CoreDevice returned invalid display dimensions for input coordinate mapping.', {
|
|
512
|
+
retryable: true,
|
|
513
|
+
details: { display: bounded(primary) },
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
return {
|
|
517
|
+
width: Math.trunc(width),
|
|
518
|
+
height: Math.trunc(height),
|
|
519
|
+
pointScale: typeof primary.pointScale === 'number' && Number.isFinite(primary.pointScale) ? primary.pointScale : undefined,
|
|
520
|
+
};
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
function pngDisplayGeometry(path: string, pointScale?: number): DisplayGeometry | undefined {
|
|
524
|
+
try {
|
|
525
|
+
const header = readFileSync(path).subarray(0, 24);
|
|
526
|
+
const signature = header.subarray(0, 8).toString('hex');
|
|
527
|
+
if (header.length < 24 || signature !== '89504e470d0a1a0a') return undefined;
|
|
528
|
+
const width = header.readUInt32BE(16);
|
|
529
|
+
const height = header.readUInt32BE(20);
|
|
530
|
+
if (width <= 1 || height <= 1) return undefined;
|
|
531
|
+
return { width, height, pointScale };
|
|
532
|
+
} catch {
|
|
533
|
+
return undefined;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
async function sessionDisplayGeometry(
|
|
538
|
+
input: AssistantPluginActionExecutionInput,
|
|
539
|
+
state: PhysicalSessionState,
|
|
540
|
+
): Promise<{ display: DisplayGeometry; cached: boolean }> {
|
|
541
|
+
const age = cacheAgeMs(state.display?.observedAt);
|
|
542
|
+
if (state.display && age !== undefined && age <= INPUT_DISPLAY_CACHE_MS) {
|
|
543
|
+
const { observedAt: _observedAt, ...display } = state.display;
|
|
544
|
+
return { display, cached: true };
|
|
545
|
+
}
|
|
546
|
+
const display = await physicalDisplayGeometry(input, state.device);
|
|
547
|
+
state.display = { ...display, observedAt: timestamp() };
|
|
548
|
+
writeState(input, state);
|
|
549
|
+
return { display, cached: false };
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
async function requireSessionUnlocked(
|
|
553
|
+
input: AssistantPluginActionExecutionInput,
|
|
554
|
+
state: PhysicalSessionState,
|
|
555
|
+
): Promise<{ cached: boolean }> {
|
|
556
|
+
const age = cacheAgeMs(state.unlockVerifiedAt);
|
|
557
|
+
if (age !== undefined && age <= INPUT_UNLOCK_CACHE_MS) return { cached: true };
|
|
558
|
+
await requirePhysicalDeviceUnlocked(input, state.device);
|
|
559
|
+
state.unlockVerifiedAt = timestamp();
|
|
560
|
+
writeState(input, state);
|
|
561
|
+
return { cached: false };
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
async function physicalDeviceInfo(input: AssistantPluginActionExecutionInput, device: PhysicalDevice): Promise<Record<string, unknown>> {
|
|
565
|
+
const detailsResponse = await runCoreJson(input, [
|
|
566
|
+
'device', 'info', 'details', '--device', device.identifier,
|
|
567
|
+
], 'IOS_DEVICE_DETAILS_FAILED', 60_000);
|
|
568
|
+
const details = objectValue(detailsResponse.result);
|
|
569
|
+
const deviceProperties = objectValue(details.deviceProperties);
|
|
570
|
+
const names = capabilityNames(details.capabilities);
|
|
571
|
+
|
|
572
|
+
const lockState = await physicalDeviceLockState(input, device);
|
|
573
|
+
const displayResponse = await runCoreJson(input, [
|
|
574
|
+
'device', 'info', 'displays', '--device', device.identifier,
|
|
575
|
+
], 'IOS_DEVICE_DISPLAY_INFO_FAILED', 30_000);
|
|
576
|
+
|
|
577
|
+
return {
|
|
578
|
+
device,
|
|
579
|
+
capabilities: capabilitySummary(names),
|
|
580
|
+
capabilityCount: names.length,
|
|
581
|
+
screenViewingURL: stringField(deviceProperties, 'screenViewingURL'),
|
|
582
|
+
lockState: bounded(lockState),
|
|
583
|
+
displays: bounded(displayResponse.result),
|
|
584
|
+
inputBackend: remoteXpcHidStatus(input.controllerHome),
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
async function installedApps(
|
|
364
589
|
input: AssistantPluginActionExecutionInput,
|
|
365
590
|
device: PhysicalDevice,
|
|
366
591
|
bundleId: string,
|
|
367
|
-
): InstalledApp[] {
|
|
592
|
+
): Promise<InstalledApp[]> {
|
|
368
593
|
let response: Record<string, unknown>;
|
|
369
594
|
try {
|
|
370
|
-
response = runCoreJson(input, [
|
|
595
|
+
response = await runCoreJson(input, [
|
|
371
596
|
'device', 'info', 'apps', '--device', device.identifier,
|
|
372
597
|
'--include-all-apps', '--bundle-id', bundleId,
|
|
373
598
|
], 'IOS_DEVICE_APPS_FAILED', 60_000);
|
|
@@ -394,110 +619,23 @@ function installedApps(
|
|
|
394
619
|
.filter((entry) => entry.bundleIdentifier === bundleId);
|
|
395
620
|
}
|
|
396
621
|
|
|
397
|
-
function
|
|
398
|
-
const
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
return { configured: true, endpoint: parsed.toString().replace(/\/$/, '') };
|
|
409
|
-
} catch {
|
|
410
|
-
return { configured: false, error: 'FORGE_IOS_DEVICE_RUNNER_URL is not a valid URL.' };
|
|
411
|
-
}
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
function runnerUrl(endpoint: string, path: string): string {
|
|
415
|
-
return `${endpoint}${path.startsWith('/') ? path : `/${path}`}`;
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
function responseValue(body: unknown): unknown {
|
|
419
|
-
const record = objectValue(body);
|
|
420
|
-
return 'value' in record ? record.value : body;
|
|
421
|
-
}
|
|
422
|
-
|
|
423
|
-
async function runnerRequest(
|
|
424
|
-
method: string,
|
|
425
|
-
endpoint: string,
|
|
426
|
-
path: string,
|
|
427
|
-
body: unknown,
|
|
428
|
-
code: string,
|
|
429
|
-
sensitive = false,
|
|
430
|
-
): Promise<unknown> {
|
|
431
|
-
let response: RunnerHttpResult;
|
|
432
|
-
try {
|
|
433
|
-
response = await hooks.requestJson(method, runnerUrl(endpoint, path), body, 30_000);
|
|
434
|
-
} catch (error) {
|
|
435
|
-
const normalized = toAssistantPluginError(error, {
|
|
436
|
-
code,
|
|
437
|
-
message: 'The local iOS UI runner could not be reached.',
|
|
438
|
-
retryable: true,
|
|
439
|
-
});
|
|
440
|
-
throw new AssistantPluginError(normalized.code, normalized.message, {
|
|
441
|
-
retryable: normalized.retryable,
|
|
442
|
-
details: sensitive ? { endpoint, response: '<redacted>' } : { endpoint },
|
|
443
|
-
});
|
|
444
|
-
}
|
|
445
|
-
if (!response.ok) {
|
|
446
|
-
throw new AssistantPluginError(code, sensitive ? 'The local iOS UI runner rejected a sensitive-input action.' : `The local iOS UI runner returned HTTP ${response.status}.`, {
|
|
447
|
-
retryable: response.status >= 500,
|
|
448
|
-
details: sensitive ? { status: response.status, response: '<redacted>' } : { status: response.status, response: bounded(response.body ?? response.text) },
|
|
622
|
+
function expirePhysicalDeviceSessions(input: AssistantPluginActionExecutionInput): number {
|
|
623
|
+
const nowMs = hooks.now().getTime();
|
|
624
|
+
let expired = 0;
|
|
625
|
+
for (const record of listInteractionSessions(input.repoRoot, PROVIDER)) {
|
|
626
|
+
if (!isInteractionSessionActive(record.status)) continue;
|
|
627
|
+
const expiresAtMs = Date.parse(record.expiresAt);
|
|
628
|
+
if (!Number.isFinite(expiresAtMs) || nowMs < expiresAtMs) continue;
|
|
629
|
+
const state = readState(input, record.interactionId);
|
|
630
|
+
patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, {
|
|
631
|
+
status: 'failed',
|
|
632
|
+
error: { code: 'IOS_DEVICE_SESSION_EXPIRED', message: 'The physical iOS interaction expired.' },
|
|
449
633
|
});
|
|
634
|
+
appendEvent(input, record.interactionId, 'session_expired');
|
|
635
|
+
if (state?.device.identifier) stopRemoteXpcHidForDevice(state.device.identifier);
|
|
636
|
+
expired += 1;
|
|
450
637
|
}
|
|
451
|
-
return
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
async function probeRunner(): Promise<{ configured: boolean; endpoint?: string; ready: boolean; error?: string; status?: unknown }> {
|
|
455
|
-
const configured = configuredRunner();
|
|
456
|
-
if (!configured.configured || !configured.endpoint) return { ...configured, ready: false };
|
|
457
|
-
try {
|
|
458
|
-
const status = await runnerRequest('GET', configured.endpoint, '/status', undefined, 'IOS_DEVICE_UI_RUNNER_UNAVAILABLE');
|
|
459
|
-
const record = objectValue(status);
|
|
460
|
-
return {
|
|
461
|
-
configured: true,
|
|
462
|
-
endpoint: configured.endpoint,
|
|
463
|
-
ready: record.ready !== false,
|
|
464
|
-
status: bounded(status),
|
|
465
|
-
};
|
|
466
|
-
} catch (error) {
|
|
467
|
-
return {
|
|
468
|
-
configured: true,
|
|
469
|
-
endpoint: configured.endpoint,
|
|
470
|
-
ready: false,
|
|
471
|
-
error: toAssistantPluginError(error, {
|
|
472
|
-
code: 'IOS_DEVICE_UI_RUNNER_UNAVAILABLE',
|
|
473
|
-
message: 'The local iOS UI runner is unavailable.',
|
|
474
|
-
retryable: true,
|
|
475
|
-
}).message,
|
|
476
|
-
};
|
|
477
|
-
}
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
async function createRunnerSession(endpoint: string, bundleId: string): Promise<string> {
|
|
481
|
-
const value = await runnerRequest('POST', endpoint, '/session', {
|
|
482
|
-
capabilities: { alwaysMatch: { bundleId } },
|
|
483
|
-
desiredCapabilities: { bundleId },
|
|
484
|
-
}, 'IOS_DEVICE_UI_SESSION_FAILED');
|
|
485
|
-
const record = objectValue(value);
|
|
486
|
-
const sessionId = stringField(record, 'sessionId') ?? stringField(objectValue(objectValue(value).capabilities), 'sessionId');
|
|
487
|
-
if (!sessionId) {
|
|
488
|
-
throw new AssistantPluginError('IOS_DEVICE_UI_SESSION_FAILED', 'The UI runner did not return a WebDriver session id.', { retryable: false });
|
|
489
|
-
}
|
|
490
|
-
return sessionId;
|
|
491
|
-
}
|
|
492
|
-
|
|
493
|
-
async function closeRunner(state: PhysicalSessionState): Promise<boolean> {
|
|
494
|
-
if (!state.runner.endpoint || !state.runner.sessionId) return true;
|
|
495
|
-
try {
|
|
496
|
-
await runnerRequest('DELETE', state.runner.endpoint, `/session/${encodeURIComponent(state.runner.sessionId)}`, undefined, 'IOS_DEVICE_UI_CLOSE_FAILED');
|
|
497
|
-
return true;
|
|
498
|
-
} catch {
|
|
499
|
-
return false;
|
|
500
|
-
}
|
|
638
|
+
return expired;
|
|
501
639
|
}
|
|
502
640
|
|
|
503
641
|
function requireRecord(input: AssistantPluginActionExecutionInput, allowTerminal = false): { record: InteractionSessionRecord; state: PhysicalSessionState } {
|
|
@@ -511,10 +649,7 @@ function requireRecord(input: AssistantPluginActionExecutionInput, allowTerminal
|
|
|
511
649
|
throw new AssistantPluginError(
|
|
512
650
|
'PLUGIN_ACTION_ARGUMENT_INVALID',
|
|
513
651
|
`Physical iOS interaction ${interactionId} belongs to ${engine ?? 'an unknown engine'}, not the CoreDevice engine.`,
|
|
514
|
-
{
|
|
515
|
-
retryable: false,
|
|
516
|
-
details: { interactionId, expectedEngine: 'coredevice', actualEngine: engine },
|
|
517
|
-
},
|
|
652
|
+
{ retryable: false, details: { interactionId, expectedEngine: 'coredevice', actualEngine: engine } },
|
|
518
653
|
);
|
|
519
654
|
}
|
|
520
655
|
const state = readState(input, interactionId);
|
|
@@ -523,8 +658,7 @@ function requireRecord(input: AssistantPluginActionExecutionInput, allowTerminal
|
|
|
523
658
|
}
|
|
524
659
|
if (!isInteractionSessionActive(record.status) && !allowTerminal) {
|
|
525
660
|
throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `Physical iOS interaction is ${record.status}.`, {
|
|
526
|
-
retryable: false,
|
|
527
|
-
details: { interactionId, status: record.status },
|
|
661
|
+
retryable: false, details: { interactionId, status: record.status },
|
|
528
662
|
});
|
|
529
663
|
}
|
|
530
664
|
if (isInteractionSessionActive(record.status) && hooks.now().getTime() >= Date.parse(record.expiresAt)) {
|
|
@@ -532,103 +666,343 @@ function requireRecord(input: AssistantPluginActionExecutionInput, allowTerminal
|
|
|
532
666
|
status: 'failed',
|
|
533
667
|
error: { code: 'IOS_DEVICE_SESSION_EXPIRED', message: 'The physical iOS interaction expired.' },
|
|
534
668
|
});
|
|
669
|
+
appendEvent(input, interactionId, 'session_expired');
|
|
670
|
+
stopRemoteXpcHidForDevice(state.device.identifier);
|
|
535
671
|
throw new AssistantPluginError('IOS_DEVICE_SESSION_EXPIRED', 'The physical iOS interaction expired; open a new session.', { retryable: false });
|
|
536
672
|
}
|
|
537
673
|
return { record, state };
|
|
538
674
|
}
|
|
539
675
|
|
|
540
|
-
function
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
676
|
+
function physicalDeviceStatusFromResult(result: CommandResult, platform: NodeJS.Platform): CoreDeviceStatus {
|
|
677
|
+
return {
|
|
678
|
+
available: result.ok,
|
|
679
|
+
platform,
|
|
680
|
+
coreDeviceReady: result.ok,
|
|
681
|
+
devicectlVersion: result.ok ? result.stdout.trim() : undefined,
|
|
682
|
+
reason: result.ok ? undefined : (result.stderr || result.stdout || 'xcrun devicectl is unavailable.'),
|
|
683
|
+
probed: true,
|
|
684
|
+
};
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
export function iosPhysicalDeviceStatus(): CoreDeviceStatus {
|
|
688
|
+
const platform = hooks.platform();
|
|
689
|
+
if (platform !== 'darwin') {
|
|
690
|
+
coreDeviceReadyCache = undefined;
|
|
691
|
+
return {
|
|
692
|
+
available: false,
|
|
693
|
+
platform,
|
|
694
|
+
coreDeviceReady: false,
|
|
695
|
+
reason: 'Physical iOS device support requires macOS and Xcode.',
|
|
696
|
+
probed: true,
|
|
697
|
+
};
|
|
698
|
+
}
|
|
699
|
+
const cached = coreDeviceReadyCache;
|
|
700
|
+
if (cached && hooks.now().getTime() - cached.checkedAtMs <= CORE_DEVICE_READY_CACHE_MS) return cached.status;
|
|
701
|
+
return {
|
|
702
|
+
available: false,
|
|
703
|
+
platform,
|
|
704
|
+
coreDeviceReady: false,
|
|
705
|
+
reason: 'CoreDevice readiness has not been probed by an explicit asynchronous action in this runtime yet.',
|
|
706
|
+
probed: false,
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
async function iosPhysicalDeviceActionStatus(input: AssistantPluginActionExecutionInput): Promise<CoreDeviceStatus> {
|
|
711
|
+
const platform = hooks.platform();
|
|
712
|
+
if (platform !== 'darwin') return iosPhysicalDeviceStatus();
|
|
713
|
+
const result = await hooks.runCommandAsync('xcrun', ['devicectl', '--version'], {
|
|
714
|
+
cwd: input.repoRoot,
|
|
715
|
+
timeoutMs: Math.min(5_000, input.timeoutMs ?? 5_000),
|
|
716
|
+
signal: input.signal,
|
|
717
|
+
});
|
|
718
|
+
const status = physicalDeviceStatusFromResult(result, platform);
|
|
719
|
+
coreDeviceReadyCache = status.coreDeviceReady ? { checkedAtMs: hooks.now().getTime(), status } : undefined;
|
|
720
|
+
return status;
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
async function cachedIosPhysicalDeviceActionStatus(input: AssistantPluginActionExecutionInput): Promise<{ status: CoreDeviceStatus; cached: boolean }> {
|
|
724
|
+
const cached = coreDeviceReadyCache;
|
|
725
|
+
if (cached && hooks.now().getTime() - cached.checkedAtMs <= CORE_DEVICE_READY_CACHE_MS) {
|
|
726
|
+
return { status: cached.status, cached: true };
|
|
553
727
|
}
|
|
554
|
-
return {
|
|
728
|
+
return { status: await iosPhysicalDeviceActionStatus(input), cached: false };
|
|
555
729
|
}
|
|
556
730
|
|
|
557
|
-
function
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
731
|
+
function recentScreenshotObservation(
|
|
732
|
+
state: PhysicalSessionState,
|
|
733
|
+
observationId: string,
|
|
734
|
+
): { observation: PhysicalScreenshotObservation; ageMs: number } {
|
|
735
|
+
const observation = state.lastScreenshot;
|
|
736
|
+
const ageMs = cacheAgeMs(observation?.observedAt);
|
|
737
|
+
if (!observation || observation.observationId !== observationId || ageMs === undefined || ageMs > INPUT_FOREGROUND_OBSERVATION_TTL_MS) {
|
|
738
|
+
throw new AssistantPluginError(
|
|
739
|
+
'IOS_DEVICE_SCREEN_OBSERVATION_REQUIRED',
|
|
740
|
+
'A fresh matching CoreDevice screenshot observation is required before screen-relative physical input.',
|
|
741
|
+
{
|
|
742
|
+
retryable: true,
|
|
743
|
+
details: {
|
|
744
|
+
deviceIdentifier: state.device.identifier,
|
|
745
|
+
expectedObservationId: observation?.observationId,
|
|
746
|
+
providedObservationId: observationId,
|
|
747
|
+
observationAgeMs: ageMs,
|
|
748
|
+
observationTtlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS,
|
|
749
|
+
hidMutationDispatched: false,
|
|
750
|
+
},
|
|
751
|
+
},
|
|
752
|
+
);
|
|
563
753
|
}
|
|
754
|
+
return { observation, ageMs };
|
|
564
755
|
}
|
|
565
756
|
|
|
566
|
-
function
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
757
|
+
function requireObservedPhysicalTargetForeground(state: PhysicalSessionState): PhysicalForegroundObservation {
|
|
758
|
+
const confirmation = state.foregroundObservation;
|
|
759
|
+
const screenshot = state.lastScreenshot;
|
|
760
|
+
const screenshotAgeMs = cacheAgeMs(screenshot?.observedAt);
|
|
761
|
+
const confirmationAgeMs = cacheAgeMs(confirmation?.confirmedAt);
|
|
762
|
+
const valid = Boolean(
|
|
763
|
+
confirmation
|
|
764
|
+
&& screenshot
|
|
765
|
+
&& confirmation.bundleId === state.bundleId
|
|
766
|
+
&& confirmation.observationId === screenshot.observationId
|
|
767
|
+
&& confirmation.screenshotObservedAt === screenshot.observedAt
|
|
768
|
+
&& screenshotAgeMs !== undefined
|
|
769
|
+
&& screenshotAgeMs <= INPUT_FOREGROUND_OBSERVATION_TTL_MS
|
|
770
|
+
&& confirmationAgeMs !== undefined
|
|
771
|
+
&& confirmationAgeMs <= INPUT_FOREGROUND_OBSERVATION_TTL_MS
|
|
772
|
+
);
|
|
773
|
+
if (!valid || !confirmation) {
|
|
774
|
+
throw new AssistantPluginError(
|
|
775
|
+
'IOS_DEVICE_FOREGROUND_OBSERVATION_REQUIRED',
|
|
776
|
+
`A recent screenshot explicitly confirmed by the Controller as ${state.bundleId} foreground is required before HID input.`,
|
|
777
|
+
{
|
|
778
|
+
retryable: true,
|
|
779
|
+
details: {
|
|
780
|
+
bundleId: state.bundleId,
|
|
781
|
+
deviceIdentifier: state.device.identifier,
|
|
782
|
+
screenshotObservationId: screenshot?.observationId,
|
|
783
|
+
screenshotAgeMs,
|
|
784
|
+
confirmationObservationId: confirmation?.observationId,
|
|
785
|
+
confirmationAgeMs,
|
|
786
|
+
observationTtlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS,
|
|
787
|
+
hidMutationDispatched: false,
|
|
788
|
+
},
|
|
789
|
+
},
|
|
596
790
|
);
|
|
597
|
-
if (typeof value === 'string') values.push(value);
|
|
598
791
|
}
|
|
599
|
-
|
|
792
|
+
return confirmation;
|
|
600
793
|
}
|
|
601
794
|
|
|
602
|
-
|
|
603
|
-
|
|
795
|
+
function resolvePhysicalBatchForegroundObservation(
|
|
796
|
+
state: PhysicalSessionState,
|
|
797
|
+
observationId: string | undefined,
|
|
798
|
+
): { foregroundObservation: PhysicalForegroundObservation; source: 'atomic_screenshot' | 'explicit_confirmation'; screenshotAgeMs?: number } {
|
|
799
|
+
if (observationId) {
|
|
800
|
+
const { observation, ageMs } = recentScreenshotObservation(state, observationId);
|
|
604
801
|
return {
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
802
|
+
foregroundObservation: {
|
|
803
|
+
observationId: observation.observationId,
|
|
804
|
+
screenshotObservedAt: observation.observedAt,
|
|
805
|
+
confirmedAt: timestamp(),
|
|
806
|
+
bundleId: state.bundleId,
|
|
807
|
+
},
|
|
808
|
+
source: 'atomic_screenshot',
|
|
809
|
+
screenshotAgeMs: ageMs,
|
|
610
810
|
};
|
|
611
811
|
}
|
|
612
|
-
const result = hooks.runCommand('xcrun', ['devicectl', '--version'], { timeoutMs: 5_000 });
|
|
613
812
|
return {
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
coreDeviceReady: result.ok,
|
|
617
|
-
devicectlVersion: result.ok ? result.stdout.trim() : undefined,
|
|
618
|
-
uiRunner: { ...configuredRunner(), ready: false, readinessCheckedOnAction: true },
|
|
619
|
-
reason: result.ok ? undefined : (result.stderr || result.stdout || 'xcrun devicectl is unavailable.'),
|
|
813
|
+
foregroundObservation: requireObservedPhysicalTargetForeground(state),
|
|
814
|
+
source: 'explicit_confirmation',
|
|
620
815
|
};
|
|
621
816
|
}
|
|
622
817
|
|
|
818
|
+
function consumePhysicalScreenObservation(input: AssistantPluginActionExecutionInput, state: PhysicalSessionState): void {
|
|
819
|
+
state.lastScreenshot = undefined;
|
|
820
|
+
state.foregroundObservation = undefined;
|
|
821
|
+
writeState(input, state);
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
type PhysicalInputBatchKind = 'tap' | 'swipe' | 'type' | 'wait';
|
|
825
|
+
|
|
826
|
+
interface PhysicalInputBatchStep {
|
|
827
|
+
kind: PhysicalInputBatchKind;
|
|
828
|
+
x?: number;
|
|
829
|
+
y?: number;
|
|
830
|
+
toX?: number;
|
|
831
|
+
toY?: number;
|
|
832
|
+
durationMs?: number;
|
|
833
|
+
text?: string;
|
|
834
|
+
textMode?: 'auto' | 'keys' | 'pasteboard';
|
|
835
|
+
replaceExisting?: boolean;
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
function physicalInputBatchSteps(value: unknown): PhysicalInputBatchStep[] {
|
|
839
|
+
if (!Array.isArray(value) || value.length === 0 || value.length > MAX_BATCH_STEPS) {
|
|
840
|
+
throw new AssistantPluginError(
|
|
841
|
+
'PLUGIN_ACTION_ARGUMENT_INVALID',
|
|
842
|
+
`steps must contain between 1 and ${MAX_BATCH_STEPS} physical input steps.`,
|
|
843
|
+
{ retryable: false },
|
|
844
|
+
);
|
|
845
|
+
}
|
|
846
|
+
return value.map((entry, index) => {
|
|
847
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry)) {
|
|
848
|
+
throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `steps[${index}] must be an object.`, { retryable: false });
|
|
849
|
+
}
|
|
850
|
+
const row = entry as Record<string, unknown>;
|
|
851
|
+
const kind = requireString(row.kind, `steps[${index}].kind`) as PhysicalInputBatchKind;
|
|
852
|
+
if (!['tap', 'swipe', 'type', 'wait'].includes(kind)) {
|
|
853
|
+
throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `Unsupported steps[${index}].kind: ${kind}.`, { retryable: false });
|
|
854
|
+
}
|
|
855
|
+
if (kind === 'tap') {
|
|
856
|
+
return {
|
|
857
|
+
kind,
|
|
858
|
+
x: requireFiniteNumber(row.x, `steps[${index}].x`),
|
|
859
|
+
y: requireFiniteNumber(row.y, `steps[${index}].y`),
|
|
860
|
+
};
|
|
861
|
+
}
|
|
862
|
+
if (kind === 'swipe') {
|
|
863
|
+
const durationMs = typeof row.duration_ms === 'number' && Number.isFinite(row.duration_ms)
|
|
864
|
+
? Math.max(80, Math.min(3_000, Math.trunc(row.duration_ms)))
|
|
865
|
+
: 250;
|
|
866
|
+
return {
|
|
867
|
+
kind,
|
|
868
|
+
x: requireFiniteNumber(row.x, `steps[${index}].x`),
|
|
869
|
+
y: requireFiniteNumber(row.y, `steps[${index}].y`),
|
|
870
|
+
toX: requireFiniteNumber(row.to_x, `steps[${index}].to_x`),
|
|
871
|
+
toY: requireFiniteNumber(row.to_y, `steps[${index}].to_y`),
|
|
872
|
+
durationMs,
|
|
873
|
+
};
|
|
874
|
+
}
|
|
875
|
+
if (kind === 'type') {
|
|
876
|
+
const textMode = optionalString(row.input_mode) ?? 'auto';
|
|
877
|
+
if (!['auto', 'keys', 'pasteboard'].includes(textMode)) {
|
|
878
|
+
throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `Unsupported steps[${index}].input_mode: ${textMode}.`, { retryable: false });
|
|
879
|
+
}
|
|
880
|
+
const text = requireString(row.text, `steps[${index}].text`);
|
|
881
|
+
if (text.length > 2048) {
|
|
882
|
+
throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `steps[${index}].text exceeds the 2048 character input limit.`, { retryable: false });
|
|
883
|
+
}
|
|
884
|
+
if (textMode === 'keys' && [...text].some((character) => character.charCodeAt(0) > 0x7f)) {
|
|
885
|
+
throw new AssistantPluginError('IOS_HID_UNICODE_TEXT_UNSUPPORTED', `steps[${index}] requests keys mode for Unicode text; use auto or pasteboard mode.`, { retryable: false });
|
|
886
|
+
}
|
|
887
|
+
return {
|
|
888
|
+
kind,
|
|
889
|
+
text,
|
|
890
|
+
textMode: textMode as 'auto' | 'keys' | 'pasteboard',
|
|
891
|
+
replaceExisting: row.replace_existing === true,
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
const durationMs = typeof row.duration_ms === 'number' && Number.isFinite(row.duration_ms)
|
|
895
|
+
? Math.max(0, Math.min(MAX_BATCH_WAIT_MS, Math.trunc(row.duration_ms)))
|
|
896
|
+
: 120;
|
|
897
|
+
return { kind, durationMs };
|
|
898
|
+
});
|
|
899
|
+
}
|
|
900
|
+
|
|
901
|
+
function validatePhysicalInputBatchCoordinates(steps: PhysicalInputBatchStep[], display: DisplayGeometry | undefined): void {
|
|
902
|
+
const touchSteps = steps.filter((step) => step.kind === 'tap' || step.kind === 'swipe');
|
|
903
|
+
if (touchSteps.length === 0) return;
|
|
904
|
+
if (!display) {
|
|
905
|
+
throw new AssistantPluginError('IOS_HID_DISPLAY_INVALID', 'Physical input batch contains touch steps without usable display geometry.', { retryable: true });
|
|
906
|
+
}
|
|
907
|
+
const points: Array<{ stepIndex: number; label: string; x: number | undefined; y: number | undefined }> = [];
|
|
908
|
+
steps.forEach((step, stepIndex) => {
|
|
909
|
+
if (step.kind === 'tap' || step.kind === 'swipe') points.push({ stepIndex, label: 'from', x: step.x, y: step.y });
|
|
910
|
+
if (step.kind === 'swipe') points.push({ stepIndex, label: 'to', x: step.toX, y: step.toY });
|
|
911
|
+
});
|
|
912
|
+
for (const point of points) {
|
|
913
|
+
if (point.x === undefined || point.y === undefined || point.x < 0 || point.y < 0 || point.x > display.width - 1 || point.y > display.height - 1) {
|
|
914
|
+
throw new AssistantPluginError(
|
|
915
|
+
'IOS_HID_COORDINATE_OUT_OF_BOUNDS',
|
|
916
|
+
`steps[${point.stepIndex}] ${point.label} coordinate is outside the current CoreDevice display.`,
|
|
917
|
+
{ retryable: false, details: { stepIndex: point.stepIndex, label: point.label, x: point.x, y: point.y, width: display.width, height: display.height, mutationDispatched: false } },
|
|
918
|
+
);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
|
|
923
|
+
async function waitPhysicalInputBatch(durationMs: number, signal?: AbortSignal): Promise<void> {
|
|
924
|
+
if (durationMs <= 0) return;
|
|
925
|
+
await new Promise<void>((resolve, reject) => {
|
|
926
|
+
let settled = false;
|
|
927
|
+
let timer: NodeJS.Timeout | undefined;
|
|
928
|
+
const finish = (error?: Error) => {
|
|
929
|
+
if (settled) return;
|
|
930
|
+
settled = true;
|
|
931
|
+
if (timer) clearTimeout(timer);
|
|
932
|
+
signal?.removeEventListener('abort', abort);
|
|
933
|
+
if (error) reject(error);
|
|
934
|
+
else resolve();
|
|
935
|
+
};
|
|
936
|
+
const abort = () => finish(new AssistantPluginError('PLUGIN_ACTION_CANCELLED', 'Physical iOS input batch was cancelled during a wait step.', { retryable: true }));
|
|
937
|
+
timer = setTimeout(() => finish(), durationMs);
|
|
938
|
+
if (signal?.aborted) abort();
|
|
939
|
+
else signal?.addEventListener('abort', abort, { once: true });
|
|
940
|
+
});
|
|
941
|
+
}
|
|
942
|
+
|
|
623
943
|
export function isIosPhysicalDeviceAction(actionId: string): boolean {
|
|
624
944
|
return actionId.startsWith('physical_device_');
|
|
625
945
|
}
|
|
626
946
|
|
|
947
|
+
function physicalDeviceAuthorizationTarget(device: PhysicalDevice): AssistantPluginAuthorizationTarget {
|
|
948
|
+
const identityFingerprint = createHash('sha256')
|
|
949
|
+
.update(`${device.identifier}\0${device.udid ?? ''}`)
|
|
950
|
+
.digest('hex');
|
|
951
|
+
return {
|
|
952
|
+
kind: 'ios-physical-device',
|
|
953
|
+
id: device.identifier,
|
|
954
|
+
identityFingerprint,
|
|
955
|
+
};
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
/**
|
|
959
|
+
* Resolve authorization against the exact CoreDevice identity. Interaction ids are
|
|
960
|
+
* intentionally never grant identities: follow-up input actions must recover and
|
|
961
|
+
* verify the immutable device target stored when the session was opened.
|
|
962
|
+
*/
|
|
963
|
+
export async function resolveIosPhysicalDeviceAuthorizationContext(
|
|
964
|
+
input: AssistantPluginActionExecutionInput,
|
|
965
|
+
): Promise<AssistantPluginAuthorizationContext | undefined> {
|
|
966
|
+
const action = iosPhysicalDeviceActions().find((entry) => entry.actionId === input.actionId);
|
|
967
|
+
if (!action || action.confirmation !== 'authorization') return undefined;
|
|
968
|
+
|
|
969
|
+
if (input.actionId === 'physical_device_open') {
|
|
970
|
+
const selected = await selectDevice(input, input.args.device);
|
|
971
|
+
return {
|
|
972
|
+
target: physicalDeviceAuthorizationTarget(selected),
|
|
973
|
+
expiresInMinutes: 30 * 24 * 60,
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
|
|
977
|
+
const { record, state } = requireRecord(input, input.actionId === 'physical_device_close');
|
|
978
|
+
const aliases = new Set([record.targetId, ...(record.targetAliases ?? [])]);
|
|
979
|
+
if (record.targetId !== state.device.identifier
|
|
980
|
+
|| !aliases.has(state.device.identifier)
|
|
981
|
+
|| (state.device.udid && !aliases.has(state.device.udid))) {
|
|
982
|
+
throw new AssistantPluginError(
|
|
983
|
+
'IOS_DEVICE_AUTHORIZATION_TARGET_MISMATCH',
|
|
984
|
+
'The persisted physical iOS session no longer matches its exact device authorization identity.',
|
|
985
|
+
{
|
|
986
|
+
retryable: false,
|
|
987
|
+
details: {
|
|
988
|
+
interactionId: record.interactionId,
|
|
989
|
+
sessionTargetId: record.targetId,
|
|
990
|
+
stateDeviceIdentifier: state.device.identifier,
|
|
991
|
+
},
|
|
992
|
+
},
|
|
993
|
+
);
|
|
994
|
+
}
|
|
995
|
+
return {
|
|
996
|
+
target: physicalDeviceAuthorizationTarget(state.device),
|
|
997
|
+
expiresInMinutes: 30 * 24 * 60,
|
|
998
|
+
};
|
|
999
|
+
}
|
|
1000
|
+
|
|
627
1001
|
export function iosPhysicalDeviceCapabilities(): AssistantPluginCapability[] {
|
|
628
1002
|
return [{
|
|
629
1003
|
capabilityId: 'ios-physical-device',
|
|
630
1004
|
title: 'Physical iOS Computer Use',
|
|
631
|
-
description: '
|
|
1005
|
+
description: 'CoreDevice lifecycle/observation plus runnerless RemoteXPC HID input for physical iPhones. XCTest remains only an explicit semantic fallback.',
|
|
632
1006
|
scopes: ['ios.discover', 'ios.device'],
|
|
633
1007
|
actions: iosPhysicalDeviceActions().map((action) => action.actionId),
|
|
634
1008
|
}];
|
|
@@ -641,10 +1015,10 @@ export function iosPhysicalDeviceActions(): AssistantPluginActionDescriptor[] {
|
|
|
641
1015
|
{ resource: 'repo-state' as const, mode: 'write' as const },
|
|
642
1016
|
];
|
|
643
1017
|
const interactionProperty = { interaction_id: { type: 'string' } };
|
|
644
|
-
|
|
1018
|
+
const actions: AssistantPluginActionDescriptor[] = [
|
|
645
1019
|
{
|
|
646
1020
|
actionId: 'physical_device_status', title: 'Physical iOS device status',
|
|
647
|
-
description: 'Report CoreDevice readiness
|
|
1021
|
+
description: 'Report CoreDevice readiness. This provider has no XCTest/WDA Runner lifecycle.',
|
|
648
1022
|
readOnly: true, risk: 'readonly', confirmation: 'none', defaultTimeoutMs: 30_000, cancellable: true, idempotent: true,
|
|
649
1023
|
scopes: ['ios.discover'], resourceClaims: [], argumentsSchema: { type: 'object', properties: {}, additionalProperties: false },
|
|
650
1024
|
},
|
|
@@ -654,6 +1028,13 @@ export function iosPhysicalDeviceActions(): AssistantPluginActionDescriptor[] {
|
|
|
654
1028
|
readOnly: true, risk: 'readonly', confirmation: 'none', defaultTimeoutMs: 30_000, cancellable: true, idempotent: true,
|
|
655
1029
|
scopes: ['ios.discover'], resourceClaims: [], argumentsSchema: { type: 'object', properties: {}, additionalProperties: false },
|
|
656
1030
|
},
|
|
1031
|
+
{
|
|
1032
|
+
actionId: 'physical_device_info', title: 'Inspect physical iPhone capabilities',
|
|
1033
|
+
description: 'Read CoreDevice details, lock state, display metadata, View Device Screen URL, and HID capability availability without starting a Runner.',
|
|
1034
|
+
readOnly: true, risk: 'readonly', confirmation: 'none', defaultTimeoutMs: 90_000, cancellable: true, idempotent: true,
|
|
1035
|
+
scopes: ['ios.device'], resourceClaims: [],
|
|
1036
|
+
argumentsSchema: { type: 'object', properties: { device: { type: 'string' } }, required: ['device'], additionalProperties: false },
|
|
1037
|
+
},
|
|
657
1038
|
{
|
|
658
1039
|
actionId: 'physical_device_apps', title: 'Find installed physical-device app',
|
|
659
1040
|
description: 'Verify one exact bundle identifier is installed on an exact paired iPhone without reading its data container.',
|
|
@@ -663,45 +1044,107 @@ export function iosPhysicalDeviceActions(): AssistantPluginActionDescriptor[] {
|
|
|
663
1044
|
},
|
|
664
1045
|
{
|
|
665
1046
|
actionId: 'physical_device_open', title: 'Open physical iOS app session',
|
|
666
|
-
description: '
|
|
1047
|
+
description: 'Request activation of one installed third-party app through CoreDevice and create a bounded interaction session. CoreDevice launch metadata is diagnostic only: capture a screenshot and explicitly confirm the observed foreground before any HID input. Set prewarm_input=true when runnerless input will follow; Forge then waits up to 4 seconds for a non-mutating RemoteXPC readiness result so the first HID action does not need a blind retry.',
|
|
667
1048
|
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 2 * 60_000, cancellable: true, idempotent: false,
|
|
668
1049
|
scopes: ['ios.device'], resourceClaims: mutationClaims,
|
|
669
|
-
argumentsSchema: { type: 'object', properties: { device: { type: 'string' }, bundle_id: { type: 'string' }, relaunch: { type: 'boolean' } }, required: ['device', 'bundle_id'], additionalProperties: false },
|
|
1050
|
+
argumentsSchema: { type: 'object', properties: { device: { type: 'string' }, bundle_id: { type: 'string' }, relaunch: { type: 'boolean' }, prewarm_input: { type: 'boolean' } }, required: ['device', 'bundle_id'], additionalProperties: false },
|
|
670
1051
|
},
|
|
671
1052
|
{
|
|
672
|
-
actionId: '
|
|
673
|
-
description: '
|
|
674
|
-
readOnly:
|
|
1053
|
+
actionId: 'physical_device_screenshot', title: 'Capture physical iOS screenshot',
|
|
1054
|
+
description: 'Capture the exact paired iPhone display through CoreDevice into Controller-owned bounded artifact storage and return a short-lived observation ID. A screenshot alone never authorizes input.',
|
|
1055
|
+
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: false,
|
|
1056
|
+
scopes: ['ios.device'], resourceClaims: mutationClaims,
|
|
1057
|
+
argumentsSchema: { type: 'object', properties: { ...interactionProperty, label: { type: 'string' } }, required: ['interaction_id'], additionalProperties: false },
|
|
1058
|
+
},
|
|
1059
|
+
{
|
|
1060
|
+
actionId: 'physical_device_confirm_foreground', title: 'Confirm observed physical iPhone foreground',
|
|
1061
|
+
description: 'Bind the latest fresh CoreDevice screenshot observation to the session target bundle after the Controller has visually verified that exact app is foreground. This only writes the short-lived observation fence and sends no device input.',
|
|
1062
|
+
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: true,
|
|
675
1063
|
scopes: ['ios.device'], resourceClaims: stateClaim,
|
|
676
|
-
argumentsSchema: {
|
|
1064
|
+
argumentsSchema: {
|
|
1065
|
+
type: 'object',
|
|
1066
|
+
properties: { ...interactionProperty, observation_id: { type: 'string' }, bundle_id: { type: 'string' } },
|
|
1067
|
+
required: ['interaction_id', 'observation_id', 'bundle_id'], additionalProperties: false,
|
|
1068
|
+
},
|
|
677
1069
|
},
|
|
678
1070
|
{
|
|
679
|
-
actionId: '
|
|
680
|
-
description: '
|
|
681
|
-
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs:
|
|
1071
|
+
actionId: 'physical_device_observed_tap', title: 'Tap a freshly observed physical iPhone screen',
|
|
1072
|
+
description: 'One-shot screen-relative tap bound to the latest fresh screenshot observation. It does not reactivate any app and consumes the observation before dispatch, so it can safely navigate system UI such as Spotlight into the target app when CoreDevice activation did not actually foreground it.',
|
|
1073
|
+
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: false,
|
|
682
1074
|
scopes: ['ios.device'], resourceClaims: mutationClaims,
|
|
683
|
-
argumentsSchema: {
|
|
1075
|
+
argumentsSchema: {
|
|
1076
|
+
type: 'object',
|
|
1077
|
+
properties: { ...interactionProperty, observation_id: { type: 'string' }, x: { type: 'number', minimum: 0 }, y: { type: 'number', minimum: 0 } },
|
|
1078
|
+
required: ['interaction_id', 'observation_id', 'x', 'y'], additionalProperties: false,
|
|
1079
|
+
},
|
|
684
1080
|
},
|
|
685
1081
|
{
|
|
686
|
-
actionId: '
|
|
687
|
-
description: '
|
|
1082
|
+
actionId: 'physical_device_batch', title: 'Run fast physical iPhone input batch',
|
|
1083
|
+
description: 'Preferred fast path for a visually confirmed target app: perform up to 20 tap, swipe, text, or bounded-wait steps under one fresh screenshot foreground fence, one unlock/display readiness pass, and one plugin round-trip. Pass observation_id to atomically bind and consume the latest visually verified screenshot without a separate confirm_foreground round-trip; omitting it preserves the explicit confirm_foreground compatibility path. Stops on the first failed step and never replays a partially-mutated batch.',
|
|
688
1084
|
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: false,
|
|
689
1085
|
scopes: ['ios.device'], resourceClaims: mutationClaims,
|
|
690
|
-
argumentsSchema: {
|
|
1086
|
+
argumentsSchema: {
|
|
1087
|
+
type: 'object',
|
|
1088
|
+
properties: {
|
|
1089
|
+
...interactionProperty,
|
|
1090
|
+
observation_id: { type: 'string' },
|
|
1091
|
+
steps: {
|
|
1092
|
+
type: 'array', minItems: 1, maxItems: MAX_BATCH_STEPS,
|
|
1093
|
+
items: {
|
|
1094
|
+
type: 'object',
|
|
1095
|
+
properties: {
|
|
1096
|
+
kind: { type: 'string', enum: ['tap', 'swipe', 'type', 'wait'] },
|
|
1097
|
+
x: { type: 'number', minimum: 0 }, y: { type: 'number', minimum: 0 },
|
|
1098
|
+
to_x: { type: 'number', minimum: 0 }, to_y: { type: 'number', minimum: 0 },
|
|
1099
|
+
duration_ms: { type: 'number', minimum: 0, maximum: MAX_BATCH_WAIT_MS },
|
|
1100
|
+
text: { type: 'string', maxLength: 2048 },
|
|
1101
|
+
input_mode: { type: 'string', enum: ['auto', 'keys', 'pasteboard'] },
|
|
1102
|
+
replace_existing: { type: 'boolean' },
|
|
1103
|
+
},
|
|
1104
|
+
required: ['kind'], additionalProperties: false,
|
|
1105
|
+
},
|
|
1106
|
+
},
|
|
1107
|
+
},
|
|
1108
|
+
required: ['interaction_id', 'steps'], additionalProperties: false,
|
|
1109
|
+
},
|
|
691
1110
|
},
|
|
692
1111
|
{
|
|
693
|
-
actionId: '
|
|
694
|
-
description: '
|
|
695
|
-
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs:
|
|
1112
|
+
actionId: 'physical_device_tap', title: 'Tap physical iPhone coordinate',
|
|
1113
|
+
description: 'Send one runnerless touch through RemoteXPC only while a fresh screenshot observation has been explicitly confirmed as the target app foreground. The observation is consumed before dispatch. For system-screen navigation use physical_device_observed_tap; for known form steps prefer physical_device_batch.',
|
|
1114
|
+
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: false,
|
|
696
1115
|
scopes: ['ios.device'], resourceClaims: mutationClaims,
|
|
697
|
-
argumentsSchema: {
|
|
1116
|
+
argumentsSchema: {
|
|
1117
|
+
type: 'object',
|
|
1118
|
+
properties: { ...interactionProperty, x: { type: 'number', minimum: 0 }, y: { type: 'number', minimum: 0 } },
|
|
1119
|
+
required: ['interaction_id', 'x', 'y'], additionalProperties: false,
|
|
1120
|
+
},
|
|
698
1121
|
},
|
|
699
1122
|
{
|
|
700
|
-
actionId: '
|
|
701
|
-
description: '
|
|
702
|
-
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs:
|
|
1123
|
+
actionId: 'physical_device_swipe', title: 'Swipe physical iPhone',
|
|
1124
|
+
description: 'Send one runnerless touchscreen swipe through RemoteXPC HID only while a fresh screenshot observation has been explicitly confirmed as the target app foreground; the observation is consumed before dispatch.',
|
|
1125
|
+
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: false,
|
|
703
1126
|
scopes: ['ios.device'], resourceClaims: mutationClaims,
|
|
704
|
-
argumentsSchema: {
|
|
1127
|
+
argumentsSchema: {
|
|
1128
|
+
type: 'object',
|
|
1129
|
+
properties: {
|
|
1130
|
+
...interactionProperty,
|
|
1131
|
+
x: { type: 'number', minimum: 0 }, y: { type: 'number', minimum: 0 },
|
|
1132
|
+
to_x: { type: 'number', minimum: 0 }, to_y: { type: 'number', minimum: 0 },
|
|
1133
|
+
duration_ms: { type: 'number', minimum: 80, maximum: 3000 },
|
|
1134
|
+
},
|
|
1135
|
+
required: ['interaction_id', 'x', 'y', 'to_x', 'to_y'], additionalProperties: false,
|
|
1136
|
+
},
|
|
1137
|
+
},
|
|
1138
|
+
{
|
|
1139
|
+
actionId: 'physical_device_type_text', title: 'Type physical iPhone text',
|
|
1140
|
+
description: 'Type bounded text through reusable runnerless RemoteXPC input only while a fresh screenshot observation has been explicitly confirmed as the target app foreground. Auto mode uses direct HID keys for short ASCII and clipboard-preserving CoreDevice pasteboard + Command-V for Unicode; the observation is consumed before dispatch.',
|
|
1141
|
+
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 30_000, cancellable: true, idempotent: false,
|
|
1142
|
+
scopes: ['ios.device'], resourceClaims: mutationClaims,
|
|
1143
|
+
argumentsSchema: {
|
|
1144
|
+
type: 'object',
|
|
1145
|
+
properties: { ...interactionProperty, text: { type: 'string', maxLength: 2048 }, input_mode: { type: 'string', enum: ['auto', 'keys', 'pasteboard'] }, replace_existing: { type: 'boolean' } },
|
|
1146
|
+
required: ['interaction_id', 'text'], additionalProperties: false,
|
|
1147
|
+
},
|
|
705
1148
|
},
|
|
706
1149
|
{
|
|
707
1150
|
actionId: 'physical_device_events', title: 'Read physical iOS events',
|
|
@@ -712,40 +1155,68 @@ export function iosPhysicalDeviceActions(): AssistantPluginActionDescriptor[] {
|
|
|
712
1155
|
},
|
|
713
1156
|
{
|
|
714
1157
|
actionId: 'physical_device_close', title: 'Close physical iOS session',
|
|
715
|
-
description: '
|
|
1158
|
+
description: 'Release Controller ownership without shutting down or modifying the iPhone. No XCTest/WDA Runner is owned by this provider.',
|
|
716
1159
|
readOnly: false, risk: 'workspace_write', confirmation: 'authorization', defaultTimeoutMs: 60_000, cancellable: true, idempotent: true,
|
|
717
1160
|
scopes: ['ios.device'], resourceClaims: mutationClaims,
|
|
718
1161
|
argumentsSchema: { type: 'object', properties: { ...interactionProperty }, required: ['interaction_id'], additionalProperties: false },
|
|
719
1162
|
},
|
|
720
1163
|
];
|
|
1164
|
+
return actions.map((action) => action.readOnly || action.actionId === 'physical_device_confirm_foreground'
|
|
1165
|
+
? action
|
|
1166
|
+
: { ...action, executionMode: 'lightweight_process' });
|
|
721
1167
|
}
|
|
722
1168
|
|
|
723
1169
|
export async function executeIosPhysicalDeviceAction(input: AssistantPluginActionExecutionInput): Promise<Record<string, unknown>> {
|
|
1170
|
+
const actionStartedAt = performance.now();
|
|
1171
|
+
const timingStages: TimingStages = {};
|
|
724
1172
|
if (input.actionId === 'physical_device_status') {
|
|
725
|
-
|
|
1173
|
+
const startedAt = performance.now();
|
|
1174
|
+
const status = await iosPhysicalDeviceActionStatus(input);
|
|
1175
|
+
recordTiming(timingStages, 'coreDeviceStatus', startedAt, false);
|
|
1176
|
+
return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', ...status });
|
|
726
1177
|
}
|
|
727
|
-
const
|
|
1178
|
+
const statusStartedAt = performance.now();
|
|
1179
|
+
const readiness = await cachedIosPhysicalDeviceActionStatus(input);
|
|
1180
|
+
recordTiming(timingStages, 'coreDeviceStatus', statusStartedAt, readiness.cached);
|
|
1181
|
+
const status = readiness.status;
|
|
728
1182
|
if (!status.coreDeviceReady) {
|
|
729
|
-
throw new AssistantPluginError('PLUGIN_DEPENDENCY_MISSING', status.reason ?? 'Xcode CoreDevice is unavailable.', { retryable: false, details: status });
|
|
1183
|
+
throw new AssistantPluginError('PLUGIN_DEPENDENCY_MISSING', status.reason ?? 'Xcode CoreDevice is unavailable.', { retryable: false, details: { ...status } });
|
|
730
1184
|
}
|
|
731
1185
|
if (input.actionId === 'physical_device_list') {
|
|
732
|
-
return { provider: 'coredevice', devices: physicalDevices(input) };
|
|
1186
|
+
return { provider: 'coredevice', devices: await physicalDevices(input) };
|
|
1187
|
+
}
|
|
1188
|
+
if (input.actionId === 'physical_device_info') {
|
|
1189
|
+
const selected = await selectDevice(input, input.args.device);
|
|
1190
|
+
return { provider: 'coredevice', ...await physicalDeviceInfo(input, selected) };
|
|
733
1191
|
}
|
|
734
1192
|
if (input.actionId === 'physical_device_apps') {
|
|
735
|
-
const selected = selectDevice(input, input.args.device);
|
|
1193
|
+
const selected = await selectDevice(input, input.args.device);
|
|
736
1194
|
const bundleId = requireBundleId(input.args.bundle_id);
|
|
737
|
-
return { provider: 'coredevice', device: selected, apps: installedApps(input, selected, bundleId) };
|
|
1195
|
+
return { provider: 'coredevice', device: selected, apps: await installedApps(input, selected, bundleId) };
|
|
738
1196
|
}
|
|
739
1197
|
if (input.actionId === 'physical_device_open') {
|
|
740
|
-
const
|
|
1198
|
+
const selectStartedAt = performance.now();
|
|
1199
|
+
const selected = await selectDevice(input, input.args.device);
|
|
1200
|
+
recordTiming(timingStages, 'deviceSelection', selectStartedAt, false);
|
|
741
1201
|
const bundleId = requireBundleId(input.args.bundle_id);
|
|
742
|
-
const
|
|
1202
|
+
const appsStartedAt = performance.now();
|
|
1203
|
+
const lockStartedAt = performance.now();
|
|
1204
|
+
const appsPromise = installedApps(input, selected, bundleId)
|
|
1205
|
+
.finally(() => recordTiming(timingStages, 'installedAppLookup', appsStartedAt, false));
|
|
1206
|
+
const lockPromise = requirePhysicalDeviceUnlocked(input, selected)
|
|
1207
|
+
.finally(() => recordTiming(timingStages, 'lockState', lockStartedAt, false));
|
|
1208
|
+
const [apps] = await Promise.all([appsPromise, lockPromise]);
|
|
743
1209
|
if (apps.length !== 1) {
|
|
744
1210
|
throw new AssistantPluginError('IOS_DEVICE_APP_NOT_INSTALLED', `The app ${bundleId} is not installed on the selected iPhone.`, { retryable: false });
|
|
745
1211
|
}
|
|
746
1212
|
const selectedAliases = [selected.identifier, selected.udid];
|
|
1213
|
+
const sessionLookupStartedAt = performance.now();
|
|
1214
|
+
const expiredSessionCount = expirePhysicalDeviceSessions(input);
|
|
1215
|
+
pruneInteractionSessions(input.repoRoot, PROVIDER, 100);
|
|
747
1216
|
const conflict = listInteractionSessions(input.repoRoot, PROVIDER).find((entry) =>
|
|
748
1217
|
isInteractionSessionActive(entry.status) && interactionMayOwnTarget(entry, selectedAliases));
|
|
1218
|
+
recordTiming(timingStages, 'sessionLookup', sessionLookupStartedAt, false);
|
|
1219
|
+
if (expiredSessionCount > 0) timingStages.sessionReconcile = { ms: timingStages.sessionLookup.ms, cached: false };
|
|
749
1220
|
if (conflict) {
|
|
750
1221
|
throw new AssistantPluginError('PLUGIN_RESOURCE_BUSY', 'The selected iPhone already has an active forge interaction.', {
|
|
751
1222
|
retryable: true,
|
|
@@ -757,7 +1228,6 @@ export async function executeIosPhysicalDeviceAction(input: AssistantPluginActio
|
|
|
757
1228
|
},
|
|
758
1229
|
});
|
|
759
1230
|
}
|
|
760
|
-
pruneInteractionSessions(input.repoRoot, PROVIDER, 100);
|
|
761
1231
|
const interactionId = `ios_device_${randomUUID()}`;
|
|
762
1232
|
const createdAt = timestamp();
|
|
763
1233
|
const record: InteractionSessionRecord = {
|
|
@@ -777,55 +1247,93 @@ export async function executeIosPhysicalDeviceAction(input: AssistantPluginActio
|
|
|
777
1247
|
expiresAt: new Date(hooks.now().getTime() + SESSION_EXPIRY_MS).toISOString(),
|
|
778
1248
|
};
|
|
779
1249
|
writeInteractionSession(input.repoRoot, record);
|
|
780
|
-
const runnerProbe = await probeRunner();
|
|
781
1250
|
const state: PhysicalSessionState = {
|
|
782
1251
|
schemaVersion: 1,
|
|
783
1252
|
interactionId,
|
|
784
1253
|
device: selected,
|
|
785
1254
|
bundleId,
|
|
786
|
-
|
|
787
|
-
configured: runnerProbe.configured,
|
|
788
|
-
endpoint: runnerProbe.endpoint,
|
|
789
|
-
ready: false,
|
|
790
|
-
error: runnerProbe.error,
|
|
791
|
-
},
|
|
1255
|
+
unlockVerifiedAt: createdAt,
|
|
792
1256
|
events: [{ at: createdAt, type: 'session_created', details: { bundleId, deviceIdentifier: selected.identifier } }],
|
|
793
1257
|
};
|
|
1258
|
+
const statePersistStartedAt = performance.now();
|
|
794
1259
|
writeState(input, state);
|
|
1260
|
+
recordTiming(timingStages, 'eventPersistence', statePersistStartedAt, false);
|
|
795
1261
|
try {
|
|
796
|
-
const args = ['device', 'process', 'launch', '--device', selected.identifier];
|
|
1262
|
+
const args = ['device', 'process', 'launch', '--device', selected.identifier, '--activate'];
|
|
797
1263
|
if (input.args.relaunch === true) args.push('--terminate-existing');
|
|
798
1264
|
args.push(bundleId);
|
|
799
|
-
const
|
|
1265
|
+
const prewarmStartedAt = performance.now();
|
|
1266
|
+
const prewarmRequested = input.args.prewarm_input === true && Boolean(selected.udid);
|
|
1267
|
+
const inputPrewarm: Record<string, unknown> = prewarmRequested && selected.udid
|
|
1268
|
+
? prewarmRemoteXpcHid({
|
|
1269
|
+
controllerHome: input.controllerHome,
|
|
1270
|
+
deviceIdentifier: selected.identifier,
|
|
1271
|
+
udid: selected.udid,
|
|
1272
|
+
})
|
|
1273
|
+
: { backend: 'remote-xpc-hid', state: 'not_requested', runnerOwned: false };
|
|
1274
|
+
recordTiming(
|
|
1275
|
+
timingStages,
|
|
1276
|
+
'hidPrewarm',
|
|
1277
|
+
prewarmStartedAt,
|
|
1278
|
+
inputPrewarm.state === 'ready' || inputPrewarm.state === 'test' || input.args.prewarm_input !== true,
|
|
1279
|
+
);
|
|
1280
|
+
const displayStartedAt = performance.now();
|
|
1281
|
+
const display = input.args.prewarm_input === true
|
|
1282
|
+
? await physicalDisplayGeometry(input, selected).then((result) => {
|
|
1283
|
+
recordTiming(timingStages, 'displayInfo', displayStartedAt, false);
|
|
1284
|
+
return result;
|
|
1285
|
+
}, (error) => {
|
|
1286
|
+
recordTiming(timingStages, 'displayInfo', displayStartedAt, false);
|
|
1287
|
+
throw error;
|
|
1288
|
+
})
|
|
1289
|
+
: undefined;
|
|
1290
|
+
if (input.args.prewarm_input !== true) recordTiming(timingStages, 'displayInfo', displayStartedAt, true);
|
|
1291
|
+
if (display) {
|
|
1292
|
+
state.display = { ...display, observedAt: createdAt };
|
|
1293
|
+
writeState(input, state);
|
|
1294
|
+
}
|
|
1295
|
+
const launchStartedAt = performance.now();
|
|
1296
|
+
const launchPromise = runCoreJson(input, args, 'IOS_DEVICE_LAUNCH_FAILED', 60_000)
|
|
1297
|
+
.finally(() => recordTiming(timingStages, 'activationRequest', launchStartedAt, false));
|
|
1298
|
+
const launch = await launchPromise;
|
|
1299
|
+
const launchEventStartedAt = performance.now();
|
|
800
1300
|
appendEvent(input, interactionId, 'app_launched', { bundleId, relaunch: input.args.relaunch === true });
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
state.
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
}
|
|
809
|
-
state.runner.ready = false;
|
|
810
|
-
state.runner.error = toAssistantPluginError(error, {
|
|
811
|
-
code: 'IOS_DEVICE_UI_SESSION_FAILED',
|
|
812
|
-
message: 'The app launched, but the UI runner session could not be created.',
|
|
813
|
-
retryable: true,
|
|
814
|
-
}).message;
|
|
815
|
-
writeState(input, state);
|
|
816
|
-
appendEvent(input, interactionId, 'ui_runner_unavailable', { error: state.runner.error });
|
|
817
|
-
}
|
|
1301
|
+
recordTiming(timingStages, 'eventPersistence', launchEventStartedAt, false);
|
|
1302
|
+
if (input.args.prewarm_input === true) {
|
|
1303
|
+
appendEvent(input, interactionId, 'input_prewarm', {
|
|
1304
|
+
state: inputPrewarm.state,
|
|
1305
|
+
...(typeof inputPrewarm.waitMs === 'number' ? { waitMs: inputPrewarm.waitMs } : {}),
|
|
1306
|
+
...(typeof inputPrewarm.errorCode === 'string' ? { errorCode: inputPrewarm.errorCode } : {}),
|
|
1307
|
+
...(typeof inputPrewarm.phase === 'string' ? { phase: inputPrewarm.phase } : {}),
|
|
1308
|
+
});
|
|
818
1309
|
}
|
|
1310
|
+
// CoreDevice is the default physical-device substrate. Never probe, start,
|
|
1311
|
+
// build, install, or attach an XCTest/WDA runner during ordinary app open.
|
|
1312
|
+
// Semantic automation is an explicit opt-in action so repeated computer-use
|
|
1313
|
+
// workflows cannot accidentally create fresh Runner lifecycles.
|
|
819
1314
|
const active = patchInteractionSession(input.repoRoot, PROVIDER, interactionId, { status: 'waiting_for_user' }) ?? record;
|
|
820
|
-
return {
|
|
1315
|
+
return finishTimedResult(actionStartedAt, timingStages, {
|
|
821
1316
|
provider: 'coredevice',
|
|
822
1317
|
interaction: active,
|
|
823
1318
|
device: selected,
|
|
824
1319
|
app: apps[0],
|
|
825
1320
|
launch: bounded(launch),
|
|
826
|
-
|
|
827
|
-
|
|
1321
|
+
foregroundVerification: {
|
|
1322
|
+
authority: 'screenshot_observation',
|
|
1323
|
+
required: true,
|
|
1324
|
+
coreDeviceActivatedWhenStartedHint: objectValue(objectValue(launch.result).launchOptions).activatedWhenStarted === true,
|
|
1325
|
+
},
|
|
1326
|
+
inputPrewarm,
|
|
1327
|
+
controlPlane: {
|
|
1328
|
+
lifecycle: 'coredevice',
|
|
1329
|
+
observation: 'coredevice',
|
|
1330
|
+
input: 'remote-xpc-hid',
|
|
1331
|
+
semanticFallback: 'agent-device',
|
|
1332
|
+
runnerOwned: false,
|
|
1333
|
+
},
|
|
1334
|
+
});
|
|
828
1335
|
} catch (error) {
|
|
1336
|
+
stopRemoteXpcHidForDevice(selected.identifier);
|
|
829
1337
|
const normalized = toAssistantPluginError(error, {
|
|
830
1338
|
code: 'IOS_DEVICE_OPEN_FAILED',
|
|
831
1339
|
message: 'The physical iOS app could not be opened.',
|
|
@@ -840,114 +1348,357 @@ export async function executeIosPhysicalDeviceAction(input: AssistantPluginActio
|
|
|
840
1348
|
}
|
|
841
1349
|
}
|
|
842
1350
|
|
|
1351
|
+
const sessionLookupStartedAt = performance.now();
|
|
843
1352
|
const { record, state } = requireRecord(input, input.actionId === 'physical_device_close');
|
|
1353
|
+
recordTiming(timingStages, 'sessionLookup', sessionLookupStartedAt, false);
|
|
844
1354
|
switch (input.actionId) {
|
|
845
|
-
case 'physical_device_snapshot': {
|
|
846
|
-
const runner = requireRunner(state);
|
|
847
|
-
const value = await runnerRequest('GET', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/source`, undefined, 'IOS_DEVICE_SNAPSHOT_FAILED');
|
|
848
|
-
appendEvent(input, record.interactionId, 'snapshot');
|
|
849
|
-
return { provider: 'coredevice', interaction: record, source: bounded(value) };
|
|
850
|
-
}
|
|
851
|
-
case 'physical_device_press': {
|
|
852
|
-
const runner = requireRunner(state);
|
|
853
|
-
const target = optionalString(input.args.target);
|
|
854
|
-
const hasPoint = typeof input.args.x === 'number' && typeof input.args.y === 'number';
|
|
855
|
-
if (!target && !hasPoint) throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', 'target or x/y is required.', { retryable: false });
|
|
856
|
-
if (target) {
|
|
857
|
-
rejectSensitiveSemantic(target, 'Press');
|
|
858
|
-
const id = await elementId(runner.endpoint, runner.sessionId, target);
|
|
859
|
-
await inspectElementSemantics(runner.endpoint, runner.sessionId, id, target, 'Press');
|
|
860
|
-
await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/element/${encodeURIComponent(id)}/click`, {}, 'IOS_DEVICE_PRESS_FAILED');
|
|
861
|
-
appendEvent(input, record.interactionId, 'press', { target });
|
|
862
|
-
} else {
|
|
863
|
-
const purpose = requireString(input.args.purpose, 'purpose');
|
|
864
|
-
rejectSensitiveSemantic(purpose, 'Coordinate press');
|
|
865
|
-
await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/wda/tap`, {
|
|
866
|
-
x: Number(input.args.x), y: Number(input.args.y),
|
|
867
|
-
}, 'IOS_DEVICE_PRESS_FAILED');
|
|
868
|
-
appendEvent(input, record.interactionId, 'press_coordinate', { x: input.args.x, y: input.args.y, purpose });
|
|
869
|
-
}
|
|
870
|
-
return { provider: 'coredevice', interaction: record, success: true };
|
|
871
|
-
}
|
|
872
|
-
case 'physical_device_fill': {
|
|
873
|
-
const runner = requireRunner(state);
|
|
874
|
-
const target = requireString(input.args.target, 'target');
|
|
875
|
-
rejectSensitiveSemantic(target, 'Fill');
|
|
876
|
-
const text = typeof input.args.text === 'string' ? input.args.text : undefined;
|
|
877
|
-
if (text === undefined) throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', 'text is required.', { retryable: false });
|
|
878
|
-
const id = await elementId(runner.endpoint, runner.sessionId, target);
|
|
879
|
-
await inspectElementSemantics(runner.endpoint, runner.sessionId, id, target, 'Fill');
|
|
880
|
-
await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/element/${encodeURIComponent(id)}/clear`, {}, 'IOS_DEVICE_FILL_FAILED', true);
|
|
881
|
-
await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/element/${encodeURIComponent(id)}/value`, {
|
|
882
|
-
text, value: Array.from(text),
|
|
883
|
-
}, 'IOS_DEVICE_FILL_FAILED', true);
|
|
884
|
-
appendEvent(input, record.interactionId, 'fill', { target, text: '<redacted>' });
|
|
885
|
-
return { provider: 'coredevice', interaction: record, success: true, text: '<redacted>' };
|
|
886
|
-
}
|
|
887
|
-
case 'physical_device_scroll': {
|
|
888
|
-
const runner = requireRunner(state);
|
|
889
|
-
const direction = requireString(input.args.direction, 'direction');
|
|
890
|
-
if (!['up', 'down', 'left', 'right', 'top', 'bottom'].includes(direction)) {
|
|
891
|
-
throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', 'Unsupported scroll direction.', { retryable: false });
|
|
892
|
-
}
|
|
893
|
-
const sizeValue = await runnerRequest('GET', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/window/size`, undefined, 'IOS_DEVICE_SCROLL_FAILED');
|
|
894
|
-
const size = objectValue(sizeValue);
|
|
895
|
-
const width = Number(size.width ?? 0);
|
|
896
|
-
const height = Number(size.height ?? 0);
|
|
897
|
-
if (!(width > 0 && height > 0)) throw new AssistantPluginError('IOS_DEVICE_SCROLL_FAILED', 'The UI runner returned an invalid window size.', { retryable: false });
|
|
898
|
-
const amount = Math.max(20, Math.min(80, typeof input.args.amount === 'number' ? Math.trunc(input.args.amount) : 60)) / 100;
|
|
899
|
-
let fromX = width / 2; let toX = width / 2; let fromY = height / 2; let toY = height / 2;
|
|
900
|
-
if (direction === 'down' || direction === 'bottom') { fromY = height * 0.8; toY = height * (0.8 - amount); }
|
|
901
|
-
if (direction === 'up' || direction === 'top') { fromY = height * 0.2; toY = height * (0.2 + amount); }
|
|
902
|
-
if (direction === 'right') { fromX = width * 0.8; toX = width * (0.8 - amount); }
|
|
903
|
-
if (direction === 'left') { fromX = width * 0.2; toX = width * (0.2 + amount); }
|
|
904
|
-
await runnerRequest('POST', runner.endpoint, `/session/${encodeURIComponent(runner.sessionId)}/wda/dragfromtoforduration`, {
|
|
905
|
-
fromX, fromY, toX, toY, duration: 0.25,
|
|
906
|
-
}, 'IOS_DEVICE_SCROLL_FAILED');
|
|
907
|
-
appendEvent(input, record.interactionId, 'scroll', { direction, amount });
|
|
908
|
-
return { provider: 'coredevice', interaction: record, success: true };
|
|
909
|
-
}
|
|
910
1355
|
case 'physical_device_screenshot': {
|
|
911
1356
|
const label = sanitize(optionalString(input.args.label) ?? 'screenshot');
|
|
912
1357
|
const path = join(artifactDir(input, record.interactionId), `${label}-${hooks.now().getTime()}.png`);
|
|
913
|
-
const result = runCoreJson(input, [
|
|
1358
|
+
const result = await runCoreJson(input, [
|
|
914
1359
|
'device', 'capture', 'screenshot', '--device', state.device.identifier, '--destination', path,
|
|
915
1360
|
], 'IOS_DEVICE_SCREENSHOT_FAILED', 60_000);
|
|
916
1361
|
if (!existsSync(path)) {
|
|
917
1362
|
throw new AssistantPluginError('IOS_DEVICE_SCREENSHOT_MISSING', 'CoreDevice succeeded without creating the requested screenshot.', { retryable: false });
|
|
918
1363
|
}
|
|
919
|
-
|
|
920
|
-
|
|
1364
|
+
const observedAt = timestamp();
|
|
1365
|
+
const observationId = randomUUID();
|
|
1366
|
+
const screenshotDisplay = pngDisplayGeometry(path, state.display?.pointScale);
|
|
1367
|
+
if (screenshotDisplay) state.display = { ...screenshotDisplay, observedAt };
|
|
1368
|
+
state.lastScreenshot = { observationId, observedAt, label };
|
|
1369
|
+
state.foregroundObservation = undefined;
|
|
1370
|
+
writeState(input, state);
|
|
1371
|
+
const eventStartedAt = performance.now();
|
|
1372
|
+
appendEvent(input, record.interactionId, 'screenshot', { label, observationId });
|
|
1373
|
+
recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
|
|
1374
|
+
return finishTimedResult(actionStartedAt, timingStages, {
|
|
921
1375
|
provider: 'coredevice', interaction: record, result: bounded(result),
|
|
1376
|
+
observation: { observationId, observedAt, label, ttlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS },
|
|
1377
|
+
fastPath: {
|
|
1378
|
+
actionId: 'physical_device_batch',
|
|
1379
|
+
arguments: { interaction_id: record.interactionId, observation_id: observationId },
|
|
1380
|
+
requires: ['steps'],
|
|
1381
|
+
ttlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS,
|
|
1382
|
+
oneShot: true,
|
|
1383
|
+
},
|
|
922
1384
|
artifactCandidates: [{ kind: 'ios_physical_device_screenshot', mediaType: 'image/png', path }],
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
case 'physical_device_confirm_foreground': {
|
|
1388
|
+
const observationId = requireString(input.args.observation_id, 'observation_id');
|
|
1389
|
+
const bundleId = requireString(input.args.bundle_id, 'bundle_id');
|
|
1390
|
+
if (bundleId !== state.bundleId) {
|
|
1391
|
+
throw new AssistantPluginError('IOS_DEVICE_FOREGROUND_OBSERVATION_MISMATCH', `Foreground confirmation bundle ${bundleId} does not match session target ${state.bundleId}.`, {
|
|
1392
|
+
retryable: false,
|
|
1393
|
+
details: { expectedBundleId: state.bundleId, providedBundleId: bundleId, hidMutationDispatched: false },
|
|
1394
|
+
});
|
|
1395
|
+
}
|
|
1396
|
+
const { observation, ageMs } = recentScreenshotObservation(state, observationId);
|
|
1397
|
+
const confirmedAt = timestamp();
|
|
1398
|
+
state.foregroundObservation = {
|
|
1399
|
+
observationId: observation.observationId,
|
|
1400
|
+
screenshotObservedAt: observation.observedAt,
|
|
1401
|
+
confirmedAt,
|
|
1402
|
+
bundleId,
|
|
1403
|
+
};
|
|
1404
|
+
writeState(input, state);
|
|
1405
|
+
appendEvent(input, record.interactionId, 'foreground_confirmed', { observationId, bundleId, screenshotAgeMs: ageMs });
|
|
1406
|
+
return {
|
|
1407
|
+
provider: 'coredevice',
|
|
1408
|
+
interaction: record,
|
|
1409
|
+
foregroundObservation: { observationId, bundleId, screenshotObservedAt: observation.observedAt, confirmedAt, ttlMs: INPUT_FOREGROUND_OBSERVATION_TTL_MS },
|
|
1410
|
+
deviceUnmodified: true,
|
|
923
1411
|
};
|
|
924
1412
|
}
|
|
1413
|
+
case 'physical_device_observed_tap': {
|
|
1414
|
+
const lockStartedAt = performance.now();
|
|
1415
|
+
const unlock = await requireSessionUnlocked(input, state);
|
|
1416
|
+
recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
|
|
1417
|
+
if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
|
|
1418
|
+
const observationId = requireString(input.args.observation_id, 'observation_id');
|
|
1419
|
+
const observationStartedAt = performance.now();
|
|
1420
|
+
const { observation, ageMs } = recentScreenshotObservation(state, observationId);
|
|
1421
|
+
recordTiming(timingStages, 'screenObservation', observationStartedAt, true);
|
|
1422
|
+
const displayStartedAt = performance.now();
|
|
1423
|
+
const geometry = await sessionDisplayGeometry(input, state);
|
|
1424
|
+
recordTiming(timingStages, 'displayInfo', displayStartedAt, geometry.cached);
|
|
1425
|
+
const display = geometry.display;
|
|
1426
|
+
const x = requireFiniteNumber(input.args.x, 'x');
|
|
1427
|
+
const y = requireFiniteNumber(input.args.y, 'y');
|
|
1428
|
+
consumePhysicalScreenObservation(input, state);
|
|
1429
|
+
const hidStartedAt = performance.now();
|
|
1430
|
+
const result = await executeRemoteXpcHidInput({
|
|
1431
|
+
controllerHome: input.controllerHome, deviceIdentifier: state.device.identifier, udid: state.device.udid,
|
|
1432
|
+
width: display.width, height: display.height, action: 'tap', x, y,
|
|
1433
|
+
});
|
|
1434
|
+
timingStages.hidWorkerReady = { ms: result.timings.workerStartupMs + result.timings.workerReadyMs, cached: result.reusedWorker };
|
|
1435
|
+
timingStages.hidWriteAck = { ms: result.timings.hidMs ?? result.timings.requestMs, cached: false };
|
|
1436
|
+
timingStages.hidRequestTotal = { ms: elapsedMs(hidStartedAt), cached: result.reusedWorker };
|
|
1437
|
+
appendEvent(input, record.interactionId, 'observed_tap', { requestId: input.requestId, observationId: observation.observationId, observationAgeMs: ageMs, x, y, reusedWorker: result.reusedWorker });
|
|
1438
|
+
return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', inputBackend: 'remote-xpc-hid', interaction: record, display, result: bounded(result), observationConsumed: true });
|
|
1439
|
+
}
|
|
1440
|
+
case 'physical_device_batch': {
|
|
1441
|
+
const lockStartedAt = performance.now();
|
|
1442
|
+
const unlock = await requireSessionUnlocked(input, state);
|
|
1443
|
+
recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
|
|
1444
|
+
if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
|
|
1445
|
+
const steps = physicalInputBatchSteps(input.args.steps);
|
|
1446
|
+
let display: DisplayGeometry | undefined;
|
|
1447
|
+
if (steps.some((step) => step.kind === 'tap' || step.kind === 'swipe')) {
|
|
1448
|
+
const displayStartedAt = performance.now();
|
|
1449
|
+
const geometry = await sessionDisplayGeometry(input, state);
|
|
1450
|
+
recordTiming(timingStages, 'displayInfo', displayStartedAt, geometry.cached);
|
|
1451
|
+
display = geometry.display;
|
|
1452
|
+
}
|
|
1453
|
+
validatePhysicalInputBatchCoordinates(steps, display);
|
|
1454
|
+
const foregroundStartedAt = performance.now();
|
|
1455
|
+
const observationId = input.args.observation_id === undefined
|
|
1456
|
+
? undefined
|
|
1457
|
+
: requireString(input.args.observation_id, 'observation_id');
|
|
1458
|
+
const foreground = resolvePhysicalBatchForegroundObservation(state, observationId);
|
|
1459
|
+
const foregroundObservation = foreground.foregroundObservation;
|
|
1460
|
+
recordTiming(timingStages, 'foregroundObservation', foregroundStartedAt, true);
|
|
1461
|
+
consumePhysicalScreenObservation(input, state);
|
|
1462
|
+
const batchStartedAt = performance.now();
|
|
1463
|
+
const completed: Array<Record<string, unknown>> = [];
|
|
1464
|
+
let completedMutations = 0;
|
|
1465
|
+
for (let index = 0; index < steps.length; index += 1) {
|
|
1466
|
+
const step = steps[index]!;
|
|
1467
|
+
try {
|
|
1468
|
+
if (step.kind === 'wait') {
|
|
1469
|
+
await waitPhysicalInputBatch(step.durationMs ?? 0, input.signal);
|
|
1470
|
+
completed.push({ index, kind: step.kind, durationMs: step.durationMs ?? 0 });
|
|
1471
|
+
continue;
|
|
1472
|
+
}
|
|
1473
|
+
let result: Awaited<ReturnType<typeof executeRemoteXpcHidInput>>;
|
|
1474
|
+
if (step.kind === 'tap') {
|
|
1475
|
+
result = await executeRemoteXpcHidInput({
|
|
1476
|
+
controllerHome: input.controllerHome,
|
|
1477
|
+
deviceIdentifier: state.device.identifier,
|
|
1478
|
+
udid: state.device.udid,
|
|
1479
|
+
width: display?.width,
|
|
1480
|
+
height: display?.height,
|
|
1481
|
+
action: 'tap', x: step.x, y: step.y,
|
|
1482
|
+
});
|
|
1483
|
+
} else if (step.kind === 'swipe') {
|
|
1484
|
+
result = await executeRemoteXpcHidInput({
|
|
1485
|
+
controllerHome: input.controllerHome,
|
|
1486
|
+
deviceIdentifier: state.device.identifier,
|
|
1487
|
+
udid: state.device.udid,
|
|
1488
|
+
width: display?.width,
|
|
1489
|
+
height: display?.height,
|
|
1490
|
+
action: 'swipe', x: step.x, y: step.y, x2: step.toX, y2: step.toY, durationMs: step.durationMs,
|
|
1491
|
+
});
|
|
1492
|
+
} else {
|
|
1493
|
+
result = await executeRemoteXpcHidInput({
|
|
1494
|
+
controllerHome: input.controllerHome,
|
|
1495
|
+
deviceIdentifier: state.device.identifier,
|
|
1496
|
+
udid: state.device.udid,
|
|
1497
|
+
action: 'type', text: step.text, textMode: step.textMode, replaceExisting: step.replaceExisting,
|
|
1498
|
+
});
|
|
1499
|
+
}
|
|
1500
|
+
completedMutations += 1;
|
|
1501
|
+
completed.push({
|
|
1502
|
+
index,
|
|
1503
|
+
kind: step.kind,
|
|
1504
|
+
reusedWorker: result.reusedWorker,
|
|
1505
|
+
requestMs: result.timings.requestMs,
|
|
1506
|
+
hidMs: result.timings.hidMs,
|
|
1507
|
+
...(step.kind === 'type' ? { inputMode: objectValue(result.result).inputMode } : {}),
|
|
1508
|
+
});
|
|
1509
|
+
} catch (error) {
|
|
1510
|
+
const normalized = toAssistantPluginError(error, {
|
|
1511
|
+
code: 'IOS_HID_BATCH_FAILED',
|
|
1512
|
+
message: `Physical iOS input batch failed at step ${index}.`,
|
|
1513
|
+
retryable: false,
|
|
1514
|
+
});
|
|
1515
|
+
const eventStartedAt = performance.now();
|
|
1516
|
+
const causeDetails = normalized.details && typeof normalized.details === 'object' && !Array.isArray(normalized.details)
|
|
1517
|
+
? normalized.details as Record<string, unknown>
|
|
1518
|
+
: {};
|
|
1519
|
+
const retryWholeBatch = completedMutations === 0 && causeDetails.mutationDispatched === false;
|
|
1520
|
+
const causePhase = typeof causeDetails.phase === 'string' ? causeDetails.phase : undefined;
|
|
1521
|
+
const causeMutationDispatched = typeof causeDetails.mutationDispatched === 'boolean'
|
|
1522
|
+
? causeDetails.mutationDispatched
|
|
1523
|
+
: undefined;
|
|
1524
|
+
const causeMessage = normalized.message.startsWith(`${normalized.code}: `)
|
|
1525
|
+
? normalized.message.slice(normalized.code.length + 2, normalized.code.length + 514)
|
|
1526
|
+
: normalized.message.slice(0, 512);
|
|
1527
|
+
appendEvent(input, record.interactionId, 'batch_failed', {
|
|
1528
|
+
requestId: input.requestId,
|
|
1529
|
+
failedStepIndex: index,
|
|
1530
|
+
failedStepKind: step.kind,
|
|
1531
|
+
completedSteps: completed.length,
|
|
1532
|
+
completedMutations,
|
|
1533
|
+
retryWholeBatch,
|
|
1534
|
+
causeCode: normalized.code,
|
|
1535
|
+
...(causePhase ? { causePhase } : {}),
|
|
1536
|
+
...(causeMutationDispatched === undefined ? {} : { mutationDispatched: causeMutationDispatched }),
|
|
1537
|
+
causeMessage,
|
|
1538
|
+
});
|
|
1539
|
+
recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
|
|
1540
|
+
throw new AssistantPluginError(
|
|
1541
|
+
'IOS_HID_BATCH_FAILED',
|
|
1542
|
+
retryWholeBatch
|
|
1543
|
+
? `Physical iOS input batch stopped at step ${index} before any HID mutation was sent; retrying the whole batch is safe.`
|
|
1544
|
+
: `Physical iOS input batch stopped at step ${index}; do not retry the whole batch because an earlier or in-flight step may already have mutated the app.`,
|
|
1545
|
+
{
|
|
1546
|
+
retryable: retryWholeBatch,
|
|
1547
|
+
details: {
|
|
1548
|
+
failedStepIndex: index,
|
|
1549
|
+
failedStepKind: step.kind,
|
|
1550
|
+
completedSteps: completed.length,
|
|
1551
|
+
completedMutations,
|
|
1552
|
+
retryWholeBatch,
|
|
1553
|
+
causeCode: normalized.code,
|
|
1554
|
+
...(causePhase ? { causePhase } : {}),
|
|
1555
|
+
...(causeMutationDispatched === undefined ? {} : { mutationDispatched: causeMutationDispatched }),
|
|
1556
|
+
causeMessage,
|
|
1557
|
+
},
|
|
1558
|
+
},
|
|
1559
|
+
);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
recordTiming(timingStages, 'hidBatchTotal', batchStartedAt, false);
|
|
1563
|
+
const eventStartedAt = performance.now();
|
|
1564
|
+
appendEvent(input, record.interactionId, 'batch_input', {
|
|
1565
|
+
requestId: input.requestId,
|
|
1566
|
+
stepCount: steps.length,
|
|
1567
|
+
mutationCount: completedMutations,
|
|
1568
|
+
kinds: steps.map((step) => step.kind),
|
|
1569
|
+
});
|
|
1570
|
+
recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
|
|
1571
|
+
return finishTimedResult(actionStartedAt, timingStages, {
|
|
1572
|
+
provider: 'coredevice',
|
|
1573
|
+
inputBackend: 'remote-xpc-hid',
|
|
1574
|
+
interaction: record,
|
|
1575
|
+
...(display ? { display } : {}),
|
|
1576
|
+
completed,
|
|
1577
|
+
foregroundObservation: {
|
|
1578
|
+
...foregroundObservation,
|
|
1579
|
+
consumed: true,
|
|
1580
|
+
source: foreground.source,
|
|
1581
|
+
...(foreground.screenshotAgeMs === undefined ? {} : { screenshotAgeMs: foreground.screenshotAgeMs }),
|
|
1582
|
+
},
|
|
1583
|
+
executionPlan: {
|
|
1584
|
+
foregroundActivations: 0,
|
|
1585
|
+
foregroundObservationChecks: 1,
|
|
1586
|
+
foregroundObservationSource: foreground.source,
|
|
1587
|
+
unlockChecks: unlock.cached ? 0 : 1,
|
|
1588
|
+
displayLookups: display ? 1 : 0,
|
|
1589
|
+
pluginRoundTrips: 1,
|
|
1590
|
+
runnerOwned: false,
|
|
1591
|
+
},
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1594
|
+
case 'physical_device_tap': {
|
|
1595
|
+
const lockStartedAt = performance.now();
|
|
1596
|
+
const unlock = await requireSessionUnlocked(input, state);
|
|
1597
|
+
recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
|
|
1598
|
+
if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
|
|
1599
|
+
const displayStartedAt = performance.now();
|
|
1600
|
+
const geometry = await sessionDisplayGeometry(input, state);
|
|
1601
|
+
recordTiming(timingStages, 'displayInfo', displayStartedAt, geometry.cached);
|
|
1602
|
+
const display = geometry.display;
|
|
1603
|
+
const x = requireFiniteNumber(input.args.x, 'x');
|
|
1604
|
+
const y = requireFiniteNumber(input.args.y, 'y');
|
|
1605
|
+
const foregroundStartedAt = performance.now();
|
|
1606
|
+
const foregroundObservation = requireObservedPhysicalTargetForeground(state);
|
|
1607
|
+
recordTiming(timingStages, 'foregroundObservation', foregroundStartedAt, true);
|
|
1608
|
+
consumePhysicalScreenObservation(input, state);
|
|
1609
|
+
const hidStartedAt = performance.now();
|
|
1610
|
+
const result = await executeRemoteXpcHidInput({
|
|
1611
|
+
controllerHome: input.controllerHome,
|
|
1612
|
+
deviceIdentifier: state.device.identifier,
|
|
1613
|
+
udid: state.device.udid,
|
|
1614
|
+
width: display.width,
|
|
1615
|
+
height: display.height,
|
|
1616
|
+
action: 'tap', x, y,
|
|
1617
|
+
});
|
|
1618
|
+
timingStages.hidWorkerReady = { ms: result.timings.workerStartupMs + result.timings.workerReadyMs, cached: result.reusedWorker };
|
|
1619
|
+
timingStages.hidWriteAck = { ms: result.timings.hidMs ?? result.timings.requestMs, cached: false };
|
|
1620
|
+
timingStages.hidRequestTotal = { ms: elapsedMs(hidStartedAt), cached: result.reusedWorker };
|
|
1621
|
+
const eventStartedAt = performance.now();
|
|
1622
|
+
appendEvent(input, record.interactionId, 'tap', { requestId: input.requestId, x, y, width: display.width, height: display.height, reusedWorker: result.reusedWorker });
|
|
1623
|
+
recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
|
|
1624
|
+
return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', inputBackend: 'remote-xpc-hid', interaction: record, display, result: bounded(result), foregroundObservation: { ...foregroundObservation, consumed: true } });
|
|
1625
|
+
}
|
|
1626
|
+
case 'physical_device_swipe': {
|
|
1627
|
+
const lockStartedAt = performance.now();
|
|
1628
|
+
const unlock = await requireSessionUnlocked(input, state);
|
|
1629
|
+
recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
|
|
1630
|
+
if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
|
|
1631
|
+
const displayStartedAt = performance.now();
|
|
1632
|
+
const geometry = await sessionDisplayGeometry(input, state);
|
|
1633
|
+
recordTiming(timingStages, 'displayInfo', displayStartedAt, geometry.cached);
|
|
1634
|
+
const display = geometry.display;
|
|
1635
|
+
const x = requireFiniteNumber(input.args.x, 'x');
|
|
1636
|
+
const y = requireFiniteNumber(input.args.y, 'y');
|
|
1637
|
+
const x2 = requireFiniteNumber(input.args.to_x, 'to_x');
|
|
1638
|
+
const y2 = requireFiniteNumber(input.args.to_y, 'to_y');
|
|
1639
|
+
const durationMs = typeof input.args.duration_ms === 'number' ? Math.trunc(input.args.duration_ms) : 250;
|
|
1640
|
+
const foregroundStartedAt = performance.now();
|
|
1641
|
+
const foregroundObservation = requireObservedPhysicalTargetForeground(state);
|
|
1642
|
+
recordTiming(timingStages, 'foregroundObservation', foregroundStartedAt, true);
|
|
1643
|
+
consumePhysicalScreenObservation(input, state);
|
|
1644
|
+
const hidStartedAt = performance.now();
|
|
1645
|
+
const result = await executeRemoteXpcHidInput({
|
|
1646
|
+
controllerHome: input.controllerHome,
|
|
1647
|
+
deviceIdentifier: state.device.identifier,
|
|
1648
|
+
udid: state.device.udid,
|
|
1649
|
+
width: display.width,
|
|
1650
|
+
height: display.height,
|
|
1651
|
+
action: 'swipe', x, y, x2, y2, durationMs,
|
|
1652
|
+
});
|
|
1653
|
+
timingStages.hidWorkerReady = { ms: result.timings.workerStartupMs + result.timings.workerReadyMs, cached: result.reusedWorker };
|
|
1654
|
+
timingStages.hidWriteAck = { ms: result.timings.hidMs ?? result.timings.requestMs, cached: false };
|
|
1655
|
+
timingStages.hidRequestTotal = { ms: elapsedMs(hidStartedAt), cached: result.reusedWorker };
|
|
1656
|
+
const eventStartedAt = performance.now();
|
|
1657
|
+
appendEvent(input, record.interactionId, 'swipe', { requestId: input.requestId, from: [x, y], to: [x2, y2], durationMs, reusedWorker: result.reusedWorker });
|
|
1658
|
+
recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
|
|
1659
|
+
return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', inputBackend: 'remote-xpc-hid', interaction: record, display, result: bounded(result), foregroundObservation: { ...foregroundObservation, consumed: true } });
|
|
1660
|
+
}
|
|
1661
|
+
case 'physical_device_type_text': {
|
|
1662
|
+
const lockStartedAt = performance.now();
|
|
1663
|
+
const unlock = await requireSessionUnlocked(input, state);
|
|
1664
|
+
recordTiming(timingStages, 'lockState', lockStartedAt, unlock.cached);
|
|
1665
|
+
if (!state.device.udid) throw new AssistantPluginError('IOS_HID_UDID_MISSING', 'The selected iPhone does not expose a hardware UDID for RemoteXPC HID.', { retryable: false });
|
|
1666
|
+
const text = requireString(input.args.text, 'text');
|
|
1667
|
+
const textMode = optionalString(input.args.input_mode) ?? 'auto';
|
|
1668
|
+
if (!['auto', 'keys', 'pasteboard'].includes(textMode)) {
|
|
1669
|
+
throw new AssistantPluginError('PLUGIN_ACTION_ARGUMENT_INVALID', `Unsupported input_mode: ${textMode}.`, { retryable: false });
|
|
1670
|
+
}
|
|
1671
|
+
const foregroundStartedAt = performance.now();
|
|
1672
|
+
const foregroundObservation = requireObservedPhysicalTargetForeground(state);
|
|
1673
|
+
recordTiming(timingStages, 'foregroundObservation', foregroundStartedAt, true);
|
|
1674
|
+
consumePhysicalScreenObservation(input, state);
|
|
1675
|
+
const hidStartedAt = performance.now();
|
|
1676
|
+
const result = await executeRemoteXpcHidInput({
|
|
1677
|
+
controllerHome: input.controllerHome,
|
|
1678
|
+
deviceIdentifier: state.device.identifier,
|
|
1679
|
+
udid: state.device.udid,
|
|
1680
|
+
action: 'type', text, textMode: textMode as 'auto' | 'keys' | 'pasteboard', replaceExisting: input.args.replace_existing === true,
|
|
1681
|
+
});
|
|
1682
|
+
timingStages.hidWorkerReady = { ms: result.timings.workerStartupMs + result.timings.workerReadyMs, cached: result.reusedWorker };
|
|
1683
|
+
timingStages.hidWriteAck = { ms: result.timings.hidMs ?? result.timings.requestMs, cached: false };
|
|
1684
|
+
timingStages.hidRequestTotal = { ms: elapsedMs(hidStartedAt), cached: result.reusedWorker };
|
|
1685
|
+
const eventStartedAt = performance.now();
|
|
1686
|
+
appendEvent(input, record.interactionId, 'type_text', { requestId: input.requestId, length: text.length, reusedWorker: result.reusedWorker });
|
|
1687
|
+
recordTiming(timingStages, 'eventPersistence', eventStartedAt, false);
|
|
1688
|
+
return finishTimedResult(actionStartedAt, timingStages, { provider: 'coredevice', inputBackend: 'remote-xpc-hid', interaction: record, result: bounded(result), text: '<redacted>', foregroundObservation: { ...foregroundObservation, consumed: true } });
|
|
1689
|
+
}
|
|
925
1690
|
case 'physical_device_events': {
|
|
926
1691
|
const limit = Math.max(1, Math.min(MAX_EVENTS, typeof input.args.limit === 'number' ? Math.trunc(input.args.limit) : 50));
|
|
927
1692
|
return { provider: 'coredevice', interaction: record, events: bounded(state.events.slice(-limit)) };
|
|
928
1693
|
}
|
|
929
1694
|
case 'physical_device_close': {
|
|
1695
|
+
const inputWorkerRelease = stopRemoteXpcHidForDevice(state.device.identifier);
|
|
930
1696
|
const wasActive = isInteractionSessionActive(record.status);
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
const closed = await closeRunner(state);
|
|
935
|
-
if (!closed) {
|
|
936
|
-
patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, {
|
|
937
|
-
status: 'closing',
|
|
938
|
-
error: { code: 'IOS_DEVICE_UI_CLOSE_FAILED', message: 'The UI runner session could not be closed; ownership remains fenced.' },
|
|
939
|
-
});
|
|
940
|
-
throw new AssistantPluginError('IOS_DEVICE_UI_CLOSE_FAILED', 'The UI runner session could not be closed; retry physical_device_close.', { retryable: true });
|
|
941
|
-
}
|
|
942
|
-
state.runner.ready = false;
|
|
943
|
-
state.runner.sessionId = undefined;
|
|
944
|
-
writeState(input, state);
|
|
945
|
-
appendEvent(input, record.interactionId, 'session_closed');
|
|
946
|
-
if (!wasActive) {
|
|
947
|
-
return { provider: 'coredevice', interaction: record, alreadyClosed: true, runnerCleaned: runnerNeedsCleanup, deviceUnmodified: true };
|
|
948
|
-
}
|
|
1697
|
+
if (!wasActive) return { provider: 'coredevice', interaction: record, alreadyClosed: true, deviceUnmodified: true, inputWorkerRelease };
|
|
1698
|
+
patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, { status: 'closing' });
|
|
1699
|
+
appendEvent(input, record.interactionId, 'session_closed', { inputWorkerRelease });
|
|
949
1700
|
const finalRecord = patchInteractionSession(input.repoRoot, PROVIDER, record.interactionId, { status: 'closed', error: undefined }) ?? record;
|
|
950
|
-
return { provider: 'coredevice', interaction: finalRecord, deviceUnmodified: true };
|
|
1701
|
+
return { provider: 'coredevice', interaction: finalRecord, deviceUnmodified: true, inputWorkerRelease };
|
|
951
1702
|
}
|
|
952
1703
|
default:
|
|
953
1704
|
throw new AssistantPluginError('PLUGIN_ACTION_NOT_SUPPORTED', `ios/${input.actionId} is not supported.`, { retryable: false });
|