@mobileaidev/ai-app-bridge 0.2.15 → 0.3.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/NOTICE +7 -0
- package/README.md +306 -53
- package/bin/ai-app-bridge.js +51 -4504
- package/bin/android-permissions.js +152 -0
- package/bin/android-uia-xml.js +74 -0
- package/bin/artifact-paths.js +127 -1
- package/bin/bridge-forward.js +56 -0
- package/bin/command-discovery.js +86 -0
- package/bin/command-errors.js +60 -0
- package/bin/command-registry.js +504 -0
- package/bin/command-request.js +14 -0
- package/bin/command-router.js +80 -0
- package/bin/connection-cache.js +119 -0
- package/bin/device-provider.js +2934 -0
- package/bin/execution-host.js +596 -0
- package/bin/execution-runtime.js +101 -0
- package/bin/fact-codec.js +321 -0
- package/bin/fact-recorder.js +691 -0
- package/bin/fact-store.js +226 -0
- package/bin/feedback-probe.js +285 -0
- package/bin/intent/install-intent.js +267 -0
- package/bin/intent/intent-action-executor.js +129 -0
- package/bin/intent/intent-autonomous-adapter.js +46 -0
- package/bin/intent/intent-capture-port.js +64 -0
- package/bin/intent/intent-entry.js +226 -0
- package/bin/intent/intent-errors.js +28 -0
- package/bin/intent/intent-evidence-store.js +121 -0
- package/bin/intent/intent-lifetime.js +62 -0
- package/bin/intent/intent-observation-target.js +33 -0
- package/bin/intent/intent-observer.js +194 -0
- package/bin/intent/intent-production-adapter.js +334 -0
- package/bin/intent/intent-provider.js +27 -0
- package/bin/intent/intent-runtime.js +63 -0
- package/bin/intent/intent-worker.js +432 -0
- package/bin/intent/ios-intent-adapter.js +90 -0
- package/bin/intent/permission-intent.js +239 -0
- package/bin/intent/web-intent-adapter.js +31 -0
- package/bin/ios-device-outcome.js +47 -0
- package/bin/ios-execution.js +108 -0
- package/bin/ios-provider.js +497 -583
- package/bin/ios-runtime-binding.js +54 -0
- package/bin/ios-wda-execution.js +70 -0
- package/bin/ios-wda-port.js +98 -0
- package/bin/ios-wda-project.js +79 -0
- package/bin/mcp-server.js +83 -1040
- package/bin/mmap-scan-index.js +329 -0
- package/bin/observation-collector.js +862 -0
- package/bin/runtime-client.js +154 -0
- package/bin/runtime-directory.js +107 -0
- package/bin/runtime-protocol.js +36 -0
- package/bin/script/bounded-script-registry.js +103 -0
- package/bin/script/node-runtime-adapter.js +233 -0
- package/bin/script/progress-projector.js +63 -0
- package/bin/script/python-runtime-adapter.js +111 -0
- package/bin/script/rolling-summary.js +134 -0
- package/bin/script/script-agent-port.js +40 -0
- package/bin/script/script-assert.js +95 -0
- package/bin/script/script-capture-port.js +76 -0
- package/bin/script/script-catalog.js +85 -0
- package/bin/script/script-durable-restore.js +195 -0
- package/bin/script/script-entry-code.js +26 -0
- package/bin/script/script-entry-route.js +38 -0
- package/bin/script/script-entry.js +3 -0
- package/bin/script/script-errors.js +29 -0
- package/bin/script/script-evidence-store.js +22 -0
- package/bin/script/script-format-removed.js +26 -0
- package/bin/script/script-host-port.js +397 -0
- package/bin/script/script-ledger.js +64 -0
- package/bin/script/script-result.js +57 -0
- package/bin/script/script-sdk.js +152 -0
- package/bin/script/script-sdk.py +153 -0
- package/bin/script/script-session-channel.js +127 -0
- package/bin/script/script-spec.js +86 -0
- package/bin/script/script-supervisor.js +919 -0
- package/bin/script/templates/checkpoint-reentry.js +13 -0
- package/bin/segment-index.js +481 -0
- package/bin/segmented-fact-store.js +1571 -0
- package/bin/shared-kernel/android-h5-target.js +10 -0
- package/bin/shared-kernel/android-install-execution.js +176 -0
- package/bin/shared-kernel/android-sdk-endpoint.js +42 -0
- package/bin/shared-kernel/android-shell-execution.js +195 -0
- package/bin/shared-kernel/argument-schema.js +117 -0
- package/bin/shared-kernel/canonical-path.js +17 -0
- package/bin/shared-kernel/device-acknowledgements.js +53 -0
- package/bin/shared-kernel/device-completion-history.js +52 -0
- package/bin/shared-kernel/device-mutation-lease.js +219 -0
- package/bin/shared-kernel/device-ownership-recovery.js +95 -0
- package/bin/shared-kernel/device-ownership-store.js +94 -0
- package/bin/shared-kernel/evidence-adapters.js +251 -0
- package/bin/shared-kernel/evidence-archive.js +329 -0
- package/bin/shared-kernel/evidence-recording.js +131 -0
- package/bin/shared-kernel/evidence-schema.js +194 -0
- package/bin/shared-kernel/evidence-store.js +190 -0
- package/bin/shared-kernel/execution-admission.js +22 -0
- package/bin/shared-kernel/execution-contracts.js +171 -0
- package/bin/shared-kernel/execution-io.js +106 -0
- package/bin/shared-kernel/execution-ledger.js +125 -0
- package/bin/shared-kernel/execution-scope.js +87 -0
- package/bin/shared-kernel/execution-target.js +115 -0
- package/bin/shared-kernel/flutter-execution.js +13 -0
- package/bin/shared-kernel/flutter-h5-port.js +60 -0
- package/bin/shared-kernel/flutter-h5-target.js +9 -0
- package/bin/shared-kernel/flutter-target.js +75 -0
- package/bin/shared-kernel/h5-execution.js +11 -0
- package/bin/shared-kernel/h5-target.js +31 -0
- package/bin/shared-kernel/host-fact-store.js +49 -0
- package/bin/shared-kernel/ios-h5-target.js +9 -0
- package/bin/shared-kernel/ios-native-target.js +71 -0
- package/bin/shared-kernel/live-capture-query.js +115 -0
- package/bin/shared-kernel/managed-sdk-execution.js +78 -0
- package/bin/shared-kernel/native-execution.js +13 -0
- package/bin/shared-kernel/native-target.js +156 -0
- package/bin/shared-kernel/provider-command-contracts.js +55 -0
- package/bin/shared-kernel/recorded-payload-archive.js +195 -0
- package/bin/shared-kernel/request-context.js +35 -0
- package/bin/shared-kernel/semantic-node.js +55 -0
- package/bin/shared-kernel/summary-transformer.js +352 -0
- package/bin/shared-kernel/target-lease-protocol.js +47 -0
- package/bin/shared-kernel/text-wait.js +111 -0
- package/bin/shared-kernel/uia-execution.js +96 -0
- package/bin/shared-kernel/uia-protocol.js +214 -0
- package/bin/shared-kernel/uia-runtime-port.js +377 -0
- package/bin/shared-kernel/uia-target.js +39 -0
- package/bin/shared-kernel/web-dom-target.js +44 -0
- package/bin/shared-kernel/xml-attributes.js +25 -0
- package/bin/target-execution.js +275 -0
- package/bin/web/command-schema.js +60 -0
- package/bin/web/session-store.js +157 -0
- package/bin/web-provider.js +334 -553
- package/docs/COMMAND_CONTRACT.md +1563 -0
- package/docs/EVIDENCE_ARCHIVE.md +214 -0
- package/docs/INTENT_FOREGROUND.md +71 -0
- package/docs/INTENT_NATIVE_EDITING.md +79 -0
- package/docs/RELEASE.md +59 -0
- package/docs/SCRIPT_AUTHORING.md +489 -0
- package/node_modules/@mobileaidev/segmented-fact-store-native/LICENSE +201 -0
- package/node_modules/@mobileaidev/segmented-fact-store-native/NOTICE +7 -0
- package/node_modules/@mobileaidev/segmented-fact-store-native/binding.gyp +36 -0
- package/node_modules/@mobileaidev/segmented-fact-store-native/bindings/node/sfs_node.c +597 -0
- package/node_modules/@mobileaidev/segmented-fact-store-native/include/sfs.h +178 -0
- package/node_modules/@mobileaidev/segmented-fact-store-native/index.js +5 -0
- package/node_modules/@mobileaidev/segmented-fact-store-native/package.json +24 -0
- package/node_modules/@mobileaidev/segmented-fact-store-native/src/sfs.c +2349 -0
- package/package.json +60 -5
- package/runtime/ios-wda/AABWDABinding.h +19 -0
- package/runtime/ios-wda/AABWDABinding.m +97 -0
- package/runtime/ios-wda/AABWDAExecution.h +26 -0
- package/runtime/ios-wda/AABWDAExecution.m +172 -0
- package/runtime/ios-wda/AABWDAIntegration.h +71 -0
- package/runtime/ios-wda/AABWDAManagedRoutes.h +392 -0
- package/runtime/ios-wda/AABWDAReceiptStore.h +10 -0
- package/runtime/ios-wda/AABWDAReceiptStore.m +116 -0
- package/runtime/uia/ai-app-bridge-uia.jar +0 -0
- package/runtime/uia/manifest.json +22 -0
- package/skills/ai-app-bridge-use/SKILL.md +23 -360
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
# Authoring a code Script
|
|
2
|
+
|
|
3
|
+
This describes the current `aab.code-script/v1` implementation. Discover the
|
|
4
|
+
running server's command allowlist with `capabilities({command:"script"})` and
|
|
5
|
+
each command's arguments with `capabilities({command: name, includeOptions:true})`.
|
|
6
|
+
An installed server can differ from a development checkout.
|
|
7
|
+
|
|
8
|
+
## Start and observe
|
|
9
|
+
|
|
10
|
+
Call MCP `run` with this shape, replacing the target and source path:
|
|
11
|
+
|
|
12
|
+
```json
|
|
13
|
+
{
|
|
14
|
+
"command": "script",
|
|
15
|
+
"arguments": {
|
|
16
|
+
"operation": "start",
|
|
17
|
+
"script": {
|
|
18
|
+
"schemaVersion": "aab.code-script/v1",
|
|
19
|
+
"name": "observed-flow",
|
|
20
|
+
"language": "javascript",
|
|
21
|
+
"sourcePath": "/absolute/flow.js",
|
|
22
|
+
"entrypoint": "main",
|
|
23
|
+
"target": {"platform":"android", "serial": "explicit-device", "packageName": "explicit.package"},
|
|
24
|
+
"inputs": {},
|
|
25
|
+
"permissions": ["app.read", "app.interact"],
|
|
26
|
+
"policy": {"timeoutMs": 180000, "restartPolicy": "none"}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
The same request is available through CLI:
|
|
33
|
+
|
|
34
|
+
```sh
|
|
35
|
+
ai-app-bridge script --operation start --script '{"schemaVersion":"aab.code-script/v1","language":"javascript","sourcePath":"./flow.js","permissions":["app.read","app.interact"]}'
|
|
36
|
+
ai-app-bridge script --operation status --operation-id RETURNED_ID
|
|
37
|
+
ai-app-bridge script --operation result --operation-id RETURNED_ID
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
The CLI returns the operation under `value`. CLI and MCP share a persistent runtime;
|
|
41
|
+
the originating connection can close before another connection queries, answers,
|
|
42
|
+
pauses or cancels the task. Both must select the same FactStore/runtime namespace.
|
|
43
|
+
|
|
44
|
+
Local source paths resolve from the initiating client's directory. `script.cwd`
|
|
45
|
+
defaults to that directory; set it explicitly to select the child working directory.
|
|
46
|
+
It is frozen into Script identity together with source and target. Resume with the
|
|
47
|
+
original `cwd` when connecting from another directory. Relative transport paths
|
|
48
|
+
in a target also bind to the originating request, rather than a later caller.
|
|
49
|
+
|
|
50
|
+
Use exactly one of `sourcePath` and `source`. JavaScript is loaded as CommonJS
|
|
51
|
+
in a real Node child. Export `main`; use `ctx.inputs` for run-specific values.
|
|
52
|
+
The source is trusted local code, not an OS sandbox. Use `ctx.call` for device
|
|
53
|
+
operations so the Host can apply permissions and record receipts.
|
|
54
|
+
|
|
55
|
+
`start` returns an operation ID without waiting for completion. Query `status`
|
|
56
|
+
or `wait` with that `operationId`; `waitMs` bounds one wait and `afterSequence`
|
|
57
|
+
pages execution events. Preserve every returned event page and inspect gaps.
|
|
58
|
+
Track sequence continuity across the event pages you receive. `history.gap`
|
|
59
|
+
reports eviction from the bounded in-process history ring; it can be true even
|
|
60
|
+
when a continuously polling client has already retained every earlier event.
|
|
61
|
+
The durable evidence archive is verified separately.
|
|
62
|
+
`cancel` uses the same operation ID. While owned Host calls, assertions or
|
|
63
|
+
checkpoint writes are settling, status is `cancelling`; new calls and resume are
|
|
64
|
+
rejected. The terminal checkpoint is committed after their receipts. Explicit runtime stop
|
|
65
|
+
and SIGINT/SIGTERM to the runtime perform this cleanup for active Scripts,
|
|
66
|
+
including those waiting for an Agent. MCP EOF or a signal to a CLI/MCP client
|
|
67
|
+
disconnects that client without cancelling the task. Cancellation closes owned local subprocesses and HTTP requests;
|
|
68
|
+
it cannot undo an effect already accepted by a device. A submitted effect without
|
|
69
|
+
a confirmed result remains `ambiguous`, and cannot be replayed from an old
|
|
70
|
+
checkpoint. Permission and lifecycle mutations use the same dispatch ledger as
|
|
71
|
+
UI actions. `policy.timeoutMs` is the whole runtime budget; command arguments may
|
|
72
|
+
set a shorter `timeoutMs`, but never extend it.
|
|
73
|
+
|
|
74
|
+
An execution status of `completed` means the child returned and its final JSON
|
|
75
|
+
and terminal checkpoint were persisted; it does not imply that assertions passed.
|
|
76
|
+
|
|
77
|
+
The JavaScript/Python session protocol bounds each UTF-8 JSON frame by the larger
|
|
78
|
+
of `maxOutputBytes`, `maxProgressBytes` and 1 MiB, plus 64 KiB for the protocol
|
|
79
|
+
envelope. Output and progress retain their separate policy limits. A Host
|
|
80
|
+
response exceeding the frame bound ends the Script with `frame_too_large`; a returned
|
|
81
|
+
device result is not replayed. For mobile capture, use a bounded `limit` and
|
|
82
|
+
follow `evidence.capture.nextCursor` until `hasMore` is false. Check every page's
|
|
83
|
+
target, runtime epoch, committed status and gap; pagination must preserve the
|
|
84
|
+
entire requested evidence window.
|
|
85
|
+
|
|
86
|
+
Start uses the `script` field and its nested `target`. `language` is exactly
|
|
87
|
+
`javascript` or `python`; provide either `source` or `sourcePath`. The Host freezes
|
|
88
|
+
the source, working directory, JSON inputs, target, permissions and policy before execution. Policy
|
|
89
|
+
contains `timeoutMs`, `restartPolicy`, `maxOutputBytes` and `maxProgressBytes`;
|
|
90
|
+
invalid values are rejected rather than replaced. `onFailure` is removed because
|
|
91
|
+
it never controlled code execution. Check call results and assertion verdicts in
|
|
92
|
+
source, throw to fail, and use explicit pause/resume/cancel control as needed.
|
|
93
|
+
Use `status` or `wait` for progress; there is no separate `progress` operation.
|
|
94
|
+
|
|
95
|
+
For a portable evidence run, add `recordingDir` to the start arguments with
|
|
96
|
+
a new output directory. The Host records returned calls, assertions and
|
|
97
|
+
referenced screenshots before bounded events are evicted. With
|
|
98
|
+
`restartPolicy: "none"`, use public `evidence export` with
|
|
99
|
+
`includeRecordedPayloads: true`, then offline `evidence verify` with the
|
|
100
|
+
saved manifest hash. See [the recording and archive contract](EVIDENCE_ARCHIVE.md).
|
|
101
|
+
|
|
102
|
+
For successive event waits, set `afterSequence` to the previous response's
|
|
103
|
+
**`eventSequence`**. `history.lastSequence` belongs to the history page; there
|
|
104
|
+
is no top-level `lastSequence`. Repeatedly passing zero re-delivers old events
|
|
105
|
+
immediately. Status, wait responses and terminal events carry a small `resultRef`,
|
|
106
|
+
not the child's complete return value. Read it with
|
|
107
|
+
`script {operation:"result",operationId:"…"}` after completion. The response
|
|
108
|
+
contains `result`, `resultRef` and `persisted:true`, read from the retained
|
|
109
|
+
FactStore record even after a runtime restart. Event eviction does not determine
|
|
110
|
+
result availability.
|
|
111
|
+
|
|
112
|
+
`resultRef` contains `evidenceId`, `bytes`, `sha256`, `originalSha256` and
|
|
113
|
+
`representation`. Password/token redaction occurs before persistence;
|
|
114
|
+
`original-json` preserves the return value, while `redacted-json` identifies a
|
|
115
|
+
sanitized value. Size and `sha256` cover persisted canonical JSON; the original
|
|
116
|
+
canonical JSON has its separate hash. `policy.maxOutputBytes` defaults to 1 MiB
|
|
117
|
+
and supports up to 64 MiB, subject to full storage capacity. A result write
|
|
118
|
+
failure fails the Script with `result_not_persisted`.
|
|
119
|
+
|
|
120
|
+
Read failures distinguish `result_not_ready` while running, `result_unavailable`
|
|
121
|
+
after failure/cancellation, `result_not_persisted`, `result_not_retained` when the
|
|
122
|
+
referenced result has been evicted, and checksum/read errors. If the whole
|
|
123
|
+
operation is gone it may be `unknown_operation`. Persist the returned value or
|
|
124
|
+
export its evidence before store retention expires; `completed` is not an
|
|
125
|
+
unlimited retention promise.
|
|
126
|
+
|
|
127
|
+
Terminal rolling summaries freeze elapsed and active durations at the terminal
|
|
128
|
+
event. Later status queries do not add idle time. For end-to-end measurement,
|
|
129
|
+
retain the public start timestamp and terminal event timestamp separately.
|
|
130
|
+
|
|
131
|
+
## Capability selection
|
|
132
|
+
|
|
133
|
+
Use `capabilities` to inspect each command's `inputSchema` and `entrypoints.script`.
|
|
134
|
+
Script supports the Android, iOS and Web commands listed in the catalog. An Android target requires
|
|
135
|
+
`platform: "android"`, `serial`, and `packageName`; optional `adb` and `port`
|
|
136
|
+
are retained in the frozen spec and passed to commands that accept them.
|
|
137
|
+
Install APKs through the CLI or MCP Intent installation workflow before starting a fixed Script.
|
|
138
|
+
Expert commands outside the catalog remain unavailable. Add `app.lifecycle`
|
|
139
|
+
explicitly for clear-data fixtures and `app.permissions` for permission
|
|
140
|
+
fixture changes. Defaults remain read/capture/interact; source runs as trusted
|
|
141
|
+
local code. Targets and permissions participate in the recovery hash.
|
|
142
|
+
|
|
143
|
+
An iOS target requires `platform: "ios"`, `deviceId` and `bundleId`. SDK transports
|
|
144
|
+
use `runtimeUrl` or `iosHost`/`iosPort`; WDA commands additionally require the
|
|
145
|
+
explicit `wdaRunnerBundleId` and, where applicable, `wdaSessionId`. The iOS
|
|
146
|
+
provider resolves and arbitrates the physical UDID; it never borrows an Android
|
|
147
|
+
serial. Script request IDs become the original SDK/WDA action IDs.
|
|
148
|
+
|
|
149
|
+
For Flutter iOS use `ios-tap-flutter` with an exact `selector`,
|
|
150
|
+
`ios-input-flutter-text` with `selector` and `text`, `ios-scroll-flutter` with
|
|
151
|
+
`selector` and `delta`, `ios-flutter-back`, and `ios-flutter-hide-keyboard`.
|
|
152
|
+
Selector actions use live Flutter Element references. Keyboard dismissal uses
|
|
153
|
+
the same managed execution receipt as other actions; observe
|
|
154
|
+
`viewport.viewInsets.bottom === 0` before interacting with controls it covered.
|
|
155
|
+
Raw `ios-flutter-action` and `ios-h5-eval` remain outside the
|
|
156
|
+
Script catalog. Use `ios-flutter-nodes` for fresh observations and device
|
|
157
|
+
assertions, and `ios-logs/network/state/events` for mobile evidence.
|
|
158
|
+
|
|
159
|
+
Web uses `platform: "web"`, `sessionId`, `runtimeEpoch` and optional
|
|
160
|
+
`targetId: "main"`. `app.read` admits `web-status` and `web-dom`;
|
|
161
|
+
`app.interact` admits `web-click`, `web-input`, `web-key`, `web-scroll` and
|
|
162
|
+
`web-wait`. The existing serving Web provider owns the connected session and
|
|
163
|
+
mutation lease. Raw `web-command` is outside the Script catalog.
|
|
164
|
+
|
|
165
|
+
Read `web-dom`, choose exactly one observed control, and send its `elementId`
|
|
166
|
+
with `expectedTarget: { pageRef, element }`. The element projection contains
|
|
167
|
+
`elementId`, `tag`, `id`, `name`, `type`, `role`, `ariaLabel`, `placeholder`,
|
|
168
|
+
`href` and `text` from that observation. `interaction.status` explains whether
|
|
169
|
+
the control is ready, disabled, hidden, outside the viewport or obscured.
|
|
170
|
+
Reobserve after an action. Wait by observing the required state; a completed
|
|
171
|
+
input or click is not evidence that an asynchronous business operation finished.
|
|
172
|
+
|
|
173
|
+
Web DOM reads issue Host tree observations usable by `ctx.assert` with
|
|
174
|
+
`requiredEvidence: ["tree"]`. Web logs/network/state/events are currently
|
|
175
|
+
ordinary MCP Host-FactStore reads; their Script/Intent capture-window adapter
|
|
176
|
+
and automatic action correlation remain open. They are not advertised as
|
|
177
|
+
Script capture evidence. The Memos validation source and independent SQLite
|
|
178
|
+
oracle are in `examples/memos-sample/validation`.
|
|
179
|
+
|
|
180
|
+
A missing target, or `target: null`, is permitted for pure code and
|
|
181
|
+
`page-summary`. Device calls still need a complete identity.
|
|
182
|
+
|
|
183
|
+
Same-platform command arguments can explicitly override the default App or
|
|
184
|
+
phone. For a complete per-call target, use the third argument:
|
|
185
|
+
|
|
186
|
+
```javascript
|
|
187
|
+
await ctx.call('uia-tree', {}, {
|
|
188
|
+
target: { platform: 'android', serial: 'explicit-device', packageName: 'com.android.documentsui' }
|
|
189
|
+
});
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
Conflicting identities in command arguments and `options.target` are rejected
|
|
193
|
+
before dispatch. Incomplete targets never borrow identity or transport fields
|
|
194
|
+
from another platform. Events, dispatch markers, receipts, and recordings retain
|
|
195
|
+
the bound target; `execution.target` describes the target used for that call.
|
|
196
|
+
Provider results remain necessary to verify the device's actual runtime identity.
|
|
197
|
+
Changing connection options also invalidates reuse of a previous capture boundary.
|
|
198
|
+
|
|
199
|
+
For a system picker, explicitly bind its package and use `tap-uia` with one
|
|
200
|
+
exact selector field (`text`, `contentDescription` or `resourceName`):
|
|
201
|
+
|
|
202
|
+
```javascript
|
|
203
|
+
await ctx.call('tap-uia', { selector: { text: 'All files' } }, {
|
|
204
|
+
target: { platform: 'android', serial: 'explicit-device', packageName: 'com.example.filemanager' }
|
|
205
|
+
});
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
The Host observes and binds the current node, then the phone revalidates it
|
|
209
|
+
before clicking. Text and accessibility-description selectors stay distinct.
|
|
210
|
+
Observe the destination after each action; after scrolling, wait for stable
|
|
211
|
+
observed geometry before selecting the next file.
|
|
212
|
+
|
|
213
|
+
`tap-text` with `provider:"auto"` discovers Native, Flutter, then UIAutomator
|
|
214
|
+
before a single action. It reports the selected provider and observed matches.
|
|
215
|
+
An explicit provider pins regression replay. A dispatch error/unknown response
|
|
216
|
+
is returned without trying another provider. For Android Native repeated labels,
|
|
217
|
+
use `ctx.call('tap-native', { selector: { resourceName: 'app.package:id/control' } })`.
|
|
218
|
+
Its exact selector and optional `within` row scope are shared with Native Intent;
|
|
219
|
+
the executor observes, revalidates and obtains the SDK target reference. Intent
|
|
220
|
+
decisions additionally bind the choice to an Agent-reviewed revision.
|
|
221
|
+
For exact Android Native editing, use `ctx.call('input-text', {
|
|
222
|
+
selector: { resourceName: 'app.package:id/search' }, text: 'Observed query' })`.
|
|
223
|
+
It shares Intent's editor binding, supports empty text to clear, and rejects a
|
|
224
|
+
noneditable or replaced target. A selector cannot be combined with coordinates.
|
|
225
|
+
For native checkboxes and switches, inspect the observed `checked` boolean
|
|
226
|
+
before choosing a tap. `null` means the View is not an Android `Checkable`
|
|
227
|
+
control. Reobserve after `native_target_changed`; never blindly replay a toggle.
|
|
228
|
+
|
|
229
|
+
## Calls and assertions
|
|
230
|
+
|
|
231
|
+
```javascript
|
|
232
|
+
'use strict';
|
|
233
|
+
|
|
234
|
+
module.exports.main = async function main(ctx) {
|
|
235
|
+
const read = await ctx.call('tree', {
|
|
236
|
+
compact: true, visibleOnly: true, maxNodes: 1000,
|
|
237
|
+
});
|
|
238
|
+
if (!read.ok) throw new Error(`tree:${read.error}`);
|
|
239
|
+
const assertion = await ctx.assert({
|
|
240
|
+
name: 'visible native tree',
|
|
241
|
+
condition: read.result.nodes.some(node => node.visible === true),
|
|
242
|
+
requiredEvidence: ['tree'],
|
|
243
|
+
evidence: read.evidence,
|
|
244
|
+
});
|
|
245
|
+
if (assertion.verdict !== 'passed') {
|
|
246
|
+
throw new Error(`visible native tree:${assertion.verdict}:${assertion.reason || ''}`);
|
|
247
|
+
}
|
|
248
|
+
return { assertion };
|
|
249
|
+
};
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
`ctx.call(command, arguments, options?)` returns an envelope with `ok`, `error`,
|
|
253
|
+
`command`, `result`, `execution`, `evidence`, and `timings`. Provider payloads
|
|
254
|
+
are inside **`result`**. This differs from ordinary MCP `run` results.
|
|
255
|
+
`execution.callId` identifies the call; a mutation also has an action ID.
|
|
256
|
+
|
|
257
|
+
`executionReceipt` is a nullable compact Native/Flutter/H5 SDK or Android shell
|
|
258
|
+
settlement proof. `kind` identifies its protocol. A command containing several
|
|
259
|
+
physical operations can instead provide `executionReceipts`.
|
|
260
|
+
It contains the original action identity and validated outcome, and is retained
|
|
261
|
+
in the persisted action receipt. Unknown settlement has no proof. The nested
|
|
262
|
+
provider result still preserves its own fields: a Native gesture's `completion`
|
|
263
|
+
is the gesture state string, distinct from `executionReceipt`. SDK settlement
|
|
264
|
+
does not prove business success or roll back already delivered input. A shell
|
|
265
|
+
receipt proves the original command process exited, including a nonzero exit;
|
|
266
|
+
assert the observed UI and independent business outcome separately. UIA/physical
|
|
267
|
+
input receipts carry the Script action ID without implying SDK event correlation.
|
|
268
|
+
|
|
269
|
+
H5 completion receipts are top-level fields of `ctx.call(...)`; `result` contains
|
|
270
|
+
the command's DOM result. `h5-wait` uses `timeoutMs`/`intervalMs` and keeps each
|
|
271
|
+
poll's original identity in `executionReceipts`. To assert their phone events,
|
|
272
|
+
read a fresh suffix from the Host-issued pre-action event watermark and compare
|
|
273
|
+
event IDs with the receipts. If the initial history page has a gap, establish a
|
|
274
|
+
complete new suffix before acting; the old gap remains part of the evidence.
|
|
275
|
+
For native `h5-dom`, use the control's `value` to verify empty input; display
|
|
276
|
+
`text` and `ariaLabel` are separate fields. See [H5 execution](COMMAND_CONTRACT.md#android-h5-execution-and-dom-operations).
|
|
277
|
+
Cancellation fences shell work that has not started. An admitted command drains
|
|
278
|
+
on the phone and can outlive the Host timeout; unknown ownership remains blocked.
|
|
279
|
+
Recovery
|
|
280
|
+
records a separate receipt in the `device-ownership` command's execution history;
|
|
281
|
+
it does not rewrite the original unknown action as successful.
|
|
282
|
+
|
|
283
|
+
Host `call_completed` and `call_failed` events expose that `callId` plus the
|
|
284
|
+
returned evidence's `observationId`, `source`, `window`, and `coverage` when
|
|
285
|
+
available. Execution history retains these fields in `payloadSummary`.
|
|
286
|
+
Use them to associate saved envelopes with Host events, including reads such
|
|
287
|
+
as `status` and `keyboard-state` whose `evidenceRefs` can be empty. References
|
|
288
|
+
are preserved unchanged; these metadata fields do not create a capture ref or
|
|
289
|
+
make incomplete or missing evidence valid for a device assertion.
|
|
290
|
+
|
|
291
|
+
`ctx.assert` returns `{verdict, name, scope, reason?}`. Verdict is `passed`,
|
|
292
|
+
`failed`, or `inconclusive`; it does not throw or stop the program. Code must
|
|
293
|
+
check the verdict and implement the intended stopping behavior. `throw` alone
|
|
294
|
+
is an execution failure, not a device assertion.
|
|
295
|
+
|
|
296
|
+
For a device assertion, pass the unchanged `evidence` object from the relevant
|
|
297
|
+
current call. The Host verifies that it issued the evidence, that coverage is
|
|
298
|
+
complete, that the required stream exists, and that the observation is still
|
|
299
|
+
in the current action window. Missing, foreign, edited, evicted or pre-mutation
|
|
300
|
+
evidence is inconclusive. The Host validates the evidence boundary, not whether
|
|
301
|
+
the author's Boolean predicate correctly expresses the business expectation.
|
|
302
|
+
|
|
303
|
+
Assert each observed page before the next mutation. Do not claim a union of
|
|
304
|
+
several scrolled pages using only the last page's evidence. Record separate
|
|
305
|
+
device assertions per page; evaluate cross-page set equality separately and
|
|
306
|
+
retain every page and its assertion. The current API has no device assertion
|
|
307
|
+
that binds multiple observation objects together.
|
|
308
|
+
|
|
309
|
+
`ctx.assert({scope:'code', name, condition})` records a local code check. It
|
|
310
|
+
cannot accept device evidence and is counted separately. Keep independent
|
|
311
|
+
database, file or network business oracles in the report when applicable.
|
|
312
|
+
|
|
313
|
+
## Flutter snapshot freshness and coordinate actions
|
|
314
|
+
|
|
315
|
+
`flutter-nodes` returns the SDK's published snapshot. A new Host
|
|
316
|
+
`evidence.observationId` can contain the same `result.updatedAtMs` and node
|
|
317
|
+
geometry as the previous response. Two equal responses from that one snapshot
|
|
318
|
+
do not demonstrate that a route animation or scroll has settled. Keep the
|
|
319
|
+
provider snapshot timestamp separate from the Host call identity. For a
|
|
320
|
+
stability predicate, require matching relevant geometry across distinct,
|
|
321
|
+
advancing SDK snapshot timestamps, with a bounded read deadline. Missing or
|
|
322
|
+
nonadvancing generations cannot establish that stability.
|
|
323
|
+
|
|
324
|
+
Before a coordinate tap, resolve the target from the latest qualifying tree
|
|
325
|
+
and validate its complete actionable bounds against the current viewport.
|
|
326
|
+
An in-bounds center alone can belong to a button that is mostly offscreen
|
|
327
|
+
during a transition. Reobserve without mutation while waiting for the declared
|
|
328
|
+
condition; a successful tap receipt does not prove navigation occurred and
|
|
329
|
+
must not authorize a blind repeat. Flutter node bounds are logical pixels;
|
|
330
|
+
public Android `tap` coordinates are physical pixels, using that observation's
|
|
331
|
+
viewport device pixel ratio.
|
|
332
|
+
|
|
333
|
+
For Flutter semantic actions, use `tap-flutter`, `input-flutter-text` or
|
|
334
|
+
`scroll-flutter` with `selector:{text:"Exact label"}` or an observed
|
|
335
|
+
`selector:{nodeId:"..."}`. These commands require `app.interact` and bind the
|
|
336
|
+
current SDK Element reference before execution. Missing references require an
|
|
337
|
+
SDK update and a new observation. Node IDs describe live Elements in one runtime;
|
|
338
|
+
do not persist them as cross-run selectors.
|
|
339
|
+
|
|
340
|
+
Input binds an EditableText and verifies that editor and its focus after the
|
|
341
|
+
awaited frames. Supply an explicit selector when several editors are visible;
|
|
342
|
+
without one, only a focused or unique editor may be selected. Scrolling likewise
|
|
343
|
+
requires one unique container or an explicit container selector. At a boundary,
|
|
344
|
+
`flutter_scroll_boundary` is an explicit no-movement result for the code to handle.
|
|
345
|
+
|
|
346
|
+
`tap-flutter` also retains its explicit `tapX`/`tapY` primitive in **logical
|
|
347
|
+
pixels**; do not combine coordinates with a selector or multiply them by DPR.
|
|
348
|
+
The Host supplies the action ID for all these paths. Raw `flutter-action`
|
|
349
|
+
remains outside the Script catalog. See [the command contract](COMMAND_CONTRACT.md)
|
|
350
|
+
for target errors and the remaining Flutter transport cancellation boundary.
|
|
351
|
+
|
|
352
|
+
The Dart runtime preserves this ID within the dispatched action's async zone,
|
|
353
|
+
and freezes it into each record before MethodChannel or HTTP transport awaits.
|
|
354
|
+
The matching Android plugin forwards the full capture payload to the current
|
|
355
|
+
Android SDK; four native streams retain their existing bounded producers.
|
|
356
|
+
Callbacks outside that async scope are unassociated, including ordinary user
|
|
357
|
+
input and unrelated background timers. This is async execution context, not
|
|
358
|
+
a proof that every later event was caused by the most recent gesture. A
|
|
359
|
+
pre-existing listener, isolate, or native physical input does not acquire an
|
|
360
|
+
ID by temporal proximity. The iOS plugin does not yet preserve this field.
|
|
361
|
+
|
|
362
|
+
A route push/pop with `semanticChanged:true` proves that route transition.
|
|
363
|
+
It does not prove persisted settings or a transfer result; keep independent
|
|
364
|
+
business oracles and preserve missing log/state/network evidence as nonpassing.
|
|
365
|
+
|
|
366
|
+
## Mobile capture before and after an action
|
|
367
|
+
|
|
368
|
+
Grant `capture.read` to query `events`, `state`, `logs` or `network`. For each
|
|
369
|
+
stream you intend to assert, first read a bounded current window **before the
|
|
370
|
+
mutation** and retain its mobile-issued `evidence.capture.watermarkCursor`.
|
|
371
|
+
Require a complete, committed page with `gap:false`, `hasMore:false`, and a
|
|
372
|
+
nonempty cursor, runtime epoch and target key. An empty item list can establish
|
|
373
|
+
a cursor; it does not establish a business outcome.
|
|
374
|
+
|
|
375
|
+
Treat mobile cursors as opaque. The Android capture reader now issues `cf3`
|
|
376
|
+
cursors that bind the committed sequence to the loss revision observed at that
|
|
377
|
+
writer barrier. This distinguishes a historical loss from a new enqueue loss
|
|
378
|
+
even when no successful record advanced the sequence. Pagination cursors do
|
|
379
|
+
not acknowledge losses beyond the returned page. Earlier `cf2` cursors return
|
|
380
|
+
`invalid_capture_cursor`; after an SDK change, clear or runtime restart, obtain
|
|
381
|
+
a fresh bounded pre-action page. Persisted fact references and archived payloads
|
|
382
|
+
retain their identities; never rewrite a cursor or suppress a gap on the Host.
|
|
383
|
+
When following `nextCursor`, preserve the original query filters, including
|
|
384
|
+
`sinceMs`/`sinceId`; removing them changes the loss window. A complete
|
|
385
|
+
pre-action `watermarkCursor` is the separately acknowledged starting boundary.
|
|
386
|
+
|
|
387
|
+
```javascript
|
|
388
|
+
const before = await ctx.call('events', { sinceMs: Date.now() - 1000, limit: 200 });
|
|
389
|
+
const capture = before.evidence.capture;
|
|
390
|
+
if (!before.ok || before.evidence.coverage.status !== 'complete'
|
|
391
|
+
|| before.evidence.coverage.gap || !before.evidence.coverage.committed
|
|
392
|
+
|| capture.hasMore !== false || !capture.watermarkCursor
|
|
393
|
+
|| !capture.runtimeEpoch || !capture.targetKey) {
|
|
394
|
+
throw new Error('events pre-action boundary unavailable');
|
|
395
|
+
}
|
|
396
|
+
// tapX/tapY must come from a fresh observation of the intended target.
|
|
397
|
+
const action = await ctx.call('tap', { tapX, tapY });
|
|
398
|
+
if (!action.ok) throw new Error('tap outcome unavailable; do not retry');
|
|
399
|
+
const after = await ctx.call('events', {
|
|
400
|
+
factCursor: capture.watermarkCursor,
|
|
401
|
+
afterActionId: action.execution.actionId,
|
|
402
|
+
runtimeEpoch: capture.runtimeEpoch,
|
|
403
|
+
limit: 200,
|
|
404
|
+
});
|
|
405
|
+
// Evaluate the declared predicate from after.result.items and pass the
|
|
406
|
+
// unchanged after.evidence to ctx.assert; require the events stream.
|
|
407
|
+
```
|
|
408
|
+
|
|
409
|
+
The timestamp bounds the initial read, while the returned cursor supplies the
|
|
410
|
+
boundary for the action. Do not replace that cursor with a host timestamp,
|
|
411
|
+
invent a cursor or reuse one across intervening mutations. A decision-window
|
|
412
|
+
query with an action ID and no lower boundary is rejected with
|
|
413
|
+
`decision_watermark_required`. The Host additionally requires the observed
|
|
414
|
+
pre-action cursor, matching target, epoch and filter before accepting a capture
|
|
415
|
+
assertion. An action targeting a system picker does not establish an app-local
|
|
416
|
+
capture boundary for another package. Missing or unassociated mobile facts
|
|
417
|
+
remain `inconclusive`; a fresh tree or screenshot cannot substitute for them.
|
|
418
|
+
|
|
419
|
+
## Native UI and waiting
|
|
420
|
+
|
|
421
|
+
Selectors for a new flow should come from its observations. Coordinates should
|
|
422
|
+
come from the current target's bounds. `input-text` accepts `text`, an optional
|
|
423
|
+
Native `selector` or paired `tapX`/`tapY`, and `hideKeyboard`; use it for Unicode
|
|
424
|
+
or empty text. Omitting both targeting forms uses the SDK's focused editor
|
|
425
|
+
contract. `tap` uses `tapX`/`tapY`, and `swipe` uses its discovered start/end
|
|
426
|
+
argument names. Pass command arguments directly, without an Intent decision wrapper.
|
|
427
|
+
|
|
428
|
+
After an action, poll a fresh tree for an explicit expected state with a bounded
|
|
429
|
+
deadline. A sleep or successful mechanical action is not an observed outcome.
|
|
430
|
+
Do not retry an uncertain mutation merely because the expected state is absent.
|
|
431
|
+
|
|
432
|
+
Native trees may contain both an indexed activity/window hierarchy and a
|
|
433
|
+
duplicate `root` hierarchy. Preserve window identity when selecting nodes.
|
|
434
|
+
`visible` describes provider metadata; it does not prove that an entire label
|
|
435
|
+
is readable or that another fixed view is not covering it. Compare screenshots
|
|
436
|
+
and clipping/container bounds where that matters. A scroll based on the whole
|
|
437
|
+
window can start in a fixed toolbar; verify that the list actually moved.
|
|
438
|
+
|
|
439
|
+
`ctx.call('screenshot', {outFile:'/absolute/screen.png'})` saves a screenshot.
|
|
440
|
+
Preserve the returned artifact path/hash and the nearby tree. A separately
|
|
441
|
+
requested screenshot and tree are sequential captures, not an atomic pair;
|
|
442
|
+
animations or other intervening changes require fresh observations.
|
|
443
|
+
|
|
444
|
+
## Progress and deliberate control points
|
|
445
|
+
|
|
446
|
+
- `await ctx.progress(event)` emits progress; use small structured events.
|
|
447
|
+
- `await ctx.askAgent(request)` suspends for a controller decision. A normal
|
|
448
|
+
unattended positive run should not need it. A cancellation experiment can
|
|
449
|
+
deliberately wait here and verify that subsequent device calls never occur.
|
|
450
|
+
- `await ctx.checkpoint(name, state)` and `ctx.resume()` support explicit
|
|
451
|
+
checkpoint reentry. They do not save a JavaScript stack. Leave restart policy
|
|
452
|
+
at `none` unless the program implements and tests reentry.
|
|
453
|
+
- `ctx.controlPoint()` returns the current control state; SDK calls also
|
|
454
|
+
cooperate with the Host's pause/cancel control.
|
|
455
|
+
|
|
456
|
+
Save call envelopes, assertions and the final result to a new output directory
|
|
457
|
+
provided through inputs. Freeze the source hash before repeats. Report authoring
|
|
458
|
+
time, execution time, interventions, positive trials and negative trials
|
|
459
|
+
separately. Preserve failed attempts rather than silently replacing them.
|
|
460
|
+
|
|
461
|
+
## Web capture around a business action
|
|
462
|
+
|
|
463
|
+
Declare `capture.read`. Read `web-network` and any other required streams before
|
|
464
|
+
mutating the App, then query with the original lower watermark:
|
|
465
|
+
|
|
466
|
+
```javascript
|
|
467
|
+
const before = await ctx.call('web-network', {});
|
|
468
|
+
const action = await ctx.call('web-click', observedSaveArguments);
|
|
469
|
+
const after = await ctx.call('web-network', {
|
|
470
|
+
factCursor: before.evidence.capture.watermarkCursor,
|
|
471
|
+
});
|
|
472
|
+
// Assert exact request/response content with after.evidence. Do not infer
|
|
473
|
+
// causal attribution from the time window or from an HTTP 200 alone.
|
|
474
|
+
```
|
|
475
|
+
|
|
476
|
+
Keep that lower watermark while waiting for the expected record and complete
|
|
477
|
+
coverage. If the real request has a known action ID, add
|
|
478
|
+
`afterActionId: action.execution.actionId`; otherwise retain its unfiltered
|
|
479
|
+
window and explicit `unattributed` origin. `hasMore: true`, partial coverage and
|
|
480
|
+
an isolated continuation page cannot prove the full decision window.
|
|
481
|
+
|
|
482
|
+
The Memos [save-and-capture Script](../../../examples/memos-sample/validation/memos-save-capture.js)
|
|
483
|
+
decodes the actual Protobuf request and response, checks the exact memo/content,
|
|
484
|
+
verifies the synchronous Save event and checks the rendered checkbox state.
|
|
485
|
+
An external read-only SQLite checkpoint checks persistence before restoration.
|
|
486
|
+
Unknown `checked: null` is not accepted as unchecked. The sample reports its
|
|
487
|
+
actual UI failure even when the backend saved correctly; it does not change the
|
|
488
|
+
business expectation to fit the observed page. The entry source is self-contained
|
|
489
|
+
because the Script runtime copies that source into its execution directory.
|