mobridge-mcp 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/README.md +155 -0
  2. package/dist/adb-baseline/act.mjs +1 -0
  3. package/dist/adb-baseline/capture-lib.mjs +2 -0
  4. package/dist/main.obf.cjs +2 -0
  5. package/dist/protocol/fixtures/01-login-views/expected.json +32 -0
  6. package/dist/protocol/fixtures/01-login-views/snapshot.json +20 -0
  7. package/dist/protocol/fixtures/02-alert-dialog/expected.json +22 -0
  8. package/dist/protocol/fixtures/02-alert-dialog/snapshot.json +20 -0
  9. package/dist/protocol/fixtures/03-recyclerview-list/expected.json +22 -0
  10. package/dist/protocol/fixtures/03-recyclerview-list/snapshot.json +19 -0
  11. package/dist/protocol/fixtures/04-ambiguous-ctas/expected.json +17 -0
  12. package/dist/protocol/fixtures/04-ambiguous-ctas/snapshot.json +21 -0
  13. package/dist/protocol/fixtures/05-compose-settings/expected.json +17 -0
  14. package/dist/protocol/fixtures/05-compose-settings/snapshot.json +19 -0
  15. package/dist/protocol/fixtures/06-canvas-game/expected.json +12 -0
  16. package/dist/protocol/fixtures/06-canvas-game/snapshot.json +16 -0
  17. package/dist/protocol/fixtures/07-search-results/expected.json +22 -0
  18. package/dist/protocol/fixtures/07-search-results/snapshot.json +19 -0
  19. package/dist/protocol/fixtures/README.md +34 -0
  20. package/dist/protocol/platform-capabilities.json +95 -0
  21. package/dist/protocol/schemas/action.schema.json +170 -0
  22. package/dist/protocol/schemas/common.schema.json +59 -0
  23. package/dist/protocol/schemas/diff.schema.json +74 -0
  24. package/dist/protocol/schemas/error.schema.json +34 -0
  25. package/dist/protocol/schemas/node-ref.schema.json +100 -0
  26. package/dist/protocol/schemas/query.schema.json +53 -0
  27. package/dist/protocol/schemas/screen-snapshot.schema.json +50 -0
  28. package/dist/protocol/schemas/semantic-node.schema.json +95 -0
  29. package/dist/protocol/schemas/tools.schema.json +215 -0
  30. package/dist/protocol/schemas/wait.schema.json +57 -0
  31. package/dist/protocol/synonyms.json +74 -0
  32. package/dist/protocol/validate/package-lock.json +71 -0
  33. package/dist/protocol/validate/package.json +13 -0
  34. package/dist/protocol/validate/query-engine.mjs +1 -0
  35. package/dist/protocol/validate/run-fixtures.mjs +6 -0
  36. package/dist/protocol/validate/schema-lint.mjs +2 -0
  37. package/dist/protocol/validate/validate-file.mjs +1 -0
  38. package/integrity.json +4 -0
  39. package/loader.cjs +85 -0
  40. package/package.json +40 -0
  41. package/resources/android/app-developer-debug.apk +0 -0
  42. package/resources/ios/MobileSemantic/Fixture/MobileSemanticFixtureApp.swift +50 -0
  43. package/resources/ios/MobileSemantic/MobileSemantic.xcodeproj/project.pbxproj +463 -0
  44. package/resources/ios/MobileSemantic/MobileSemantic.xcodeproj/xcshareddata/xcschemes/MobileSemanticUITests.xcscheme +97 -0
  45. package/resources/ios/MobileSemantic/UITests/BridgeServer.swift +216 -0
  46. package/resources/ios/MobileSemantic/UITests/XcuiSnapshot.swift +73 -0
  47. package/resources/ios/MobileSemantic/gen_project.rb +67 -0
@@ -0,0 +1,170 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "msm:action.schema.json",
4
+ "title": "Action + Result",
5
+ "description": "An action performed on a NodeRef (or a global/app action), plus its verified result. performAction()==true is NOT success; a VerificationPolicy must be satisfied.",
6
+ "$defs": {
7
+ "actionVerb": {
8
+ "enum": [
9
+ "CLICK", "LONG_CLICK", "SET_TEXT", "CLEAR_TEXT", "FOCUS",
10
+ "SCROLL_FORWARD", "SCROLL_BACKWARD", "SELECT", "EXPAND", "COLLAPSE",
11
+ "DISMISS", "COPY", "PASTE"
12
+ ]
13
+ },
14
+ "globalAction": {
15
+ "enum": ["BACK", "HOME", "RECENTS", "NOTIFICATION_SHADE"]
16
+ },
17
+ "appAction": {
18
+ "enum": ["APP_OPEN", "APP_DEEP_LINK", "APP_BRING_TO_FRONT", "APP_CLOSE_UI"],
19
+ "description": "APP_OPEN launches the launcher intent; APP_DEEP_LINK opens a URI; APP_BRING_TO_FRONT foregrounds an existing task; APP_CLOSE_UI = Back/Home and does NOT guarantee process death. Killing a process (force-stop) is a DESTRUCTIVE op and is NOT available here — use mobile.app.forceStop under MSM_APP_CONTROL (per-package authorize + critical denylist)."
20
+ },
21
+ "verificationPolicy": {
22
+ "enum": [
23
+ "NODE_STATE_CHANGED", "NODE_VALUE_EQUALS", "NODE_DISAPPEARED", "NODE_APPEARED",
24
+ "PACKAGE_CHANGED", "WINDOW_CHANGED", "COLLECTION_POSITION_CHANGED",
25
+ "CLIPBOARD_EQUALS", "EXPLICIT_CONDITION", "DISPATCH_ONLY"
26
+ ],
27
+ "description": "How success is verified. DISPATCH_ONLY is FORBIDDEN for destructive/financial/external-side-effect actions."
28
+ },
29
+ "dispatchStatus": {
30
+ "enum": ["DISPATCHED", "REJECTED", "FAILED"],
31
+ "description": "Purely whether the native action was SENT — never business success. DISPATCHED = the accessibility/gesture action was performed; REJECTED = a precondition blocked it; FAILED = native dispatch failed. Precondition rejections (TARGET_NOT_FOUND / AMBIGUOUS_MATCH / STALE_NODE_REF) are returned as errors, not here."
32
+ },
33
+ "verificationResult": {
34
+ "enum": ["CONDITION_MET", "CONDITION_NOT_MET", "CONDITION_UNKNOWN"],
35
+ "description": "Result of evaluating a SPECIFIC predicate (caller-supplied `expect`, or an intrinsic device-operation postcondition). This is predicate measurement, NOT task/business success — the AI concludes success from the evidence."
36
+ },
37
+ "stability": {
38
+ "enum": ["STABLE", "TRANSITIONING", "UNKNOWN"],
39
+ "description": "Whether the after-snapshot was captured on a settled screen."
40
+ },
41
+ "actionMethod": {
42
+ "enum": ["ACCESSIBILITY_ACTION", "ACCESSIBILITY_ACTION_ANCESTOR", "GESTURE_FALLBACK"]
43
+ },
44
+
45
+ "actionRequest": {
46
+ "type": "object",
47
+ "additionalProperties": false,
48
+ "required": ["action"],
49
+ "properties": {
50
+ "action": { "$ref": "#/$defs/actionVerb" },
51
+ "target": {
52
+ "$ref": "msm:node-ref.schema.json",
53
+ "description": "Required for node actions."
54
+ },
55
+ "value": {
56
+ "type": ["string", "null"],
57
+ "description": "Literal text for SET_TEXT/PASTE. Mutually exclusive with valueRef."
58
+ },
59
+ "valueRef": {
60
+ "type": ["string", "null"],
61
+ "pattern": "^secret://",
62
+ "description": "Opaque secret reference, e.g. secret://account/password. Host never sends plaintext; runtime/broker resolves at execution; plaintext never appears in logs/diff/audit/result."
63
+ },
64
+ "allowGestureFallback": {
65
+ "type": "boolean",
66
+ "default": false,
67
+ "description": "Permit coordinate gesture into bounds if no accessibility action path exists."
68
+ },
69
+ "verify": {
70
+ "type": "object",
71
+ "additionalProperties": false,
72
+ "required": ["policy"],
73
+ "properties": {
74
+ "policy": { "$ref": "#/$defs/verificationPolicy" },
75
+ "expected": {
76
+ "description": "Policy-specific expectation (e.g. value string for NODE_VALUE_EQUALS, a wait-condition for EXPLICIT_CONDITION).",
77
+ "$ref": "msm:wait.schema.json#/$defs/condition"
78
+ },
79
+ "timeoutMs": { "type": "integer", "minimum": 0, "default": 8000 },
80
+ "stableForMs": { "type": "integer", "minimum": 0, "default": 400 }
81
+ }
82
+ },
83
+ "idempotencyKey": {
84
+ "type": ["string", "null"],
85
+ "description": "Required for side-effect actions to make retry safe."
86
+ },
87
+ "generation": {
88
+ "type": ["integer", "null"],
89
+ "description": "Capture generation the target ref was built against (from screen.find/target). When set, the runtime hard-checks it and returns STALE_NODE_REF if the screen advanced — it never re-resolves a stale ref to a similar node."
90
+ }
91
+ },
92
+ "allOf": [
93
+ {
94
+ "$comment": "value and valueRef are mutually exclusive.",
95
+ "not": {
96
+ "type": "object",
97
+ "required": ["value", "valueRef"],
98
+ "properties": {
99
+ "value": { "type": "string" },
100
+ "valueRef": { "type": "string" }
101
+ }
102
+ }
103
+ }
104
+ ]
105
+ },
106
+
107
+ "actionResult": {
108
+ "type": "object",
109
+ "additionalProperties": false,
110
+ "description": "Three separated layers. (1) dispatch = was the native action sent. (2) observation = objective before/after device measurement. (3) verification = OPTIONAL predicate evaluation, present only when a caller `expect` OR an intrinsic device postcondition applies. There is intentionally NO 'SUCCESS_VERIFIED' and NO 'verifiedBy': MCP never infers business success (login/payment/order); the AI concludes that from the evidence.",
111
+ "required": ["dispatch", "observation"],
112
+ "properties": {
113
+ "dispatch": {
114
+ "type": "object",
115
+ "additionalProperties": false,
116
+ "required": ["status"],
117
+ "properties": {
118
+ "status": { "$ref": "#/$defs/dispatchStatus" },
119
+ "method": { "$ref": "#/$defs/actionMethod" },
120
+ "fallbackUsed": { "type": "boolean", "default": false },
121
+ "fallbackType": { "enum": ["VISION_COORDINATE", "GESTURE", null], "default": null }
122
+ }
123
+ },
124
+ "target": {
125
+ "type": "object",
126
+ "description": "Human-readable identity of the acted node.",
127
+ "properties": {
128
+ "role": { "$ref": "msm:common.schema.json#/$defs/role" },
129
+ "label": { "type": ["string", "null"] }
130
+ }
131
+ },
132
+ "mutationClass": {
133
+ "enum": ["READ_ONLY", "REVERSIBLE", "SIDE_EFFECT", "EXTERNAL_SIDE_EFFECT", "DESTRUCTIVE"],
134
+ "description": "Advisory, payload-derived classification of how mutating this specific action was. Metadata for the host, not a gate."
135
+ },
136
+ "observation": {
137
+ "type": "object",
138
+ "additionalProperties": false,
139
+ "description": "Objective measurement of how the device changed. No success semantics.",
140
+ "required": ["beforeSnapshotId", "stability", "screenChanged", "windowChanged"],
141
+ "properties": {
142
+ "beforeSnapshotId": { "type": "string" },
143
+ "afterSnapshotId": { "type": ["string", "null"] },
144
+ "stability": { "$ref": "#/$defs/stability" },
145
+ "screenChanged": { "type": "boolean", "description": "treeHash differs (UI content changed) — NOT success." },
146
+ "windowChanged": { "type": "boolean", "description": "window IDENTITY changed (windowId, else app/package) — never derived from treeHash." },
147
+ "diff": { "type": ["object", "null"], "additionalProperties": true, "description": "added/removed/moved/changed node deltas." },
148
+ "diffRef": { "type": ["string", "null"], "description": "beforeSnapshotId:afterSnapshotId." }
149
+ }
150
+ },
151
+ "verification": {
152
+ "type": "object",
153
+ "additionalProperties": false,
154
+ "description": "Present ONLY when a predicate applies. Measures that predicate, nothing more.",
155
+ "required": ["predicate", "result"],
156
+ "properties": {
157
+ "predicate": {
158
+ "type": "object",
159
+ "additionalProperties": true,
160
+ "description": "The evaluated predicate: an intrinsic postcondition (via `policy`) or a caller-supplied condition.",
161
+ "properties": { "policy": { "$ref": "#/$defs/verificationPolicy" }, "source": { "enum": ["CALLER", "INTRINSIC"] } }
162
+ },
163
+ "result": { "$ref": "#/$defs/verificationResult" },
164
+ "evidence": { "type": "array", "items": { "type": "object", "additionalProperties": true } }
165
+ }
166
+ }
167
+ }
168
+ }
169
+ }
170
+ }
@@ -0,0 +1,59 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "msm:common.schema.json",
4
+ "title": "Common definitions",
5
+ "description": "Shared primitives for the Mobile Semantic MCP protocol. schemaVersion frozen at 1.",
6
+ "$defs": {
7
+ "schemaVersion": {
8
+ "const": 1,
9
+ "description": "Protocol schema version. Bump on any breaking change to any schema."
10
+ },
11
+ "deviceId": { "type": "string", "minLength": 1, "maxLength": 128 },
12
+ "snapshotId": {
13
+ "type": "string",
14
+ "pattern": "^s_[0-9a-zA-Z]+$",
15
+ "description": "Opaque snapshot id, e.g. s_1832. Valid only within its snapshot lifetime."
16
+ },
17
+ "nodeId": {
18
+ "type": "string",
19
+ "description": "Node id UNIQUE ONLY within one snapshot. Never reuse across snapshots; never click an old nodeId directly. Vision nodes use the 'vision:' prefix.",
20
+ "pattern": "^(n[0-9]+|vision:[0-9]+)$"
21
+ },
22
+ "timestampMs": { "type": "integer", "minimum": 0 },
23
+ "packageName": {
24
+ "type": "string",
25
+ "pattern": "^[a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)+$"
26
+ },
27
+ "bundleId": {
28
+ "type": "string",
29
+ "description": "iOS bundle identifier (reverse-DNS, hyphens allowed), e.g. com.apple.Preferences.",
30
+ "pattern": "^[A-Za-z0-9][A-Za-z0-9-]*(\\.[A-Za-z0-9][A-Za-z0-9-]*)+$",
31
+ "maxLength": 255
32
+ },
33
+ "bounds": {
34
+ "type": "array",
35
+ "description": "[left, top, right, bottom] in device pixels.",
36
+ "items": { "type": "integer" },
37
+ "minItems": 4,
38
+ "maxItems": 4
39
+ },
40
+ "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
41
+ "source": {
42
+ "enum": ["ACCESSIBILITY", "APP_SDK", "VISION"],
43
+ "description": "Origin of a node's data. Always reported so the AI knows semantic vs vision."
44
+ },
45
+ "role": {
46
+ "enum": [
47
+ "SCREEN", "DIALOG", "LIST", "LIST_ITEM", "BUTTON", "TEXT", "INPUT",
48
+ "IMAGE", "CHECKBOX", "RADIO", "SWITCH", "TAB", "MENU", "LINK",
49
+ "SLIDER", "PROGRESS", "WEB_VIEW", "CANVAS", "UNKNOWN"
50
+ ],
51
+ "description": "Normalized role. Raw Android class names are NEVER exposed as roles."
52
+ },
53
+ "hash": {
54
+ "type": "string",
55
+ "pattern": "^[0-9a-f]+$",
56
+ "description": "Lowercase hex hash (node/subtree/tree fingerprint)."
57
+ }
58
+ }
59
+ }
@@ -0,0 +1,74 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "msm:diff.schema.json",
4
+ "title": "ScreenDiff",
5
+ "description": "Delta between two snapshots. 'moved' is distinguished from removed+added; tiny animation-only bounds changes are ignored when semantics are unchanged. Target typical size < 5 KB.",
6
+ "type": "object",
7
+ "additionalProperties": false,
8
+ "required": ["schemaVersion", "fromSnapshot", "toSnapshot", "added", "removed", "moved", "changed", "screenChanged"],
9
+ "properties": {
10
+ "schemaVersion": { "$ref": "msm:common.schema.json#/$defs/schemaVersion" },
11
+ "fromSnapshot": { "$ref": "msm:common.schema.json#/$defs/snapshotId" },
12
+ "toSnapshot": { "$ref": "msm:common.schema.json#/$defs/snapshotId" },
13
+ "added": {
14
+ "type": "array",
15
+ "description": "Newly present nodes (full SemanticNode).",
16
+ "items": { "$ref": "msm:semantic-node.schema.json" }
17
+ },
18
+ "removed": {
19
+ "type": "array",
20
+ "items": { "$ref": "msm:common.schema.json#/$defs/nodeId" }
21
+ },
22
+ "moved": {
23
+ "type": "array",
24
+ "items": {
25
+ "type": "object",
26
+ "additionalProperties": false,
27
+ "required": ["node"],
28
+ "properties": {
29
+ "node": { "$ref": "msm:common.schema.json#/$defs/nodeId" },
30
+ "fromBounds": { "$ref": "msm:common.schema.json#/$defs/bounds" },
31
+ "toBounds": { "$ref": "msm:common.schema.json#/$defs/bounds" }
32
+ }
33
+ }
34
+ },
35
+ "changed": {
36
+ "type": "array",
37
+ "items": {
38
+ "type": "object",
39
+ "additionalProperties": false,
40
+ "required": ["node", "fields"],
41
+ "properties": {
42
+ "node": { "$ref": "msm:common.schema.json#/$defs/nodeId" },
43
+ "fields": {
44
+ "type": "object",
45
+ "description": "Map of fieldName -> {before, after}.",
46
+ "additionalProperties": {
47
+ "type": "object",
48
+ "additionalProperties": false,
49
+ "required": ["before", "after"],
50
+ "properties": {
51
+ "before": {},
52
+ "after": {}
53
+ }
54
+ }
55
+ }
56
+ }
57
+ }
58
+ },
59
+ "focusChanged": {
60
+ "type": ["object", "null"],
61
+ "additionalProperties": false,
62
+ "description": "null = no focus on either side. Otherwise a tri-state: focus identity is resolved by stable identity across snapshots, never by raw nodeId. UNKNOWN when the old focused element cannot be resolved uniquely (no fabricated boolean).",
63
+ "properties": {
64
+ "state": { "type": "string", "enum": ["UNCHANGED", "CHANGED", "UNKNOWN"] },
65
+ "from": { "type": ["string", "null"] },
66
+ "to": { "type": ["string", "null"] }
67
+ }
68
+ },
69
+ "screenChanged": {
70
+ "type": "boolean",
71
+ "description": "true when screenStateId/treeHash indicates a materially different screen."
72
+ }
73
+ }
74
+ }
@@ -0,0 +1,34 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "msm:error.schema.json",
4
+ "title": "Error",
5
+ "description": "Uniform error envelope. Every error carries a recovery hint list.",
6
+ "type": "object",
7
+ "additionalProperties": false,
8
+ "required": ["error", "message", "recovery"],
9
+ "properties": {
10
+ "error": { "$ref": "#/$defs/code" },
11
+ "message": { "type": "string" },
12
+ "recovery": {
13
+ "type": "array",
14
+ "description": "Ordered, human/AI-actionable recovery suggestions.",
15
+ "items": { "type": "string" }
16
+ },
17
+ "candidates": {
18
+ "type": "array",
19
+ "description": "For AMBIGUOUS_MATCH: the competing NodeRefs so the caller can disambiguate.",
20
+ "items": { "$ref": "msm:node-ref.schema.json" }
21
+ }
22
+ },
23
+ "$defs": {
24
+ "code": {
25
+ "enum": [
26
+ "DEVICE_OFFLINE", "SESSION_EXPIRED", "PERMISSION_REQUIRED", "WINDOW_UNAVAILABLE",
27
+ "SECURE_WINDOW", "NODE_NOT_FOUND", "AMBIGUOUS_MATCH", "STALE_NODE_REF",
28
+ "WEAK_REFERENCE_UNRESOLVABLE",
29
+ "ACTION_UNSUPPORTED", "ACTION_REJECTED", "NO_EFFECT", "VERIFY_TIMEOUT",
30
+ "VISION_REQUIRED", "POLICY_BLOCKED", "USER_CONFIRMATION_REQUIRED"
31
+ ]
32
+ }
33
+ }
34
+ }
@@ -0,0 +1,100 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "msm:node-ref.schema.json",
4
+ "title": "NodeRef",
5
+ "description": "A re-resolvable reference to a node. Actions take a NodeRef, never a bare nodeId. Before every action the runtime fetches the newest snapshot and re-resolves: 0 match=NODE_NOT_FOUND, >1=AMBIGUOUS_MATCH, 1 with far fingerprint=STALE_NODE_REF, 1 valid=perform. Never auto-pick the first of many.",
6
+ "type": "object",
7
+ "additionalProperties": false,
8
+ "required": ["snapshotId", "nodeId", "fingerprint", "locators"],
9
+ "properties": {
10
+ "snapshotId": { "$ref": "msm:common.schema.json#/$defs/snapshotId" },
11
+ "nodeId": { "$ref": "msm:common.schema.json#/$defs/nodeId" },
12
+ "fingerprint": {
13
+ "$ref": "msm:common.schema.json#/$defs/hash",
14
+ "description": "Node attribute hash captured at ref creation; large drift on re-resolve => STALE_NODE_REF."
15
+ },
16
+ "ancestorFingerprint": {
17
+ "$ref": "msm:common.schema.json#/$defs/hash",
18
+ "description": "OPTIONAL (added N=1 migration window). Fingerprint of the node's ancestor chain; lets the runtime distinguish a moved/duplicated node from the intended one before acting. Absent on N-1 refs => decode still valid, no ancestor cross-check."
19
+ },
20
+ "source": {
21
+ "type": "string",
22
+ "enum": ["SEMANTIC", "VISION", "COORDINATE"],
23
+ "description": "OPTIONAL. Provenance of this ref. Absent => SEMANTIC (back-compat)."
24
+ },
25
+ "confidence": {
26
+ "type": "number",
27
+ "minimum": 0,
28
+ "maximum": 1,
29
+ "description": "OPTIONAL. Resolver confidence at ref creation. Absent => 1 (back-compat)."
30
+ },
31
+ "locators": {
32
+ "type": "array",
33
+ "minItems": 1,
34
+ "description": "Ordered by strength; resolver tries strongest first.",
35
+ "items": { "$ref": "#/$defs/locator" }
36
+ }
37
+ },
38
+ "$defs": {
39
+ "locator": {
40
+ "type": "object",
41
+ "required": ["type"],
42
+ "oneOf": [
43
+ {
44
+ "properties": {
45
+ "type": { "const": "RESOURCE_ID" },
46
+ "value": { "type": "string" }
47
+ },
48
+ "required": ["type", "value"],
49
+ "additionalProperties": false
50
+ },
51
+ {
52
+ "properties": {
53
+ "type": { "const": "RESOURCE_ID_SCOPED" },
54
+ "value": { "type": "string" },
55
+ "windowId": { "type": "integer" },
56
+ "ancestor": { "type": "string" }
57
+ },
58
+ "required": ["type", "value"],
59
+ "additionalProperties": false
60
+ },
61
+ {
62
+ "properties": {
63
+ "type": { "const": "ROLE_LABEL_ANCESTOR" },
64
+ "role": { "$ref": "msm:common.schema.json#/$defs/role" },
65
+ "label": { "type": "string" },
66
+ "ancestor": { "type": ["string", "null"] }
67
+ },
68
+ "required": ["type", "role", "label"],
69
+ "additionalProperties": false
70
+ },
71
+ {
72
+ "properties": {
73
+ "type": { "const": "ROLE_STATE_COLLECTION_ITEM" },
74
+ "role": { "$ref": "msm:common.schema.json#/$defs/role" },
75
+ "itemKey": { "type": "string" },
76
+ "scrollContainer": { "type": ["string", "null"] }
77
+ },
78
+ "required": ["type", "role", "itemKey"],
79
+ "additionalProperties": false
80
+ },
81
+ {
82
+ "properties": {
83
+ "type": { "const": "STRUCTURAL_PATH" },
84
+ "value": { "type": "string", "description": "e.g. '0/2/1' child-index path from root." }
85
+ },
86
+ "required": ["type", "value"],
87
+ "additionalProperties": false
88
+ },
89
+ {
90
+ "properties": {
91
+ "type": { "const": "BOUNDS_NEAREST" },
92
+ "bounds": { "$ref": "msm:common.schema.json#/$defs/bounds" }
93
+ },
94
+ "required": ["type", "bounds"],
95
+ "additionalProperties": false
96
+ }
97
+ ]
98
+ }
99
+ }
100
+ }
@@ -0,0 +1,53 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "msm:query.schema.json",
4
+ "title": "SemanticQuery",
5
+ "description": "Declarative node query. Result always reports match count, NodeRef(s), match reason, confidence, and other candidates when ambiguous.",
6
+ "type": "object",
7
+ "additionalProperties": false,
8
+ "minProperties": 1,
9
+ "properties": {
10
+ "role": { "$ref": "msm:common.schema.json#/$defs/role" },
11
+ "text": { "$ref": "#/$defs/textMatch" },
12
+ "label": { "$ref": "#/$defs/textMatch" },
13
+ "resourceId": { "type": "string" },
14
+ "state": {
15
+ "type": "object",
16
+ "additionalProperties": false,
17
+ "properties": {
18
+ "enabled": { "type": "boolean" },
19
+ "clickable": { "type": "boolean" },
20
+ "checked": { "type": "boolean" },
21
+ "selected": { "type": "boolean" },
22
+ "focused": { "type": "boolean" }
23
+ }
24
+ },
25
+ "within": {
26
+ "type": "object",
27
+ "additionalProperties": false,
28
+ "description": "Restrict to descendants of a matching ancestor.",
29
+ "properties": {
30
+ "role": { "$ref": "msm:common.schema.json#/$defs/role" },
31
+ "text": { "$ref": "#/$defs/textMatch" },
32
+ "resourceId": { "type": "string" }
33
+ }
34
+ },
35
+ "firstVisible": { "type": "boolean", "description": "Return only the first on-screen match." },
36
+ "nth": { "type": "integer", "minimum": 0, "description": "Zero-based index among ordered matches." }
37
+ },
38
+ "$defs": {
39
+ "textMatch": {
40
+ "type": "object",
41
+ "additionalProperties": false,
42
+ "required": ["value"],
43
+ "properties": {
44
+ "value": { "type": "string" },
45
+ "match": {
46
+ "enum": ["EXACT", "NORMALIZED_EXACT", "CONTAINS", "REGEX", "SEMANTIC_SYNONYM"],
47
+ "default": "NORMALIZED_EXACT",
48
+ "description": "REGEX is length/complexity limited. SEMANTIC_SYNONYM is opt-in only."
49
+ }
50
+ }
51
+ }
52
+ }
53
+ }
@@ -0,0 +1,50 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "msm:screen-snapshot.schema.json",
4
+ "title": "ScreenSnapshot",
5
+ "description": "A point-in-time semantic capture of the foreground window(s). May be full or compact.",
6
+ "type": "object",
7
+ "additionalProperties": false,
8
+ "required": [
9
+ "schemaVersion", "deviceId", "snapshotId", "timestamp",
10
+ "packageName", "windowId", "orientation", "screenStateId", "compact", "nodes", "treeHash"
11
+ ],
12
+ "properties": {
13
+ "schemaVersion": { "$ref": "msm:common.schema.json#/$defs/schemaVersion" },
14
+ "deviceId": { "$ref": "msm:common.schema.json#/$defs/deviceId" },
15
+ "snapshotId": { "$ref": "msm:common.schema.json#/$defs/snapshotId" },
16
+ "timestamp": { "$ref": "msm:common.schema.json#/$defs/timestampMs" },
17
+ "packageName": { "$ref": "msm:common.schema.json#/$defs/packageName" },
18
+ "windowId": { "type": "integer" },
19
+ "orientation": { "enum": ["portrait", "landscape"] },
20
+ "screenStateId": {
21
+ "type": "string",
22
+ "description": "Coarse screen identity, e.g. 'login:a91f' — role+structure hash. Used to detect same-screen."
23
+ },
24
+ "compact": {
25
+ "type": "boolean",
26
+ "description": "true => pruned to actionable/meaningful nodes (target < 30 KB). false => full raw tree."
27
+ },
28
+ "truncated": {
29
+ "type": "boolean",
30
+ "default": false,
31
+ "description": "true when node set was capped by a token/size budget."
32
+ },
33
+ "nodes": {
34
+ "type": "array",
35
+ "items": { "$ref": "msm:semantic-node.schema.json" }
36
+ },
37
+ "treeHash": {
38
+ "$ref": "msm:common.schema.json#/$defs/hash",
39
+ "description": "Hash of the whole tree; equal treeHash => identical semantic screen."
40
+ },
41
+ "platform": {
42
+ "enum": ["ANDROID", "IOS"],
43
+ "description": "Originating platform. Optional for back-compat; absent => ANDROID. iOS snapshots set IOS."
44
+ },
45
+ "appId": {
46
+ "type": ["string", "null"],
47
+ "description": "Platform-neutral foreground app id: Android package name, iOS bundle identifier. Mirrors packageName (the legacy Android alias)."
48
+ }
49
+ }
50
+ }
@@ -0,0 +1,95 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "msm:semantic-node.schema.json",
4
+ "title": "SemanticNode",
5
+ "description": "One normalized node in a ScreenSnapshot. nodeId is valid only within its snapshot. Wire economy: fields carrying their default value are OMITTED (state defaults, empty actions, source=ACCESSIBILITY, confidence=1). A reader MUST apply the documented default when a field is absent; absence never means unknown. Relaxing these from required is backward compatible — a payload that spells every field out still validates.",
6
+ "type": "object",
7
+ "additionalProperties": false,
8
+ "required": ["nodeId", "role"],
9
+ "properties": {
10
+ "nodeId": { "$ref": "msm:common.schema.json#/$defs/nodeId" },
11
+ "parentId": {
12
+ "oneOf": [
13
+ { "$ref": "msm:common.schema.json#/$defs/nodeId" },
14
+ { "type": "null" }
15
+ ],
16
+ "description": "Parent nodeId, or null for the snapshot root."
17
+ },
18
+ "role": { "$ref": "msm:common.schema.json#/$defs/role" },
19
+ "label": {
20
+ "type": ["string", "null"],
21
+ "description": "Best human label: contentDescription, or accessible name."
22
+ },
23
+ "text": { "type": ["string", "null"], "description": "Visible text content." },
24
+ "value": {
25
+ "type": ["string", "null"],
26
+ "description": "Current value (e.g. INPUT text, SLIDER position). NOT the same as text."
27
+ },
28
+ "hint": { "type": ["string", "null"], "description": "Placeholder / hint text." },
29
+ "resourceId": {
30
+ "type": ["string", "null"],
31
+ "description": "LEGACY Android locator alias (== automationId on Android). e.g. com.example.app:id/login. Strongest cheap locator when unique."
32
+ },
33
+ "className": {
34
+ "type": ["string", "null"],
35
+ "description": "Raw Android class, kept only for disambiguation; omitted from compact snapshot when not needed."
36
+ },
37
+ "automationId": {
38
+ "type": ["string", "null"],
39
+ "description": "Platform-neutral stable identifier: Android resource-id, iOS accessibilityIdentifier. Preferred cheap locator; resourceId is the Android alias."
40
+ },
41
+ "nativeType": {
42
+ "type": ["string", "null"],
43
+ "description": "Raw platform element type for disambiguation only, e.g. XCUIElementTypeButton on iOS (className is the Android equivalent)."
44
+ },
45
+ "state": {
46
+ "type": "object",
47
+ "additionalProperties": false,
48
+ "description": "Omitted entirely when every field is at its default. Defaults: enabled=true, everything else false, checked=null.",
49
+ "properties": {
50
+ "enabled": { "type": "boolean", "default": true },
51
+ "clickable": { "type": "boolean", "default": false },
52
+ "focused": { "type": "boolean", "default": false },
53
+ "selected": { "type": "boolean", "default": false },
54
+ "checked": { "type": ["boolean", "null"], "description": "null when not a checkable node." },
55
+ "password": { "type": "boolean", "default": false, "description": "true => value is redacted before leaving device." }
56
+ }
57
+ },
58
+ "actions": {
59
+ "type": "array",
60
+ "default": [],
61
+ "description": "Actions this node supports (subset of the action verb set). Omitted when empty.",
62
+ "items": { "type": "string" },
63
+ "uniqueItems": true
64
+ },
65
+ "bounds": {
66
+ "oneOf": [
67
+ { "$ref": "msm:common.schema.json#/$defs/bounds" },
68
+ { "type": "null" }
69
+ ],
70
+ "description": "Omitted from compact snapshot when a strong locator exists."
71
+ },
72
+ "collection": {
73
+ "type": ["object", "null"],
74
+ "additionalProperties": false,
75
+ "description": "Present on LIST_ITEM / collection children. Index is never sole identity (views recycle).",
76
+ "properties": {
77
+ "index": { "type": "integer", "minimum": 0 },
78
+ "itemKey": { "type": ["string", "null"], "description": "Stable item key/text used as identity." },
79
+ "rowCount": { "type": ["integer", "null"] },
80
+ "scrollContainerId": { "$ref": "msm:common.schema.json#/$defs/nodeId" },
81
+ "ancestorFingerprint": { "$ref": "msm:common.schema.json#/$defs/hash" }
82
+ }
83
+ },
84
+ "source": {
85
+ "allOf": [{ "$ref": "msm:common.schema.json#/$defs/source" }],
86
+ "default": "ACCESSIBILITY",
87
+ "description": "Omitted when ACCESSIBILITY. A VISION node always states source and confidence explicitly."
88
+ },
89
+ "confidence": {
90
+ "allOf": [{ "$ref": "msm:common.schema.json#/$defs/confidence" }],
91
+ "default": 1,
92
+ "description": "Omitted when 1."
93
+ }
94
+ }
95
+ }