mobile-debug-mcp 0.32.2 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +20 -0
- package/{skills → .agents/skills}/mcp-builder/SKILL.md +1 -2
- package/{skills → .agents/skills}/spec-review/SKILL.md +1 -1
- package/{skills → .agents/skills}/test-authoring/SKILL.md +1 -1
- package/.claude-plugin/marketplace.json +14 -0
- package/.cursor-plugin/marketplace.json +16 -0
- package/.github/copilot-instructions.md +7 -0
- package/AGENTS.md +21 -10
- package/README.md +67 -5
- package/agent-plugin/.claude-plugin/plugin.json +20 -0
- package/agent-plugin/.codex-plugin/plugin.json +34 -0
- package/agent-plugin/.cursor-plugin/plugin.json +23 -0
- package/agent-plugin/.mcp.json +14 -0
- package/agent-plugin/LICENSE +201 -0
- package/agent-plugin/README.md +29 -0
- package/agent-plugin/assets/mobile-debug-tools.svg +28 -0
- package/agent-plugin/mcp.json +11 -0
- package/agent-plugin/plugin.json +29 -0
- package/agent-plugin/skills/mobile-debug-mcp/SKILL.md +124 -0
- package/dist/interact/android.js +10 -4
- package/dist/interact/conditional-action.js +68 -0
- package/dist/interact/index.js +54 -14
- package/dist/interact/ios.js +14 -8
- package/dist/manage/android.js +4 -3
- package/dist/manage/ios.js +3 -2
- package/dist/observe/android.js +27 -22
- package/dist/observe/evidence.js +273 -0
- package/dist/observe/index.js +33 -6
- package/dist/observe/ios.js +25 -6
- package/dist/observe/readiness-service.js +115 -0
- package/dist/observe/readiness.js +81 -0
- package/dist/observe/recording.js +165 -0
- package/dist/observe/snapshot-metadata.js +10 -2
- package/dist/server/common.js +9 -0
- package/dist/server/condition-schema.js +32 -0
- package/dist/server/journey-schema.js +3 -2
- package/dist/server/readiness-schema.js +12 -0
- package/dist/server/tool-definitions.js +46 -0
- package/dist/server/tool-handlers.js +99 -17
- package/dist/server-core.js +1 -1
- package/dist/utils/ios/utils.js +4 -4
- package/dist/utils/operation-budget.js +24 -2
- package/dist/utils/resolve-device.js +7 -0
- package/docs/CHANGELOG.md +16 -2
- package/docs/benchmarks/ui-tree-performance/README.md +4 -3
- package/docs/specs/020-conditional-actions/intent.md +62 -0
- package/docs/specs/020-conditional-actions/plan.md +214 -0
- package/docs/specs/020-conditional-actions/spec.md +301 -0
- package/docs/specs/021-app-readiness-signals/intent.md +40 -0
- package/docs/specs/021-app-readiness-signals/plan.md +102 -0
- package/docs/specs/021-app-readiness-signals/spec.md +144 -0
- package/docs/specs/022-correlated-evidence/device-evidence.json +60 -0
- package/docs/specs/022-correlated-evidence/intent.md +37 -0
- package/docs/specs/022-correlated-evidence/modul8-evidence.json +1062 -0
- package/docs/specs/022-correlated-evidence/modul8-selector-fix-evidence.json +336 -0
- package/docs/specs/022-correlated-evidence/plan.md +171 -0
- package/docs/specs/022-correlated-evidence/spec.md +123 -0
- package/docs/specs/process.md +213 -0
- package/docs/tools/TOOLS.md +3 -0
- package/docs/tools/evidence.md +79 -0
- package/docs/tools/interact.md +125 -2
- package/docs/tools/observe.md +64 -2
- package/package.json +4 -2
- package/scripts/agent-plugin.mjs +98 -0
- package/src/interact/android.ts +10 -4
- package/src/interact/conditional-action.ts +75 -0
- package/src/interact/index.ts +42 -14
- package/src/interact/ios.ts +14 -8
- package/src/manage/android.ts +4 -3
- package/src/manage/ios.ts +3 -2
- package/src/observe/android.ts +35 -21
- package/src/observe/evidence.ts +255 -0
- package/src/observe/index.ts +32 -8
- package/src/observe/ios.ts +32 -8
- package/src/observe/readiness-service.ts +89 -0
- package/src/observe/readiness.ts +78 -0
- package/src/observe/snapshot-metadata.ts +12 -1
- package/src/server/common.ts +7 -0
- package/src/server/condition-schema.ts +32 -0
- package/src/server/journey-schema.ts +3 -2
- package/src/server/readiness-schema.ts +12 -0
- package/src/server/tool-definitions.ts +44 -0
- package/src/server/tool-handlers.ts +93 -17
- package/src/server-core.ts +1 -1
- package/src/types.ts +30 -1
- package/src/utils/ios/utils.ts +4 -4
- package/src/utils/operation-budget.ts +22 -2
- package/src/utils/resolve-device.ts +8 -0
- package/test/device/automated/observe/correlated-evidence.smoke.ts +70 -0
- package/test/device/manual/interact/conditional_actions.manual.ts +95 -0
- package/test/device/manual/interact/modul8_selector_tap.manual.ts +63 -0
- package/test/device/manual/interact/readiness.manual.ts +99 -0
- package/test/fixtures/readiness-app/MainActivity.java +23 -0
- package/test/fixtures/readiness-app/README.md +12 -0
- package/test/fixtures/readiness-app/build.ts +37 -0
- package/test/fixtures/readiness-app/main.m +27 -0
- package/test/unit/observe/android_tree_acquisition.test.ts +6 -0
- package/test/unit/observe/capture_debug_snapshot.test.ts +42 -32
- package/test/unit/observe/compact_tree_error.test.ts +24 -0
- package/test/unit/observe/conditional_snapshots.test.ts +33 -0
- package/test/unit/observe/correlated-evidence.test.ts +128 -0
- package/test/unit/observe/ios-getlogs.test.ts +3 -3
- package/test/unit/observe/ios_tree_acquisition.test.ts +2 -0
- package/test/unit/observe/readiness.test.ts +39 -0
- package/test/unit/observe/readiness_native.test.ts +68 -0
- package/test/unit/server/conditional_actions.test.ts +299 -0
- package/test/unit/server/conditional_schema.test.ts +59 -0
- package/test/unit/server/correlated-evidence.test.ts +194 -0
- package/test/unit/server/readiness.test.ts +170 -0
- package/test/unit/server/selector_tap_target.test.ts +79 -0
- package/test/unit/server/tap_selector.test.ts +2 -0
- package/skills/mobile-debug-tools-usage/SKILL.md +0 -74
- package/src/observe/android-tree-bridge.ts +0 -178
- package/test/device/automated/ui-tree-performance/bridge-probe.ts +0 -197
- package/test/fixtures/ui-tree-bridge/AndroidManifest.xml +0 -6
- package/test/fixtures/ui-tree-bridge/build.ts +0 -34
- package/test/fixtures/ui-tree-bridge/src/TreeInstrumentation.java +0 -126
- package/test/unit/observe/android_tree_bridge_cleanup.test.ts +0 -95
- /package/{skills → .agents/skills}/README.md +0 -0
- /package/{skills → .agents/skills}/mcp-builder/references/build-flags.md +0 -0
- /package/{skills → .agents/skills}/mcp-builder/references/diagnostics-schema.md +0 -0
- /package/{skills → .agents/skills}/mcp-builder/references/toolchain-details.md +0 -0
- /package/{skills → .agents/skills}/spec-review/references/spec-review-checklist.md +0 -0
- /package/{skills → .agents/skills}/spec-review/references/spec-review-template.md +0 -0
- /package/{skills → .agents/skills}/test-authoring/references/repo-test-layout.md +0 -0
- /package/{skills → .agents/skills}/test-authoring/references/test-authoring-workflow.md +0 -0
- /package/{skills → .agents/skills}/test-authoring/references/test-quality-checklist.md +0 -0
package/src/server/common.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { currentEvidenceActionId } from '../observe/evidence.js'
|
|
1
2
|
import { recordTiming } from '../utils/timing.js'
|
|
2
3
|
import type {
|
|
3
4
|
ActionTrace,
|
|
@@ -11,6 +12,7 @@ import type {
|
|
|
11
12
|
TraceStep
|
|
12
13
|
} from '../types.js'
|
|
13
14
|
import { ToolsObserve } from '../observe/index.js'
|
|
15
|
+
import { isConditionBlocked } from '../interact/conditional-action.js'
|
|
14
16
|
|
|
15
17
|
export const DEFAULT_MAX_RECOVERY_ATTEMPTS = 3
|
|
16
18
|
export const DEFAULT_MAX_RETRY_DEPTH = 3
|
|
@@ -87,11 +89,14 @@ export function getArrayArg<T>(args: ToolCallArgs, key: string): T[] | undefined
|
|
|
87
89
|
let actionSequence = 0
|
|
88
90
|
|
|
89
91
|
export function nextActionId(actionType: string, timestamp: number) {
|
|
92
|
+
const current = currentEvidenceActionId()
|
|
93
|
+
if (current) return current
|
|
90
94
|
actionSequence += 1
|
|
91
95
|
return `${actionType}_${timestamp}_${actionSequence}`
|
|
92
96
|
}
|
|
93
97
|
|
|
94
98
|
export async function captureActionFingerprint(platform?: 'android' | 'ios', deviceId?: string): Promise<string | null> {
|
|
99
|
+
if (isConditionBlocked()) return null
|
|
95
100
|
if (!platform) return null
|
|
96
101
|
try {
|
|
97
102
|
const result = await ToolsObserve.getScreenFingerprintHandler({ platform, deviceId }) as any
|
|
@@ -317,6 +322,8 @@ function mapFailureCodeToFailureClass(code: ActionFailureCode): FailureClass {
|
|
|
317
322
|
case 'STALE_REFERENCE':
|
|
318
323
|
return 'TargetResolutionFailure'
|
|
319
324
|
case 'ELEMENT_NOT_INTERACTABLE':
|
|
325
|
+
case 'PRECONDITION_FAILED':
|
|
326
|
+
case 'PRECONDITION_UNAVAILABLE':
|
|
320
327
|
case 'ACTION_TIMEOUT':
|
|
321
328
|
return 'ExecutionFailure'
|
|
322
329
|
case 'TIMEOUT':
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
|
|
3
|
+
export const conditionalTools = new Set(['tap', 'tap_element', 'swipe', 'type_text', 'press_back'])
|
|
4
|
+
export const expectedStateSchema = z.object({
|
|
5
|
+
snapshotSessionId: z.string().min(1),
|
|
6
|
+
snapshotRevision: z.number().int().positive().max(Number.MAX_SAFE_INTEGER),
|
|
7
|
+
timeoutMs: z.number().int().min(100).max(60000).optional()
|
|
8
|
+
}).strict()
|
|
9
|
+
|
|
10
|
+
export type ExpectedState = z.infer<typeof expectedStateSchema>
|
|
11
|
+
|
|
12
|
+
export const expectedStateJson = {
|
|
13
|
+
type: 'object', additionalProperties: false,
|
|
14
|
+
properties: {
|
|
15
|
+
snapshotSessionId: { type: 'string', minLength: 1 },
|
|
16
|
+
snapshotRevision: { type: 'integer', minimum: 1, maximum: Number.MAX_SAFE_INTEGER },
|
|
17
|
+
timeoutMs: { type: 'integer', minimum: 100, maximum: 60000, default: 15000 }
|
|
18
|
+
},
|
|
19
|
+
required: ['snapshotSessionId', 'snapshotRevision']
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function validateExpectedState(name: string, args: Record<string, unknown>): ExpectedState | undefined {
|
|
23
|
+
if (args.expectedState === undefined) return undefined
|
|
24
|
+
if (!conditionalTools.has(name)) throw new Error('INVALID_ARGUMENT: expectedState is not supported on this tool')
|
|
25
|
+
const parsed = expectedStateSchema.safeParse(args.expectedState)
|
|
26
|
+
if (!parsed.success) throw new Error('INVALID_ARGUMENT: invalid expectedState')
|
|
27
|
+
if ((args.platform !== 'android' && args.platform !== 'ios') ||
|
|
28
|
+
typeof args.deviceId !== 'string' || !args.deviceId.trim() ||
|
|
29
|
+
['booted', 'default'].includes(args.deviceId)) throw new Error('INVALID_ARGUMENT: expectedState requires an explicit platform and concrete deviceId')
|
|
30
|
+
if (['type_text', 'press_back'].includes(name) && args.platform !== 'android') throw new Error('INVALID_ARGUMENT: this action supports only Android')
|
|
31
|
+
return parsed.data
|
|
32
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod'
|
|
2
|
+
import { expectedStateSchema, expectedStateJson } from './condition-schema.js'
|
|
2
3
|
|
|
3
4
|
const selector = z.object({
|
|
4
5
|
text: z.string().min(1).optional(), resource_id: z.string().min(1).optional(),
|
|
@@ -23,7 +24,7 @@ const assertion = z.union([
|
|
|
23
24
|
const id = z.string().min(1)
|
|
24
25
|
const step = z.union([
|
|
25
26
|
z.object({ id, type: z.literal('start_app'), appId: z.string().min(1), verificationMode: verificationMode.optional() }).strict(),
|
|
26
|
-
z.object({ id, type: z.literal('tap'), ...target, waitFor: waitFor.optional(), verificationMode: verificationMode.optional() }).strict().refine(exactlyOneTarget),
|
|
27
|
+
z.object({ id, type: z.literal('tap'), ...target, waitFor: waitFor.optional(), verificationMode: verificationMode.optional(), expectedState: expectedStateSchema.optional() }).strict().refine(exactlyOneTarget),
|
|
27
28
|
z.object({ id, type: z.literal('wait'), selector, condition: z.enum(['exists', 'not_exists', 'visible', 'clickable']).optional(), timeoutMs, pollIntervalMs }).strict(),
|
|
28
29
|
z.object({ id, type: z.literal('assert'), assertion, timeoutMs, pollIntervalMs }).strict()
|
|
29
30
|
])
|
|
@@ -49,7 +50,7 @@ const objectJson = (properties: Record<string, unknown>, required: string[], ext
|
|
|
49
50
|
export const journeyStepsJson = {
|
|
50
51
|
type: 'array', minItems: 1, maxItems: 50, items: { oneOf: [
|
|
51
52
|
objectJson({ id: { type: 'string', minLength: 1 }, type: { const: 'start_app' }, appId: { type: 'string', minLength: 1 }, verificationMode: modeJson }, ['id', 'type', 'appId']),
|
|
52
|
-
objectJson({ id: { type: 'string', minLength: 1 }, type: { const: 'tap' }, ...targetJson, verificationMode: modeJson,
|
|
53
|
+
objectJson({ id: { type: 'string', minLength: 1 }, type: { const: 'tap' }, ...targetJson, verificationMode: modeJson, expectedState: expectedStateJson,
|
|
53
54
|
waitFor: objectJson({ ...intervalJson, timeoutMs: { type: 'integer', minimum: 100, maximum: 10000 }, condition: { enum: ['exists', 'visible', 'clickable'] }, match: objectJson({ index: { type: 'integer', minimum: 0 } }, []) }, [])
|
|
54
55
|
}, ['id', 'type'], { oneOf: oneTargetJson }),
|
|
55
56
|
objectJson({ id: { type: 'string', minLength: 1 }, type: { const: 'wait' }, selector: selectorJson, condition: { enum: ['exists', 'not_exists', 'visible', 'clickable'] }, ...intervalJson }, ['id', 'type', 'selector']),
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
const timeout = z.number().int().min(100).max(60000)
|
|
3
|
+
export const readinessSchema = z.object({
|
|
4
|
+
platform: z.enum(['android', 'ios']), deviceId: z.string().min(1).optional(),
|
|
5
|
+
timeoutMs: timeout.optional(), stabilityMs: z.number().int().min(100).max(10000).optional()
|
|
6
|
+
}).strict()
|
|
7
|
+
export const readinessWaitSchema = readinessSchema.extend({ pollIntervalMs: z.number().int().min(100).max(1000).optional() }).strict()
|
|
8
|
+
export const readinessPropertiesJson = {
|
|
9
|
+
platform: { type: 'string', enum: ['android', 'ios'] }, deviceId: { type: 'string', minLength: 1 },
|
|
10
|
+
timeoutMs: { type: 'integer', minimum: 100, maximum: 60000 },
|
|
11
|
+
stabilityMs: { type: 'integer', minimum: 100, maximum: 10000, default: 500 }
|
|
12
|
+
}
|
|
@@ -1,6 +1,17 @@
|
|
|
1
1
|
import { journeyStepsJson } from './journey-schema.js'
|
|
2
|
+
import { conditionalTools, expectedStateJson } from './condition-schema.js'
|
|
3
|
+
import { readinessPropertiesJson } from './readiness-schema.js'
|
|
2
4
|
|
|
3
5
|
export const toolDefinitions: any[] = [
|
|
6
|
+
...['get_app_readiness', 'wait_for_app_ready'].map(name => ({
|
|
7
|
+
name,
|
|
8
|
+
description: `${name === 'get_app_readiness' ? 'Observe readiness with at most two fresh UI-tree samples.' : 'Wait within a single bounded budget for settled UI evidence.'} Reports loading, transitioning, settled or unknown with evidence and limitations. Unresponsive/debugger-paused detection is unsupported. Settled normalized foreground UI is not proof of app identity, responsiveness or backend completion.`,
|
|
9
|
+
inputSchema: { type: 'object', additionalProperties: false, required: ['platform'], properties: {
|
|
10
|
+
...readinessPropertiesJson,
|
|
11
|
+
timeoutMs: { ...readinessPropertiesJson.timeoutMs, default: name === 'get_app_readiness' ? 15000 : 30000 },
|
|
12
|
+
...(name === 'wait_for_app_ready' ? { pollIntervalMs: { type: 'integer', minimum: 100, maximum: 1000, default: 500 } } : {})
|
|
13
|
+
} }
|
|
14
|
+
})),
|
|
4
15
|
{
|
|
5
16
|
name: 'start_app',
|
|
6
17
|
description: `Purpose:
|
|
@@ -1130,6 +1141,26 @@ Returns { requests: [], count: 0 } when no credible network signals are found.`,
|
|
|
1130
1141
|
|
|
1131
1142
|
// Keep the additive execution controls consistent across all action tools.
|
|
1132
1143
|
for (const tool of toolDefinitions) {
|
|
1144
|
+
if (['wait_for_ui', 'wait_for_ui_change'].includes(tool.name)) {
|
|
1145
|
+
tool.inputSchema.properties.requireReady = { type: 'boolean', default: false }
|
|
1146
|
+
tool.description += '\nSet requireReady=true to require both the existing predicate and settled UI readiness, using the same fresh samples and one timeout budget. The readiness summary explains uncertainty. Default predicate semantics are unchanged.'
|
|
1147
|
+
}
|
|
1148
|
+
if (tool.name === 'get_ui_tree') tool.description += '\nSuccessful observations include snapshot_session_id and snapshot_revision for same-device conditional input. Compact errors preserve the error and cannot authorize an expectation.'
|
|
1149
|
+
if (tool.name === 'run_journey') tool.description += '\nTap steps may include expectedState from a successful same-device get_ui_tree. A failed condition blocks that input, retains fresh condition evidence even in compact mode, and stops subsequent steps. Conditions are not journey defaults or supported on other step types.'
|
|
1150
|
+
if (conditionalTools.has(tool.name)) {
|
|
1151
|
+
tool.inputSchema.properties.expectedState = expectedStateJson
|
|
1152
|
+
tool.inputSchema.properties.platform ??= { type: 'string', enum: ['android'] }
|
|
1153
|
+
tool.inputSchema.allOf = [...(tool.inputSchema.allOf ?? []), {
|
|
1154
|
+
if: { required: ['expectedState'] },
|
|
1155
|
+
then: { required: ['platform', 'deviceId'], properties: {
|
|
1156
|
+
platform: { enum: ['type_text', 'press_back'].includes(tool.name) ? ['android'] : ['android', 'ios'] },
|
|
1157
|
+
deviceId: { type: 'string', minLength: 1, pattern: '\\S', not: { enum: ['booted', 'default'] } }
|
|
1158
|
+
} }
|
|
1159
|
+
}]
|
|
1160
|
+
tool.description += '\n\nConditional input: optionally supply expectedState { snapshotSessionId, snapshotRevision, timeoutMs? } from a successful get_ui_tree on the same explicit platform/deviceId. A fresh normalized UI-tree check runs immediately before input, even with verification disabled. PRECONDITION_FAILED or PRECONDITION_UNAVAILABLE means no input was dispatched; inspect condition evidence and re-observe before deciding what to do next. Compact failures include the fresh UI tree. This is not atomic with OS input delivery.'
|
|
1161
|
+
} else {
|
|
1162
|
+
tool.inputSchema.allOf = [...(tool.inputSchema.allOf ?? []), { not: { required: ['expectedState'] } }]
|
|
1163
|
+
}
|
|
1133
1164
|
if (['start_app', 'restart_app', 'tap', 'tap_element', 'swipe', 'scroll_to_element', 'type_text', 'press_back'].includes(tool.name)) {
|
|
1134
1165
|
Object.assign(tool.inputSchema.properties, {
|
|
1135
1166
|
verificationMode: { type: 'string', enum: ['none', 'light', 'full'], default: 'full' },
|
|
@@ -1148,3 +1179,16 @@ for (const tool of toolDefinitions) {
|
|
|
1148
1179
|
} }
|
|
1149
1180
|
})
|
|
1150
1181
|
}
|
|
1182
|
+
|
|
1183
|
+
// Correlation is explicit; an omitted reference creates an independent observation.
|
|
1184
|
+
for (const tool of toolDefinitions) {
|
|
1185
|
+
if (['capture_screenshot', 'get_ui_tree', 'get_screen_fingerprint', 'capture_debug_snapshot', 'get_logs', 'read_log_stream'].includes(tool.name)) {
|
|
1186
|
+
Object.assign(tool.inputSchema.properties, { actionRef: {
|
|
1187
|
+
type: 'object', additionalProperties: false, required: ['sessionId', 'actionId'], properties: {
|
|
1188
|
+
sessionId: { type: 'string', minLength: 1, maxLength: 128 },
|
|
1189
|
+
actionId: { type: 'string', minLength: 1, maxLength: 128 }
|
|
1190
|
+
}
|
|
1191
|
+
} })
|
|
1192
|
+
tool.description += '\nReturns evidence metadata with acquisition intervals and conservative freshness. Optional actionRef links a known same-session/device action; unavailable references collect independently. Device log times are not comparable with host dispatch times.'
|
|
1193
|
+
}
|
|
1194
|
+
}
|
|
@@ -1,7 +1,12 @@
|
|
|
1
|
+
import { withEvidence, currentEvidenceActionId, validateActionRef, markEvidenceVerification, refreshJourneyEvidence, completeJourneyVerification } from '../observe/evidence.js'
|
|
1
2
|
import { measure, withTiming } from '../utils/timing.js'
|
|
2
3
|
import { randomUUID } from 'node:crypto'
|
|
3
4
|
import { runWithBudget } from '../utils/operation-budget.js'
|
|
4
5
|
import { journeySchema, tapElementControlsSchema } from './journey-schema.js'
|
|
6
|
+
import { readinessSchema, readinessWaitSchema } from './readiness-schema.js'
|
|
7
|
+
import { getAppReadiness, withReadyWait } from '../observe/readiness-service.js'
|
|
8
|
+
import { validateExpectedState } from './condition-schema.js'
|
|
9
|
+
import { conditionFailureCode, createConditionalAction, notifyActionStart, withConditionalAction } from '../interact/conditional-action.js'
|
|
5
10
|
import { resolveTargetDevice, withResolvedDevice } from '../utils/resolve-device.js'
|
|
6
11
|
import type {
|
|
7
12
|
InstallAppResponse,
|
|
@@ -76,17 +81,42 @@ function compactAction(result: any, verificationMode: VerificationMode, started
|
|
|
76
81
|
}
|
|
77
82
|
}
|
|
78
83
|
|
|
84
|
+
function correlatedHandler(name: string, handler: ToolHandler, action = false, journey = false): ToolHandler {
|
|
85
|
+
return async args => {
|
|
86
|
+
const ref = validateActionRef(args.actionRef)
|
|
87
|
+
const { result, evidence } = await withEvidence({ action, journey, ref }, async () => {
|
|
88
|
+
try { return await handler(args) }
|
|
89
|
+
catch (error) { return wrapToolError(name, error) }
|
|
90
|
+
})
|
|
91
|
+
const item = result.content.find(item => item.type === 'text')
|
|
92
|
+
if (item?.text) {
|
|
93
|
+
const payload = JSON.parse(item.text)
|
|
94
|
+
payload.evidence = evidence
|
|
95
|
+
item.text = JSON.stringify(payload)
|
|
96
|
+
}
|
|
97
|
+
return result
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
79
101
|
function actionHandler(type: string, execute: ToolHandler): ToolHandler {
|
|
80
|
-
return async (args) => {
|
|
102
|
+
return correlatedHandler(type, async (args) => {
|
|
81
103
|
const timing: Record<string, number> = {}
|
|
82
104
|
return withTiming(timing, async () => {
|
|
83
105
|
const started = performance.now()
|
|
84
106
|
const controls = interactionControls(args)
|
|
107
|
+
const expected = validateExpectedState(type, args)
|
|
108
|
+
const condition = expected ? createConditionalAction(args.platform as PlatformArg, args.deviceId as string, expected) : undefined
|
|
85
109
|
const budgetMs = controls.actionTimeoutMs ?? ({ start_app: 10000, restart_app: 15000, scroll_to_element: 10000 }[type] ?? 5000)
|
|
86
110
|
let dispatchStarted = false
|
|
87
111
|
let result: any
|
|
88
112
|
const dispatchWithBudget = (timeoutMs: number) => runWithBudget(timeoutMs, async (budget) => {
|
|
89
|
-
try {
|
|
113
|
+
try {
|
|
114
|
+
if (!condition) return payloadOf(await execute(args))
|
|
115
|
+
return await withConditionalAction(condition, async () => {
|
|
116
|
+
const device = await resolveTargetDevice({ platform: condition.platform, deviceId: condition.deviceId })
|
|
117
|
+
return withResolvedDevice(device, async () => payloadOf(await execute(args)))
|
|
118
|
+
})
|
|
119
|
+
}
|
|
90
120
|
finally { dispatchStarted = budget.dispatchStarted }
|
|
91
121
|
})
|
|
92
122
|
try {
|
|
@@ -95,13 +125,24 @@ function actionHandler(type: string, execute: ToolHandler): ToolHandler {
|
|
|
95
125
|
? await dispatchWithBudget(Infinity) : await dispatchWithBudget(budgetMs)
|
|
96
126
|
if (result.success) dispatchStarted = true
|
|
97
127
|
} catch (error) {
|
|
98
|
-
if (controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined && !/TIMEOUT|timed out/i.test(String(error))) return wrapToolError(type, error)
|
|
128
|
+
if (!condition && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined && !/TIMEOUT|timed out/i.test(String(error))) return wrapToolError(type, error)
|
|
99
129
|
result = { success: false, failure_code: /TIMEOUT|timed out/i.test(String(error)) ? 'ACTION_TIMEOUT' : 'UNKNOWN', error: String(error) }
|
|
100
130
|
}
|
|
131
|
+
const blocked = condition && ['mismatch', 'unavailable'].includes(condition.result.status)
|
|
132
|
+
if (blocked) {
|
|
133
|
+
dispatchStarted = false
|
|
134
|
+
result = buildActionExecutionResult({
|
|
135
|
+
actionType: type, sourceModule: 'server', device: condition.result.observed?.device,
|
|
136
|
+
selector: result?.target?.selector ?? null, resolved: result?.target?.resolved,
|
|
137
|
+
success: false, uiFingerprintBefore: result?.ui_fingerprint_before ?? null, uiFingerprintAfter: null,
|
|
138
|
+
failure: { failureCode: conditionFailureCode(condition), retryable: false },
|
|
139
|
+
traceSteps: [{ stage: 'verify', timestamp: Date.now(), result: 'failure', attempt_index: 0, metadata: { phase: 'precondition', dispatch_started: false } }]
|
|
140
|
+
})
|
|
141
|
+
}
|
|
101
142
|
if (result.failure_code === 'TIMEOUT' || /ACTION_TIMEOUT|timeout/i.test(result.error ?? result.reason ?? result.details?.error ?? '')) result.failure_code = 'ACTION_TIMEOUT'
|
|
102
143
|
if (result.failure_code === 'ACTION_TIMEOUT') result.retryable = !dispatchStarted
|
|
103
144
|
if (!result.success && typeof result.retryable !== 'boolean') result.retryable = !dispatchStarted
|
|
104
|
-
result.action_id ??= randomUUID()
|
|
145
|
+
result.action_id ??= currentEvidenceActionId() ?? randomUUID()
|
|
105
146
|
result.action_type ??= type
|
|
106
147
|
recordObservationEvent({
|
|
107
148
|
step_id: args._stepId as string | undefined,
|
|
@@ -117,6 +158,7 @@ function actionHandler(type: string, execute: ToolHandler): ToolHandler {
|
|
|
117
158
|
const tree = await withObservationContext({ stepId: args._stepId as string | undefined, purpose: 'verification' }, () =>
|
|
118
159
|
measure('post_observation_ms', () => runWithBudget(controls.verificationTimeoutMs, () =>
|
|
119
160
|
ToolsObserve.getUITreeHandler({ platform: (args.platform as PlatformArg | undefined) ?? result.device?.platform, deviceId: (args.deviceId as string | undefined) ?? result.device?.id })))).catch(() => null) as any
|
|
161
|
+
markEvidenceVerification()
|
|
120
162
|
observationAvailable = !!tree && !tree.error
|
|
121
163
|
recordObservationEvent({
|
|
122
164
|
verification_budget_ms: controls.verificationTimeoutMs,
|
|
@@ -126,10 +168,18 @@ function actionHandler(type: string, execute: ToolHandler): ToolHandler {
|
|
|
126
168
|
})
|
|
127
169
|
if (observationAvailable && tree.device) delta = ToolsObserve.getLatestStateDelta(tree.device.platform, tree.device.id) ?? undefined
|
|
128
170
|
}
|
|
171
|
+
if (!blocked && controls.verificationMode === 'full' && dispatchStarted) markEvidenceVerification()
|
|
129
172
|
const compact = compactAction(result, controls.verificationMode, dispatchStarted, delta, performance.now() - started)
|
|
173
|
+
if (blocked) compact.verification.status = 'not_requested'
|
|
130
174
|
if (result.success && controls.verificationMode === 'light' && !observationAvailable) compact.verification.status = 'unavailable'
|
|
131
175
|
const evidence = {
|
|
132
176
|
...compact,
|
|
177
|
+
...(condition ? { condition: controls.responseMode === 'compact' && condition.result.status === 'matched'
|
|
178
|
+
? { ...condition.result, observed: condition.result.observed ? {
|
|
179
|
+
snapshot_session_id: condition.result.observed.snapshot_session_id,
|
|
180
|
+
snapshot_revision: condition.result.observed.snapshot_revision,
|
|
181
|
+
captured_at_ms: condition.result.observed.captured_at_ms
|
|
182
|
+
} : null } : condition.result } : {}),
|
|
133
183
|
...(controls.verificationMode === 'light' ? { state_delta_available: !!delta } : {}),
|
|
134
184
|
...(result.success && controls.verificationMode === 'light' && !observationAvailable ? { verification_diagnostic: { code: 'VERIFICATION_UNAVAILABLE', retryable: true } } : {})
|
|
135
185
|
}
|
|
@@ -138,7 +188,7 @@ function actionHandler(type: string, execute: ToolHandler): ToolHandler {
|
|
|
138
188
|
effective_controls: { ...controls, actionTimeoutMs: args._journey !== true && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined ? null : budgetMs, timeout_bound: !(args._journey !== true && controls.verificationMode === 'full' && controls.responseMode === 'debug' && controls.actionTimeoutMs === undefined) }
|
|
139
189
|
}, controls.responseMode === 'debug')
|
|
140
190
|
})
|
|
141
|
-
}
|
|
191
|
+
}, true)
|
|
142
192
|
}
|
|
143
193
|
const handleStartApp = actionHandler('start_app', executeStartApp)
|
|
144
194
|
const handleRestartApp = actionHandler('restart_app', executeRestartApp)
|
|
@@ -155,7 +205,7 @@ async function executeStartApp(args: ToolCallArgs) {
|
|
|
155
205
|
const deviceId = getStringArg(args, 'deviceId')
|
|
156
206
|
const { verificationMode, actionTimeoutMs } = interactionControls(args)
|
|
157
207
|
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
158
|
-
|
|
208
|
+
notifyActionStart()
|
|
159
209
|
const res = await measure('dispatch_ms', () => (platform === 'android' ? new AndroidManage().startApp(appId, deviceId, actionTimeoutMs ?? (verificationMode === 'full' && getStringArg(args, 'responseMode') === undefined && args._journey !== true ? undefined : 10000)) : new iOSManage().startApp(appId, deviceId, verificationMode === 'full')))
|
|
160
210
|
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
161
211
|
const result = buildActionExecutionResult({
|
|
@@ -194,7 +244,7 @@ async function executeRestartApp(args: ToolCallArgs) {
|
|
|
194
244
|
const deviceId = getStringArg(args, 'deviceId')
|
|
195
245
|
const { verificationMode, actionTimeoutMs } = interactionControls(args)
|
|
196
246
|
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
197
|
-
|
|
247
|
+
notifyActionStart()
|
|
198
248
|
const bounded = args._journey === true || verificationMode !== 'full' || args.responseMode !== undefined || actionTimeoutMs !== undefined
|
|
199
249
|
const res = await measure('dispatch_ms', () => (platform === 'android' ? new AndroidManage().restartApp(appId, deviceId, bounded, actionTimeoutMs ?? 15000) : new iOSManage().restartApp(appId, deviceId, verificationMode === 'full', bounded, actionTimeoutMs ?? 15000)))
|
|
200
250
|
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
@@ -425,10 +475,12 @@ async function handleAdjustControl(args: ToolCallArgs) {
|
|
|
425
475
|
platform,
|
|
426
476
|
deviceId
|
|
427
477
|
})
|
|
478
|
+
markEvidenceVerification()
|
|
428
479
|
return wrapResponse(res)
|
|
429
480
|
}
|
|
430
481
|
|
|
431
482
|
async function handleWaitForUI(args: ToolCallArgs) {
|
|
483
|
+
if (args.requireReady !== undefined && typeof args.requireReady !== 'boolean') throw new Error('INVALID_ARGUMENT: requireReady must be boolean')
|
|
432
484
|
const selector = getObjectArg<ExpectElementSelectorArg>(args, 'selector')
|
|
433
485
|
const condition = (getStringArg(args, 'condition') as 'exists' | 'not_exists' | 'visible' | 'clickable' | undefined) ?? 'exists'
|
|
434
486
|
const timeout_ms = getNumberArg(args, 'timeout_ms') ?? 60000
|
|
@@ -437,11 +489,14 @@ async function handleWaitForUI(args: ToolCallArgs) {
|
|
|
437
489
|
const retry = getObjectArg<WaitForUiRetryArg>(args, 'retry')
|
|
438
490
|
const platform = getStringArg(args, 'platform') as PlatformArg | undefined
|
|
439
491
|
const deviceId = getStringArg(args, 'deviceId')
|
|
440
|
-
const
|
|
492
|
+
const execute = (targetPlatform = platform, targetId = deviceId) => ToolsInteract.waitForUIHandler({ selector, condition, timeout_ms, poll_interval_ms, match, retry, platform: targetPlatform, deviceId: targetId })
|
|
493
|
+
const res = args.requireReady === true ? await withReadyWait({ platform, deviceId }, timeout_ms,
|
|
494
|
+
device => execute(device.platform as PlatformArg, device.id), (code, elapsed, samples) => ({ status: 'timeout', error: { code, message: 'Readiness wait could not complete' }, metrics: { latency_ms: elapsed, poll_count: samples, attempts: samples ? 1 : 0 }, requested: { selector, condition, match }, observed: { matched_count: 0, condition_satisfied: false, selected_index: null, last_matched_element: null } })) : await execute()
|
|
441
495
|
return wrapResponse(res)
|
|
442
496
|
}
|
|
443
497
|
|
|
444
498
|
async function handleWaitForUIChange(args: ToolCallArgs) {
|
|
499
|
+
if (args.requireReady !== undefined && typeof args.requireReady !== 'boolean') throw new Error('INVALID_ARGUMENT: requireReady must be boolean')
|
|
445
500
|
const platform = getStringArg(args, 'platform') as PlatformArg | undefined
|
|
446
501
|
const deviceId = getStringArg(args, 'deviceId')
|
|
447
502
|
const timeout_ms = getNumberArg(args, 'timeout_ms') ?? 60000
|
|
@@ -449,7 +504,9 @@ async function handleWaitForUIChange(args: ToolCallArgs) {
|
|
|
449
504
|
const expected_change = getStringArg(args, 'expected_change') as 'hierarchy_diff' | 'text_change' | 'state_change' | undefined
|
|
450
505
|
const scope = getStringArg(args, 'scope') as 'screen' | 'subtree' | undefined
|
|
451
506
|
const target = getStringArg(args, 'target')
|
|
452
|
-
const
|
|
507
|
+
const execute = (targetPlatform = platform, targetId = deviceId) => ToolsInteract.waitForUIChangeHandler({ platform: targetPlatform, deviceId: targetId, timeout_ms, stability_window_ms, expected_change, scope, target })
|
|
508
|
+
const res = args.requireReady === true ? await withReadyWait({ platform, deviceId }, timeout_ms,
|
|
509
|
+
device => execute(device.platform as PlatformArg, device.id), (code, elapsed) => ({ success: false, timeout: true, observed_change: null, elapsed_ms: elapsed, reason: code, error: { code, message: 'Readiness wait could not complete' } })) : await execute()
|
|
453
510
|
return wrapResponse(res)
|
|
454
511
|
}
|
|
455
512
|
|
|
@@ -472,7 +529,7 @@ async function executeTap(args: ToolCallArgs) {
|
|
|
472
529
|
const deviceId = getStringArg(args, 'deviceId')
|
|
473
530
|
const { verificationMode, actionTimeoutMs } = interactionControls(args)
|
|
474
531
|
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
475
|
-
|
|
532
|
+
notifyActionStart()
|
|
476
533
|
const res = await measure('dispatch_ms', () => ToolsInteract.tapHandler({ platform, x, y, deviceId, timeoutMs: actionTimeoutMs }))
|
|
477
534
|
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
478
535
|
const result = buildActionExecutionResult({
|
|
@@ -498,14 +555,15 @@ async function executeTapElement(args: ToolCallArgs) {
|
|
|
498
555
|
if (selector) {
|
|
499
556
|
const waitFor = getObjectArg<WaitForUiMatchArg>(args, 'waitFor') as any
|
|
500
557
|
const targetStarted = performance.now()
|
|
558
|
+
const resolutionTimeoutMs = waitFor?.timeoutMs ?? actionTimeoutMs ?? 5000
|
|
501
559
|
const resolution = await withObservationContext({ stepId: args._stepId as string | undefined, purpose: 'target' }, () =>
|
|
502
|
-
measure('target_resolution_ms', () => runWithBudget(
|
|
560
|
+
measure('target_resolution_ms', () => runWithBudget(resolutionTimeoutMs, () => ToolsInteract.waitForUIHandler({ selector, condition: waitFor?.condition ?? 'clickable', timeout_ms: resolutionTimeoutMs, poll_interval_ms: waitFor?.pollIntervalMs ?? 100, match: waitFor?.match, singleObservation: true, singleAttempt: !waitFor, rejectAmbiguous: !waitFor, resolveTapTarget: true, platform: getStringArg(args, 'platform') as PlatformArg | undefined, deviceId: getStringArg(args, 'deviceId') })))) as any
|
|
503
561
|
recordObservationEvent({ step_id: args._stepId as string | undefined, purpose: 'target', target_resolution_ms: performance.now() - targetStarted, target_resolution_outcome: resolution.status })
|
|
504
562
|
if (resolution.status !== 'success' || !resolution.element?.elementId) return wrapResponse({ success: false, action_type: 'tap_element', failure_code: resolution.error?.code ?? 'ELEMENT_NOT_FOUND', retryable: true })
|
|
505
563
|
resolvedId = resolution.element.elementId
|
|
506
564
|
freshTree = resolution._tree
|
|
507
565
|
}
|
|
508
|
-
|
|
566
|
+
notifyActionStart()
|
|
509
567
|
if (freshTree) recordObservationConsumer(freshTree, 'target', args._stepId as string | undefined)
|
|
510
568
|
const res = await measure('dispatch_ms', () => ToolsInteract.tapElementHandler({ elementId: resolvedId!, freshTree, verificationMode, timeoutMs: actionTimeoutMs, platform: getStringArg(args, 'platform') as PlatformArg | undefined, deviceId: getStringArg(args, 'deviceId') })) as any
|
|
511
569
|
return wrapResponse(res)
|
|
@@ -598,7 +656,8 @@ async function handleRunJourney(args: ToolCallArgs) {
|
|
|
598
656
|
if (step.type === 'assert' && !result?.success && /TIMEOUT/.test(result?.failure_code ?? '')) result.failure_code = 'ASSERTION_TIMEOUT'
|
|
599
657
|
const success = !!result?.success
|
|
600
658
|
if (lightCoveredByAssertion) {
|
|
601
|
-
result.verification = { mode: 'light', status: success ? 'unavailable' :
|
|
659
|
+
result.verification = { mode: 'light', status: success ? 'unavailable' :
|
|
660
|
+
(['mismatch', 'unavailable'].includes(result.condition?.status) ? 'not_requested' : 'failed') }
|
|
602
661
|
if (success) {
|
|
603
662
|
result.verification_diagnostic = { code: 'VERIFICATION_UNAVAILABLE', retryable: true }
|
|
604
663
|
pendingLightStep = results.length
|
|
@@ -608,6 +667,7 @@ async function handleRunJourney(args: ToolCallArgs) {
|
|
|
608
667
|
const pending = results[pendingLightStep].result
|
|
609
668
|
if (typeof result?.observed?.matched_count === 'number') {
|
|
610
669
|
pending.verification.status = 'observed'
|
|
670
|
+
if (pending.evidence) completeJourneyVerification(pending.evidence.action_id)
|
|
611
671
|
delete pending.verification_diagnostic
|
|
612
672
|
const delta = ToolsObserve.getLatestStateDelta(platform, deviceId)
|
|
613
673
|
if (delta) { pending.state_delta = delta; pending.state_delta_available = true }
|
|
@@ -630,6 +690,7 @@ async function handleRunJourney(args: ToolCallArgs) {
|
|
|
630
690
|
for (const step of results) for (const [key, value] of Object.entries(step.result?.timing ?? {})) {
|
|
631
691
|
if (key !== 'total_ms' && typeof value === 'number') componentTiming[key] = (componentTiming[key] ?? 0) + value
|
|
632
692
|
}
|
|
693
|
+
refreshJourneyEvidence(results)
|
|
633
694
|
return wrapResponse({ success: stoppedAt === null, stopped_at_step_id: stoppedAt, steps: results, ...(failureSnapshot ? { failure_snapshot: failureSnapshot } : {}), timing: responseMode === 'compact' ? { total_ms: performance.now() - started } : { ...componentTiming, total_ms: performance.now() - started } }, responseMode === 'debug')
|
|
634
695
|
})
|
|
635
696
|
}
|
|
@@ -644,7 +705,7 @@ async function executeSwipe(args: ToolCallArgs) {
|
|
|
644
705
|
const deviceId = getStringArg(args, 'deviceId')
|
|
645
706
|
const { verificationMode, actionTimeoutMs } = interactionControls(args)
|
|
646
707
|
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
647
|
-
|
|
708
|
+
notifyActionStart()
|
|
648
709
|
const res = await measure('dispatch_ms', () => ToolsInteract.swipeHandler({ platform, x1, y1, x2, y2, duration, deviceId, timeoutMs: actionTimeoutMs }))
|
|
649
710
|
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
650
711
|
const result = buildActionExecutionResult({
|
|
@@ -668,7 +729,7 @@ async function executeScrollToElement(args: ToolCallArgs) {
|
|
|
668
729
|
const deviceId = getStringArg(args, 'deviceId')
|
|
669
730
|
const { verificationMode } = interactionControls(args)
|
|
670
731
|
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
671
|
-
|
|
732
|
+
notifyActionStart()
|
|
672
733
|
const res = await measure('dispatch_ms', () => ToolsInteract.scrollToElementHandler({ platform, selector, direction, maxScrolls, scrollAmount, deviceId }))
|
|
673
734
|
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint(platform, deviceId)) : null
|
|
674
735
|
const result = buildActionExecutionResult({
|
|
@@ -697,7 +758,7 @@ async function executeTypeText(args: ToolCallArgs) {
|
|
|
697
758
|
const deviceId = getStringArg(args, 'deviceId')
|
|
698
759
|
const { verificationMode, actionTimeoutMs } = interactionControls(args)
|
|
699
760
|
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint('android', deviceId)) : null
|
|
700
|
-
|
|
761
|
+
notifyActionStart()
|
|
701
762
|
const res = await measure('dispatch_ms', () => ToolsInteract.typeTextHandler({ text, deviceId, timeoutMs: actionTimeoutMs }))
|
|
702
763
|
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint('android', deviceId)) : null
|
|
703
764
|
const result = buildActionExecutionResult({
|
|
@@ -716,7 +777,7 @@ async function executePressBack(args: ToolCallArgs) {
|
|
|
716
777
|
const deviceId = getStringArg(args, 'deviceId')
|
|
717
778
|
const { verificationMode, actionTimeoutMs } = interactionControls(args)
|
|
718
779
|
const uiFingerprintBefore = verificationMode === 'full' ? await measure('pre_observation_ms', () => captureActionFingerprint('android', deviceId)) : null
|
|
719
|
-
|
|
780
|
+
notifyActionStart()
|
|
720
781
|
const res = await measure('dispatch_ms', () => ToolsInteract.pressBackHandler({ deviceId, timeoutMs: actionTimeoutMs }))
|
|
721
782
|
const uiFingerprintAfter = verificationMode === 'full' ? await measure('post_observation_ms', () => captureActionFingerprint('android', deviceId)) : null
|
|
722
783
|
const result = buildActionExecutionResult({
|
|
@@ -781,6 +842,16 @@ async function handleGetNetworkActivity(args: ToolCallArgs) {
|
|
|
781
842
|
}
|
|
782
843
|
|
|
783
844
|
export const toolHandlers: Record<string, ToolHandler> = {
|
|
845
|
+
get_app_readiness: async args => {
|
|
846
|
+
const parsed = readinessSchema.safeParse(args)
|
|
847
|
+
if (!parsed.success) throw new Error('INVALID_ARGUMENT: invalid readiness arguments')
|
|
848
|
+
return wrapResponse(await getAppReadiness(parsed.data))
|
|
849
|
+
},
|
|
850
|
+
wait_for_app_ready: async args => {
|
|
851
|
+
const parsed = readinessWaitSchema.safeParse(args)
|
|
852
|
+
if (!parsed.success) throw new Error('INVALID_ARGUMENT: invalid readiness arguments')
|
|
853
|
+
return wrapResponse(await getAppReadiness(parsed.data, true))
|
|
854
|
+
},
|
|
784
855
|
start_app: handleStartApp,
|
|
785
856
|
terminate_app: handleTerminateApp,
|
|
786
857
|
restart_app: handleRestartApp,
|
|
@@ -818,6 +889,10 @@ export const toolHandlers: Record<string, ToolHandler> = {
|
|
|
818
889
|
get_network_activity: handleGetNetworkActivity
|
|
819
890
|
}
|
|
820
891
|
|
|
892
|
+
for (const name of ['capture_screenshot', 'get_ui_tree', 'get_screen_fingerprint', 'capture_debug_snapshot', 'get_logs', 'read_log_stream', 'run_journey', 'adjust_control']) {
|
|
893
|
+
toolHandlers[name] = correlatedHandler(name, toolHandlers[name], name === 'adjust_control', name === 'run_journey')
|
|
894
|
+
}
|
|
895
|
+
|
|
821
896
|
export async function handleToolCall(name: string, args: ToolCallArgs = {}) {
|
|
822
897
|
const handler = toolHandlers[name]
|
|
823
898
|
if (!handler) throw new Error(`Unknown tool: ${name}`)
|
|
@@ -838,6 +913,7 @@ export async function handleToolCall(name: string, args: ToolCallArgs = {}) {
|
|
|
838
913
|
}
|
|
839
914
|
return withTiming(measured, async () => {
|
|
840
915
|
try {
|
|
916
|
+
validateExpectedState(name, args)
|
|
841
917
|
return addTiming(await handler(args))
|
|
842
918
|
} catch (error) {
|
|
843
919
|
console.error(`Error executing tool ${name}:`, error)
|
package/src/server-core.ts
CHANGED
package/src/types.ts
CHANGED
|
@@ -218,6 +218,7 @@ export interface UIElement {
|
|
|
218
218
|
}
|
|
219
219
|
|
|
220
220
|
export interface GetUITreeResponse {
|
|
221
|
+
readiness?: ReadinessSummary;
|
|
221
222
|
device: DeviceInfo;
|
|
222
223
|
screen: string;
|
|
223
224
|
resolution: {
|
|
@@ -226,12 +227,27 @@ export interface GetUITreeResponse {
|
|
|
226
227
|
};
|
|
227
228
|
elements: UIElement[];
|
|
228
229
|
snapshot_revision: number;
|
|
230
|
+
snapshot_session_id?: string;
|
|
229
231
|
captured_at_ms: number;
|
|
230
232
|
snapshot_delta?: SnapshotDelta | null;
|
|
231
233
|
loading_state?: LoadingState | null;
|
|
232
234
|
error?: string;
|
|
233
235
|
}
|
|
234
236
|
|
|
237
|
+
export interface ReadinessSummary {
|
|
238
|
+
state: 'loading' | 'transitioning' | 'settled' | 'unknown' | 'unresponsive' | 'debugger_paused';
|
|
239
|
+
scope: 'device_ui';
|
|
240
|
+
basis: 'observed' | 'heuristic' | 'insufficient';
|
|
241
|
+
captured_at_ms: number | null;
|
|
242
|
+
sample_count: number;
|
|
243
|
+
stable_for_ms: number;
|
|
244
|
+
snapshot_session_id: string | null;
|
|
245
|
+
snapshot_revision: number | null;
|
|
246
|
+
evidence: Array<{ code: string; source: string; captured_at_ms: number | null; snapshot_revision: number | null }>;
|
|
247
|
+
limitations: string[];
|
|
248
|
+
capabilities: { loading: boolean; transitioning: boolean; settled: boolean; unresponsive: boolean; debugger_paused: boolean };
|
|
249
|
+
}
|
|
250
|
+
|
|
235
251
|
export interface GetCurrentScreenResponse {
|
|
236
252
|
device: DeviceInfo;
|
|
237
253
|
package: string;
|
|
@@ -289,6 +305,8 @@ export interface TapResponse {
|
|
|
289
305
|
}
|
|
290
306
|
|
|
291
307
|
export type ActionFailureCode =
|
|
308
|
+
| 'PRECONDITION_FAILED'
|
|
309
|
+
| 'PRECONDITION_UNAVAILABLE'
|
|
292
310
|
| 'ELEMENT_NOT_FOUND'
|
|
293
311
|
| 'ELEMENT_NOT_INTERACTABLE'
|
|
294
312
|
| 'TIMEOUT'
|
|
@@ -417,6 +435,15 @@ export interface TimingBreakdown {
|
|
|
417
435
|
serialization_ms?: number;
|
|
418
436
|
}
|
|
419
437
|
|
|
438
|
+
export interface ActionConditionResult {
|
|
439
|
+
status: 'matched' | 'mismatch' | 'unavailable' | 'not_checked';
|
|
440
|
+
expected: { snapshot_session_id: string; snapshot_revision: number };
|
|
441
|
+
observed: GetUITreeResponse | null;
|
|
442
|
+
elapsed_ms: number;
|
|
443
|
+
reason?: 'BASELINE_UNAVAILABLE' | 'SESSION_MISMATCH' | 'DEVICE_MISMATCH' |
|
|
444
|
+
'OBSERVATION_FAILED' | 'OBSERVATION_INVALID' | 'CONDITION_TIMEOUT' | 'ACTION_NOT_REACHED';
|
|
445
|
+
}
|
|
446
|
+
|
|
420
447
|
export interface ActionExecutionResult {
|
|
421
448
|
action_id: string;
|
|
422
449
|
timestamp: string;
|
|
@@ -435,6 +462,7 @@ export interface ActionExecutionResult {
|
|
|
435
462
|
dispatch_started?: boolean;
|
|
436
463
|
verification?: { mode: 'none' | 'light' | 'full'; status: 'not_requested' | 'observed' | 'verified' | 'unavailable' | 'failed' };
|
|
437
464
|
state_delta?: StateDelta;
|
|
465
|
+
condition?: ActionConditionResult;
|
|
438
466
|
timing?: TimingBreakdown;
|
|
439
467
|
recovery?: RecoveryState;
|
|
440
468
|
trace: ActionTrace;
|
|
@@ -535,6 +563,7 @@ export interface AdjustControlResponse extends ActionExecutionResult {
|
|
|
535
563
|
}
|
|
536
564
|
|
|
537
565
|
export interface WaitForUIChangeResponse {
|
|
566
|
+
readiness?: ReadinessSummary;
|
|
538
567
|
success: boolean;
|
|
539
568
|
observed_change: 'hierarchy_diff' | 'text_change' | 'state_change' | null;
|
|
540
569
|
snapshot_revision?: number;
|
|
@@ -554,7 +583,7 @@ export interface WaitForUIChangeResponse {
|
|
|
554
583
|
reason?: string;
|
|
555
584
|
loading_state?: LoadingState | null;
|
|
556
585
|
error?: {
|
|
557
|
-
code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND';
|
|
586
|
+
code: 'INVALID_SCOPE' | 'ELEMENT_NOT_FOUND' | 'TIMEOUT' | 'DEVICE_UNAVAILABLE';
|
|
558
587
|
message: string;
|
|
559
588
|
};
|
|
560
589
|
}
|
package/src/utils/ios/utils.ts
CHANGED
|
@@ -62,7 +62,7 @@ export async function isIDBInstalled(): Promise<boolean> {
|
|
|
62
62
|
const cmd = getIdbCmd()
|
|
63
63
|
if (!cmd) return false
|
|
64
64
|
try {
|
|
65
|
-
execSync(`${cmd} list-targets --json`, { stdio: ['ignore','pipe','pipe'], timeout: 5000 })
|
|
65
|
+
execSync(`${cmd} list-targets --json`, { stdio: ['ignore','pipe','pipe'], timeout: remainingBudget(5000), killSignal: 'SIGKILL' })
|
|
66
66
|
return true
|
|
67
67
|
} catch (e: unknown) {
|
|
68
68
|
console.debug(`[isIDBInstalled] idb probe failed for '${cmd}': ${e instanceof Error ? e.message : String(e)}`)
|
|
@@ -200,7 +200,7 @@ export async function findAppBundle(dir: string): Promise<string | undefined> {
|
|
|
200
200
|
export async function getIOSDeviceMetadata(deviceId: string = "booted"): Promise<DeviceInfo> {
|
|
201
201
|
return new Promise((resolve) => {
|
|
202
202
|
// If deviceId is provided (and not "booted"), attempt to find that device among booted simulators.
|
|
203
|
-
execFile(getXcrunCmd(), ['simctl', 'list', 'devices', 'booted', '--json'], (err, stdout) => {
|
|
203
|
+
execFile(getXcrunCmd(), ['simctl', 'list', 'devices', 'booted', '--json'], { timeout: remainingBudget(), killSignal: 'SIGKILL' }, (err, stdout) => {
|
|
204
204
|
const fallback: DeviceInfo = {
|
|
205
205
|
platform: "ios",
|
|
206
206
|
id: deviceId,
|
|
@@ -247,7 +247,7 @@ export async function getIOSDeviceMetadata(deviceId: string = "booted"): Promise
|
|
|
247
247
|
export async function listIOSDevices(appId?: string): Promise<DeviceInfo[]> {
|
|
248
248
|
return new Promise((resolve) => {
|
|
249
249
|
// Query all devices and separately query booted devices to mark them
|
|
250
|
-
execFile(getXcrunCmd(), ['simctl', 'list', 'devices', '--json'], (err, stdout) => {
|
|
250
|
+
execFile(getXcrunCmd(), ['simctl', 'list', 'devices', '--json'], { timeout: remainingBudget(), killSignal: 'SIGKILL' }, (err, stdout) => {
|
|
251
251
|
if (err || !stdout) return resolve([])
|
|
252
252
|
try {
|
|
253
253
|
const data = JSON.parse(stdout)
|
|
@@ -256,7 +256,7 @@ export async function listIOSDevices(appId?: string): Promise<DeviceInfo[]> {
|
|
|
256
256
|
const checks: Promise<void>[] = []
|
|
257
257
|
|
|
258
258
|
// Get booted devices set
|
|
259
|
-
execFile(getXcrunCmd(), ['simctl', 'list', 'devices', 'booted', '--json'], (err2, stdout2) => {
|
|
259
|
+
execFile(getXcrunCmd(), ['simctl', 'list', 'devices', 'booted', '--json'], { timeout: remainingBudget(), killSignal: 'SIGKILL' }, (err2, stdout2) => {
|
|
260
260
|
const bootedSet = new Set<string>()
|
|
261
261
|
if (!err2 && stdout2) {
|
|
262
262
|
try {
|
|
@@ -10,11 +10,31 @@ export function runWithBudget<T>(timeoutMs: number, operation: (budget: Operatio
|
|
|
10
10
|
}
|
|
11
11
|
export function remainingBudget(fallback = 0): number {
|
|
12
12
|
const budget = budgets.getStore()
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
let deadline = budget?.deadline ?? Infinity
|
|
14
|
+
for (let ancestor = budget?.parent; ancestor; ancestor = ancestor.parent) deadline = Math.min(deadline, ancestor.deadline)
|
|
15
|
+
if (!Number.isFinite(deadline)) return fallback
|
|
16
|
+
const remaining = Math.floor(deadline - performance.now())
|
|
15
17
|
if (remaining <= 0) throw new Error('ACTION_TIMEOUT')
|
|
16
18
|
return fallback > 0 ? Math.min(fallback, remaining) : remaining
|
|
17
19
|
}
|
|
20
|
+
|
|
21
|
+
// Pause only this action's budget. Any enclosing operation still bounds the read.
|
|
22
|
+
export async function runWithSeparateBudget<T>(timeoutMs: number, operation: () => Promise<T>): Promise<T> {
|
|
23
|
+
remainingBudget()
|
|
24
|
+
const action = budgets.getStore()
|
|
25
|
+
const started = performance.now()
|
|
26
|
+
const parent = action?.parent
|
|
27
|
+
const budget: OperationBudget = { deadline: Math.min(parent?.deadline ?? Infinity, started + timeoutMs), dispatchStarted: false, parent }
|
|
28
|
+
try {
|
|
29
|
+
return await budgets.run(budget, async () => {
|
|
30
|
+
const result = await operation()
|
|
31
|
+
remainingBudget()
|
|
32
|
+
return result
|
|
33
|
+
})
|
|
34
|
+
} finally {
|
|
35
|
+
if (action) action.deadline += performance.now() - started
|
|
36
|
+
}
|
|
37
|
+
}
|
|
18
38
|
export function markDispatchStarted() {
|
|
19
39
|
remainingBudget()
|
|
20
40
|
let budget = budgets.getStore()
|