@aarwitz/tapp 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +123 -0
- package/Harness/OCQAHarness/AppDelegate.swift +21 -0
- package/Harness/OCQAHarness/Info.plist +26 -0
- package/Harness/OCQAHarness.xcodeproj/project.pbxproj +199 -0
- package/Harness/OCQAHarness.xcodeproj/xcshareddata/xcschemes/OCQAHarnessUITests.xcscheme +22 -0
- package/Harness/OCQAHarnessUITests/ExplorerTests.swift +4526 -0
- package/Harness/OCQAHarnessUITests/Info.plist +22 -0
- package/Harness/generate-harness-xcodeproj.rb +254 -0
- package/LICENSE +21 -0
- package/README.md +374 -0
- package/bin/tapp.js +1382 -0
- package/browser/app.css +227 -0
- package/browser/app.js +675 -0
- package/browser/index.html +195 -0
- package/browser/product-contract.js +25 -0
- package/browser/view-model.js +16 -0
- package/docs/BROWSER-PRODUCT.md +72 -0
- package/docs/PRODUCT-ENGINE.md +102 -0
- package/docs/application-model.md +276 -0
- package/docs/scenarios.md +95 -0
- package/mcp-server/src/android-driver.js +287 -0
- package/mcp-server/src/android-explorer.js +197 -0
- package/mcp-server/src/android-flow.js +89 -0
- package/mcp-server/src/application-model.js +1597 -0
- package/mcp-server/src/browser-product.js +659 -0
- package/mcp-server/src/browser-workspaces.js +234 -0
- package/mcp-server/src/ci-report.js +557 -0
- package/mcp-server/src/ci-setup.js +359 -0
- package/mcp-server/src/contract-authoring.js +10 -0
- package/mcp-server/src/enrich.js +57 -0
- package/mcp-server/src/flow-runtime.js +127 -0
- package/mcp-server/src/html-report.js +124 -0
- package/mcp-server/src/index.js +3775 -0
- package/mcp-server/src/maintenance-proposal.js +178 -0
- package/mcp-server/src/managed-operation.js +61 -0
- package/mcp-server/src/pr-selection.js +841 -0
- package/mcp-server/src/product-execution.js +155 -0
- package/mcp-server/src/product-operations.js +526 -0
- package/mcp-server/src/project-config.js +101 -0
- package/mcp-server/src/release-contract.d.ts +81 -0
- package/mcp-server/src/release-contract.js +226 -0
- package/mcp-server/src/report.js +363 -0
- package/mcp-server/src/scenario-runtime.js +139 -0
- package/mcp-server/src/static-server.js +44 -0
- package/mcp-server/src/task-runtime.js +266 -0
- package/mcp-server/src/ui-map.js +661 -0
- package/mcp-server/src/web-explorer.js +493 -0
- package/mcp-server/src/web-flow.js +238 -0
- package/package.json +82 -0
- package/scripts/android-corpus-e2e.sh +30 -0
- package/scripts/ci-gate.sh +323 -0
- package/scripts/cleanup-xcode.sh +157 -0
- package/scripts/compile-contract.js +27 -0
- package/scripts/compile-flow.js +18 -0
- package/scripts/corpus-apps.txt +9 -0
- package/scripts/corpus-sweep.sh +121 -0
- package/scripts/coverage-eval.sh +92 -0
- package/scripts/coverage_eval_parse.py +95 -0
- package/scripts/deploy-and-build.sh +99 -0
- package/scripts/flow-platform.js +18 -0
- package/scripts/flow_ai_judge.py +102 -0
- package/scripts/flow_lib.py +154 -0
- package/scripts/mutation-recall-desktop.sh +186 -0
- package/scripts/mutation-recall.sh +121 -0
- package/scripts/mutation_lib.py +128 -0
- package/scripts/mutation_operators.py +144 -0
- package/scripts/platform-gate.js +186 -0
- package/scripts/pr-plan.js +68 -0
- package/scripts/quick-capture.sh +419 -0
- package/scripts/run-android-flow.js +27 -0
- package/scripts/run-flow.sh +90 -0
- package/scripts/run-web-flow.js +28 -0
- package/scripts/run-web-scenario.js +23 -0
- package/scripts/validation-matrix.sh +146 -0
- package/scripts/vision-fp-eval.sh +206 -0
- package/scripts/vision_escalation_responder.py +147 -0
- package/scripts/vision_fp_probe.py +221 -0
|
@@ -0,0 +1,4526 @@
|
|
|
1
|
+
import XCTest
|
|
2
|
+
|
|
3
|
+
/// Autonomous exploration engine for iOS QA.
|
|
4
|
+
/// Runs as a UI test that attaches to any app via bundle ID.
|
|
5
|
+
/// Communicates results via OCQA_ prefixed stdout markers.
|
|
6
|
+
///
|
|
7
|
+
/// Fully generalized — no app-specific logic. Uses depth-first exploration
|
|
8
|
+
/// that prioritizes in-screen content over persistent navigation (tab bars).
|
|
9
|
+
///
|
|
10
|
+
/// Modes:
|
|
11
|
+
/// - testAutonomousExploration: Full autonomous exploration loop
|
|
12
|
+
/// - testDumpUITree: One-shot accessibility tree dump
|
|
13
|
+
/// - testTapAtCoordinate / testTapById: Single action for engine control
|
|
14
|
+
/// - testScreenshot: Capture and attach a screenshot
|
|
15
|
+
class ExplorerTests: XCTestCase {
|
|
16
|
+
|
|
17
|
+
private struct InputDescriptor {
|
|
18
|
+
let key: String
|
|
19
|
+
let label: String
|
|
20
|
+
let secure: Bool
|
|
21
|
+
let placeholder: String
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
var app: XCUIApplication!
|
|
25
|
+
/// Set whenever OCQA_COMPLETE is printed — the crash-teardown net only fires without it.
|
|
26
|
+
var didEmitComplete = false
|
|
27
|
+
/// Persistence probe: "screen|field" → text we typed AND verified visible in the a11y value.
|
|
28
|
+
/// On a later REVISIT of that screen, an empty field means the value silently didn't persist.
|
|
29
|
+
var typedFieldMemory: [String: String] = [:]
|
|
30
|
+
var reportedPersistenceKeys = Set<String>()
|
|
31
|
+
var lastProbeTitle = ""
|
|
32
|
+
/// Keyboard-occlusion memory: screen → primary-action button labels seen with NO keyboard up.
|
|
33
|
+
var settledPrimaryButtons: [String: Set<String>] = [:]
|
|
34
|
+
var config: [String: Any] = [:]
|
|
35
|
+
/// Detected once at setUp; avoids hardcoded device dimensions
|
|
36
|
+
private var screenBounds: CGRect = .zero
|
|
37
|
+
/// Set by performSmartAction for the most recent type action, read by narrate()
|
|
38
|
+
private var lastTypedWasOverride = false
|
|
39
|
+
private var lastTypedSecure = false
|
|
40
|
+
/// Screens already reported for a11y-invisible field content (one finding per screen).
|
|
41
|
+
private var valueHiddenReported = Set<String>()
|
|
42
|
+
|
|
43
|
+
var targetBundleId: String { config["OCQA_BUNDLE_ID"] as? String ?? ProcessInfo.processInfo.environment["OCQA_BUNDLE_ID"] ?? "" }
|
|
44
|
+
var maxActions: Int { Int(config["OCQA_MAX_ACTIONS"] as? String ?? ProcessInfo.processInfo.environment["OCQA_MAX_ACTIONS"] ?? "400") ?? 400 }
|
|
45
|
+
var timeoutSeconds: Int { Int(config["OCQA_TIMEOUT_SECONDS"] as? String ?? ProcessInfo.processInfo.environment["OCQA_TIMEOUT_SECONDS"] ?? "1800") ?? 1800 }
|
|
46
|
+
/// Launch arguments to forward to the target app (e.g. ["--uitesting"])
|
|
47
|
+
var appLaunchArgs: [String] { config["OCQA_APP_LAUNCH_ARGS"] as? [String] ?? [] }
|
|
48
|
+
/// Environment variables to forward to the target app (e.g. ["UI_TEST_ROLE": "resident"])
|
|
49
|
+
var appLaunchEnv: [String: String] {
|
|
50
|
+
if let dict = config["OCQA_APP_LAUNCH_ENV"] as? [String: String] { return dict }
|
|
51
|
+
return [:]
|
|
52
|
+
}
|
|
53
|
+
/// Targeted exploration: beeline to this screen (following `route`, a sequence of control
|
|
54
|
+
/// labels) before exploring. A PR gate supplies the same data as one structured, bounded
|
|
55
|
+
/// UI-Map target so customer-derived labels never become shell source.
|
|
56
|
+
var prExplorationTarget: [String: Any]? { config["OCQA_PR_TARGET"] as? [String: Any] }
|
|
57
|
+
var targetScreen: String {
|
|
58
|
+
if let explicit = config["OCQA_TARGET_SCREEN"] as? String, !explicit.isEmpty { return explicit }
|
|
59
|
+
return (prExplorationTarget?["node"] as? [String: Any])?["name"] as? String ?? ""
|
|
60
|
+
}
|
|
61
|
+
var route: [String] {
|
|
62
|
+
if let explicit = config["OCQA_ROUTE"] as? [String], !explicit.isEmpty { return explicit }
|
|
63
|
+
let navigation = prExplorationTarget?["navigation"] as? [String: Any]
|
|
64
|
+
let steps = navigation?["steps"] as? [[String: Any]] ?? []
|
|
65
|
+
return steps.compactMap { ($0["action"] as? [String: Any])?["target"] as? String }
|
|
66
|
+
}
|
|
67
|
+
var routeTimeouts: [TimeInterval] {
|
|
68
|
+
let navigation = prExplorationTarget?["navigation"] as? [String: Any]
|
|
69
|
+
let steps = navigation?["steps"] as? [[String: Any]] ?? []
|
|
70
|
+
return steps.map { step in
|
|
71
|
+
let wait = step["wait"] as? [String: Any]
|
|
72
|
+
let milliseconds = (wait?["timeoutMs"] as? NSNumber)?.doubleValue ?? 6000
|
|
73
|
+
return max(0.25, min(30, milliseconds / 1000))
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
var prExplorationTargetId: String { prExplorationTarget?["id"] as? String ?? "" }
|
|
77
|
+
|
|
78
|
+
/// Resolve a string key: config (as String) -> process environment -> fallback
|
|
79
|
+
private func resolve(_ key: String, fallback: String = "") -> String {
|
|
80
|
+
if let v = config[key] as? String { return v }
|
|
81
|
+
if let v = ProcessInfo.processInfo.environment[key], !v.isEmpty { return v }
|
|
82
|
+
return fallback
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
private func loadConfig() {
|
|
86
|
+
// OCQA_CONFIG_PATH (forwarded from the host via TEST_RUNNER_OCQA_CONFIG_PATH) is
|
|
87
|
+
// authoritative and per-run — checked FIRST so each device reads its own config and
|
|
88
|
+
// never picks up a stale /tmp file from a previous run.
|
|
89
|
+
let paths = [
|
|
90
|
+
ProcessInfo.processInfo.environment["OCQA_CONFIG_PATH"] ?? "",
|
|
91
|
+
"/tmp/ocqa-run-config.json",
|
|
92
|
+
NSTemporaryDirectory() + "ocqa-run-config.json",
|
|
93
|
+
]
|
|
94
|
+
for path in paths where !path.isEmpty {
|
|
95
|
+
if let data = try? Data(contentsOf: URL(fileURLWithPath: path)),
|
|
96
|
+
let dict = try? JSONSerialization.jsonObject(with: data) as? [String: Any] {
|
|
97
|
+
config = dict
|
|
98
|
+
print("OCQA_STATE:config_loaded path=\(path) overrides=\((dict["OCQA_INPUT_OVERRIDES"] as? [String: String])?.count ?? 0)")
|
|
99
|
+
return
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
print("OCQA_STATE:config_not_found — using defaults")
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
override func setUp() {
|
|
106
|
+
super.setUp()
|
|
107
|
+
continueAfterFailure = true
|
|
108
|
+
loadConfig()
|
|
109
|
+
if !targetBundleId.isEmpty {
|
|
110
|
+
app = XCUIApplication(bundleIdentifier: targetBundleId)
|
|
111
|
+
} else {
|
|
112
|
+
app = XCUIApplication()
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Crash safety net, registered BEFORE launch: an app that dies in setUp (instant
|
|
116
|
+
// launch crash) or mid-run aborts the test on the next XCUITest query — teardown
|
|
117
|
+
// blocks still run, so a dead app + no OCQA_COMPLETE becomes a CRITICAL crash
|
|
118
|
+
// finding instead of a zero-marker run. (Corpus finding: Yattee dies at launch;
|
|
119
|
+
// the frame query at the end of setUp threw before the test body could arm a net.)
|
|
120
|
+
addTeardownBlock { [self] in
|
|
121
|
+
if !didEmitComplete, let app, app.state != .runningForeground {
|
|
122
|
+
didEmitComplete = true
|
|
123
|
+
print("OCQA_ISSUE:{\"type\":\"crash\",\"severity\":\"critical\",\"title\":\"App crashed at launch or during the run (terminated mid-query)\",\"screen\":\"Launch\",\"step\":0}")
|
|
124
|
+
print("OCQA_COMPLETE:{\"actions\":0,\"states\":0,\"issues\":1,\"screens\":\"\",\"outcome\":\"crash_teardown\"}")
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Handle system alerts (location, notifications, tracking, etc.)
|
|
129
|
+
addUIInterruptionMonitor(withDescription: "System Alert") { alert in
|
|
130
|
+
let allowLabels = ["Allow", "Allow While Using App", "OK", "Continue", "Allow Full Access"]
|
|
131
|
+
for label in allowLabels {
|
|
132
|
+
let btn = alert.buttons[label]
|
|
133
|
+
if btn.exists {
|
|
134
|
+
btn.tap()
|
|
135
|
+
return true
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
if alert.buttons.count > 0 {
|
|
139
|
+
alert.buttons.element(boundBy: 0).tap()
|
|
140
|
+
return true
|
|
141
|
+
}
|
|
142
|
+
return false
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// Launch with auth-bypass args if configured, otherwise just activate
|
|
146
|
+
if !appLaunchArgs.isEmpty || !appLaunchEnv.isEmpty {
|
|
147
|
+
app.launchArguments = appLaunchArgs
|
|
148
|
+
app.launchEnvironment = appLaunchEnv
|
|
149
|
+
app.launch()
|
|
150
|
+
_ = app.wait(for: .runningForeground, timeout: 10)
|
|
151
|
+
} else {
|
|
152
|
+
app.activate()
|
|
153
|
+
let started = app.wait(for: .runningForeground, timeout: 10)
|
|
154
|
+
if !started {
|
|
155
|
+
app.launch()
|
|
156
|
+
_ = app.wait(for: .runningForeground, timeout: 10)
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Detect actual screen dimensions from the running app — but only if it survived
|
|
161
|
+
// launch (a frame query on a dead app throws; the teardown net reports the crash).
|
|
162
|
+
guard app.state == .runningForeground else { return }
|
|
163
|
+
let windowFrame = app.windows.firstMatch.frame
|
|
164
|
+
if windowFrame.width > 0 && windowFrame.height > 0 {
|
|
165
|
+
screenBounds = windowFrame
|
|
166
|
+
} else {
|
|
167
|
+
screenBounds = app.frame
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// MARK: - UI Tree Dump
|
|
172
|
+
|
|
173
|
+
func testDumpUITree() {
|
|
174
|
+
let elements = readUITree(app)
|
|
175
|
+
let state = buildAppState(elements: elements)
|
|
176
|
+
emitUITree(state)
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// MARK: - Tap Actions
|
|
180
|
+
|
|
181
|
+
func testTapAtCoordinate() {
|
|
182
|
+
let xStr = resolve("OCQA_TAP_X")
|
|
183
|
+
let yStr = resolve("OCQA_TAP_Y")
|
|
184
|
+
guard !xStr.isEmpty, !yStr.isEmpty,
|
|
185
|
+
let x = Double(xStr), let y = Double(yStr) else {
|
|
186
|
+
XCTFail("OCQA_TAP_X and OCQA_TAP_Y must be set")
|
|
187
|
+
return
|
|
188
|
+
}
|
|
189
|
+
let coord = app.coordinate(withNormalizedOffset: .zero)
|
|
190
|
+
.withOffset(CGVector(dx: x, dy: y))
|
|
191
|
+
coord.tap()
|
|
192
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"x\":\(x),\"y\":\(y)}")
|
|
193
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
func testTapById() {
|
|
197
|
+
let identifier = resolve("OCQA_TAP_ID")
|
|
198
|
+
guard !identifier.isEmpty else {
|
|
199
|
+
XCTFail("OCQA_TAP_ID must be set")
|
|
200
|
+
return
|
|
201
|
+
}
|
|
202
|
+
let queries: [XCUIElementQuery] = [
|
|
203
|
+
app.buttons, app.staticTexts, app.cells,
|
|
204
|
+
app.links, app.switches, app.textFields
|
|
205
|
+
]
|
|
206
|
+
for query in queries {
|
|
207
|
+
let element = query[identifier]
|
|
208
|
+
if element.exists && element.isHittable {
|
|
209
|
+
element.tap()
|
|
210
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"identifier\":\"\(identifier)\"}")
|
|
211
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
212
|
+
return
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
let predicate = NSPredicate(format: "label == %@", identifier)
|
|
216
|
+
let match = app.descendants(matching: .any).matching(predicate).firstMatch
|
|
217
|
+
if match.exists && match.isHittable {
|
|
218
|
+
match.tap()
|
|
219
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"label\":\"\(identifier)\"}")
|
|
220
|
+
} else {
|
|
221
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"identifier\":\"\(identifier)\",\"status\":\"not_found\"}")
|
|
222
|
+
}
|
|
223
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
// MARK: - Swipe Actions
|
|
227
|
+
|
|
228
|
+
func testSwipe() {
|
|
229
|
+
let dir = resolve("OCQA_SWIPE_DIR", fallback: "up")
|
|
230
|
+
switch dir {
|
|
231
|
+
case "up": app.swipeUp()
|
|
232
|
+
case "down": app.swipeDown()
|
|
233
|
+
case "left": app.swipeLeft()
|
|
234
|
+
case "right": app.swipeRight()
|
|
235
|
+
default: app.swipeUp()
|
|
236
|
+
}
|
|
237
|
+
print("OCQA_ACTION:{\"type\":\"swipe\",\"direction\":\"\(dir)\"}")
|
|
238
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// MARK: - Type Text
|
|
242
|
+
|
|
243
|
+
func testTypeText() {
|
|
244
|
+
let text = resolve("OCQA_TYPE_TEXT")
|
|
245
|
+
guard !text.isEmpty else {
|
|
246
|
+
XCTFail("OCQA_TYPE_TEXT must be set")
|
|
247
|
+
return
|
|
248
|
+
}
|
|
249
|
+
let identifier = resolve("OCQA_TYPE_ID")
|
|
250
|
+
if !identifier.isEmpty {
|
|
251
|
+
let field = app.textFields[identifier]
|
|
252
|
+
if field.exists {
|
|
253
|
+
field.tap()
|
|
254
|
+
field.typeText(text)
|
|
255
|
+
print("OCQA_ACTION:{\"type\":\"typeText\",\"identifier\":\"\(identifier)\"}")
|
|
256
|
+
return
|
|
257
|
+
}
|
|
258
|
+
let secure = app.secureTextFields[identifier]
|
|
259
|
+
if secure.exists {
|
|
260
|
+
secure.tap()
|
|
261
|
+
secure.typeText(text)
|
|
262
|
+
print("OCQA_ACTION:{\"type\":\"typeText\",\"identifier\":\"\(identifier)\"}")
|
|
263
|
+
return
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
let firstField = app.textFields.firstMatch
|
|
267
|
+
if firstField.exists {
|
|
268
|
+
firstField.tap()
|
|
269
|
+
firstField.typeText(text)
|
|
270
|
+
}
|
|
271
|
+
print("OCQA_ACTION:{\"type\":\"typeText\"}")
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// MARK: - Navigation
|
|
275
|
+
|
|
276
|
+
func testGoBack() {
|
|
277
|
+
let backButton = app.navigationBars.buttons.firstMatch
|
|
278
|
+
if backButton.exists && backButton.isHittable {
|
|
279
|
+
backButton.tap()
|
|
280
|
+
print("OCQA_ACTION:{\"type\":\"back\",\"method\":\"button\"}")
|
|
281
|
+
} else {
|
|
282
|
+
let start = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0.5))
|
|
283
|
+
let end = app.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
|
|
284
|
+
start.press(forDuration: 0.05, thenDragTo: end)
|
|
285
|
+
print("OCQA_ACTION:{\"type\":\"back\",\"method\":\"swipe\"}")
|
|
286
|
+
}
|
|
287
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// MARK: - Screenshot
|
|
291
|
+
|
|
292
|
+
func testScreenshot() {
|
|
293
|
+
let label = resolve("OCQA_SCREENSHOT_LABEL", fallback: "screenshot")
|
|
294
|
+
let screenshot = app.screenshot()
|
|
295
|
+
let attachment = XCTAttachment(screenshot: screenshot)
|
|
296
|
+
attachment.name = label
|
|
297
|
+
attachment.lifetime = .keepAlways
|
|
298
|
+
add(attachment)
|
|
299
|
+
print("OCQA_ACTION:{\"type\":\"screenshot\",\"label\":\"\(label)\"}")
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
// MARK: - Interactive Session (Playwright-style tap → inspect loop)
|
|
303
|
+
|
|
304
|
+
/// Launches the app ONCE and then services a queue of single commands from a file, emitting the
|
|
305
|
+
/// fresh accessibility tree after each — so a client (the MCP server / Copilot) can drive
|
|
306
|
+
/// tap/type/swipe/inspect loops without paying a cold XCUITest launch per action. Commands are
|
|
307
|
+
/// JSON `{seq, action, id?, x?, y?, text?, direction?, label?}` written to OCQA_SESSION_CMD_PATH;
|
|
308
|
+
/// a `{seq, status, action}` ack is written to OCQA_SESSION_RESULT_PATH and the tree goes to stdout.
|
|
309
|
+
func testInteractiveSession() {
|
|
310
|
+
// Fresh launch for a deterministic starting screen — terminate any leftover instance first
|
|
311
|
+
// so the session always begins from the app's launch state, not whatever a prior run left.
|
|
312
|
+
// Reapply the configured launch args/env (e.g. backend override, login bypass) on relaunch.
|
|
313
|
+
if !targetBundleId.isEmpty { app.terminate() }
|
|
314
|
+
app.launchArguments = appLaunchArgs
|
|
315
|
+
app.launchEnvironment = appLaunchEnv
|
|
316
|
+
app.launch()
|
|
317
|
+
var lastCount = 0
|
|
318
|
+
for _ in 0..<5 {
|
|
319
|
+
Thread.sleep(forTimeInterval: 0.4)
|
|
320
|
+
let q = app.descendants(matching: .any)
|
|
321
|
+
_ = q.firstMatch.waitForExistence(timeout: 3)
|
|
322
|
+
let c = q.count
|
|
323
|
+
if c > 10 && c == lastCount { break }
|
|
324
|
+
lastCount = c
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
let cmdPath = resolve("OCQA_SESSION_CMD_PATH", fallback: "/tmp/ocqa-session-cmd.json")
|
|
328
|
+
let resultPath = resolve("OCQA_SESSION_RESULT_PATH", fallback: "/tmp/ocqa-session-result.json")
|
|
329
|
+
let sessionTimeout = Double(resolve("OCQA_SESSION_TIMEOUT", fallback: "1800")) ?? 1800
|
|
330
|
+
|
|
331
|
+
try? FileManager.default.removeItem(atPath: cmdPath)
|
|
332
|
+
print("OCQA_SESSION:ready")
|
|
333
|
+
emitSessionTree()
|
|
334
|
+
|
|
335
|
+
let start = Date()
|
|
336
|
+
var lastSeq = -1
|
|
337
|
+
while Date().timeIntervalSince(start) < sessionTimeout {
|
|
338
|
+
Thread.sleep(forTimeInterval: 0.25)
|
|
339
|
+
guard let data = try? Data(contentsOf: URL(fileURLWithPath: cmdPath)),
|
|
340
|
+
let cmd = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
|
341
|
+
let seq = cmd["seq"] as? Int, seq != lastSeq else { continue }
|
|
342
|
+
lastSeq = seq
|
|
343
|
+
let action = (cmd["action"] as? String ?? "").lowercased()
|
|
344
|
+
var status = "ok"
|
|
345
|
+
var loginDetail = ""
|
|
346
|
+
|
|
347
|
+
switch action {
|
|
348
|
+
case "login":
|
|
349
|
+
let email = (cmd["email"] as? String ?? "").isEmpty ? resolve("OCQA_TEST_EMAIL") : (cmd["email"] as? String ?? "")
|
|
350
|
+
let password = (cmd["password"] as? String ?? "").isEmpty ? resolve("OCQA_TEST_PASSWORD") : (cmd["password"] as? String ?? "")
|
|
351
|
+
if email.isEmpty || password.isEmpty {
|
|
352
|
+
status = "missing_credentials"
|
|
353
|
+
} else {
|
|
354
|
+
let r = sessionLogin(email: email, password: password)
|
|
355
|
+
status = r.status
|
|
356
|
+
loginDetail = r.detail
|
|
357
|
+
}
|
|
358
|
+
case "tap":
|
|
359
|
+
if let id = cmd["id"] as? String, !id.isEmpty {
|
|
360
|
+
status = sessionTapById(id)
|
|
361
|
+
} else if let x = cmd["x"] as? Double, let y = cmd["y"] as? Double {
|
|
362
|
+
app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: x, dy: y)).tap()
|
|
363
|
+
} else { status = "bad_args" }
|
|
364
|
+
case "type":
|
|
365
|
+
status = sessionType(cmd["text"] as? String ?? "", id: cmd["id"] as? String) ? "ok" : "not_found"
|
|
366
|
+
case "swipe":
|
|
367
|
+
switch (cmd["direction"] as? String ?? "up") {
|
|
368
|
+
case "down": app.swipeDown(); case "left": app.swipeLeft(); case "right": app.swipeRight(); default: app.swipeUp()
|
|
369
|
+
}
|
|
370
|
+
case "back":
|
|
371
|
+
sessionBack()
|
|
372
|
+
case "wait":
|
|
373
|
+
let target = (cmd["id"] as? String).flatMap { $0.isEmpty ? nil : $0 } ?? (cmd["text"] as? String ?? "")
|
|
374
|
+
let waitMs = (cmd["timeoutMs"] as? Int) ?? 5000
|
|
375
|
+
status = sessionWaitFor(target, timeoutMs: waitMs) ? "ok" : "timeout"
|
|
376
|
+
case "tree", "inspect":
|
|
377
|
+
break
|
|
378
|
+
case "screenshot":
|
|
379
|
+
let shot = app.screenshot()
|
|
380
|
+
let a = XCTAttachment(screenshot: shot); a.name = cmd["label"] as? String ?? "session"; a.lifetime = .keepAlways; add(a)
|
|
381
|
+
case "quit":
|
|
382
|
+
print("OCQA_SESSION:bye")
|
|
383
|
+
try? "{\"seq\":\(seq),\"status\":\"ok\",\"action\":\"quit\"}".write(toFile: resultPath, atomically: true, encoding: .utf8)
|
|
384
|
+
return
|
|
385
|
+
default:
|
|
386
|
+
status = "unknown_action"
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// Ack IMMEDIATELY after the action — settle/tree time on busy, animated screens
|
|
390
|
+
// (chat composers especially) made acks exceed the client's budget and cascade
|
|
391
|
+
// into phantom "timeouts" while the action had actually succeeded. The host waits
|
|
392
|
+
// separately for the fresh tree (treeVersion bump) after the ack.
|
|
393
|
+
var extraJson = (action == "type" && !lastTypedInto.isEmpty)
|
|
394
|
+
? ",\"typedInto\":\"\(escapeJSON(lastTypedInto))\"" : ""
|
|
395
|
+
if !loginDetail.isEmpty { extraJson += ",\"detail\":\"\(escapeJSON(loginDetail))\"" }
|
|
396
|
+
try? "{\"seq\":\(seq),\"status\":\"\(status)\",\"action\":\"\(escapeJSON(action))\"\(extraJson)}"
|
|
397
|
+
.write(toFile: resultPath, atomically: true, encoding: .utf8)
|
|
398
|
+
waitForAnimationsToSettle()
|
|
399
|
+
emitSessionTree()
|
|
400
|
+
}
|
|
401
|
+
print("OCQA_SESSION:timeout")
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
private func emitSessionTree() {
|
|
405
|
+
emitUITree(buildAppState(elements: readUITree(app)))
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/// Returns "ok" if tapped, "not_hittable" if the element exists but couldn't be tapped (disabled,
|
|
409
|
+
/// or covered by the keyboard even after dismissing it), or "not_found" if nothing matched. The
|
|
410
|
+
/// distinction tells the caller whether to enter valid input first vs. that the target is absent.
|
|
411
|
+
private func sessionTapById(_ identifier: String) -> String {
|
|
412
|
+
var existedButNotHittable = false
|
|
413
|
+
func tryTap(_ el: XCUIElement) -> Bool {
|
|
414
|
+
guard el.exists else { return false }
|
|
415
|
+
if el.isHittable { el.tap(); return true }
|
|
416
|
+
// Often covered by the keyboard (a submit button below filled fields) — dismiss + retry,
|
|
417
|
+
// like Playwright auto-scrolls a target into view.
|
|
418
|
+
dismissKeyboardIfPresent()
|
|
419
|
+
if el.isHittable { el.tap(); return true }
|
|
420
|
+
existedButNotHittable = true
|
|
421
|
+
return false
|
|
422
|
+
}
|
|
423
|
+
let queries: [XCUIElementQuery] = [app.buttons, app.staticTexts, app.cells, app.links, app.switches, app.textFields, app.secureTextFields, app.textViews]
|
|
424
|
+
for query in queries where tryTap(query[identifier]) { return "ok" }
|
|
425
|
+
// Exact label, then a forgiving case-insensitive "contains" match so callers can tap by the
|
|
426
|
+
// visible text they see in the tree without an exact accessibility id.
|
|
427
|
+
let exactMatches = app.descendants(matching: .any).matching(NSPredicate(format: "label == %@", identifier)).allElementsBoundByIndex
|
|
428
|
+
for match in exactMatches where tryTap(match) { return "ok" }
|
|
429
|
+
let containsMatches = app.descendants(matching: .any).matching(NSPredicate(format: "label CONTAINS[c] %@", identifier)).allElementsBoundByIndex
|
|
430
|
+
for match in containsMatches where tryTap(match) { return "ok" }
|
|
431
|
+
// A semantic control can exist below a SwiftUI ScrollView fold. Match Playwright's
|
|
432
|
+
// scroll-into-view behavior with a small deterministic native bound; never retry the
|
|
433
|
+
// action after it has fired, and never turn an absent control into a pass.
|
|
434
|
+
if existedButNotHittable {
|
|
435
|
+
for _ in 0..<3 {
|
|
436
|
+
app.swipeUp()
|
|
437
|
+
Thread.sleep(forTimeInterval: 0.25)
|
|
438
|
+
for query in queries where tryTap(query[identifier]) { return "ok" }
|
|
439
|
+
let scrolledExact = app.descendants(matching: .any).matching(NSPredicate(format: "label == %@", identifier)).allElementsBoundByIndex
|
|
440
|
+
for match in scrolledExact where tryTap(match) { return "ok" }
|
|
441
|
+
let scrolledContains = app.descendants(matching: .any).matching(NSPredicate(format: "label CONTAINS[c] %@", identifier)).allElementsBoundByIndex
|
|
442
|
+
for match in scrolledContains where tryTap(match) { return "ok" }
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
// Fields are the one control class whose visible name is usually a *placeholder* (not a
|
|
446
|
+
// label) — "Password" must reach the secureTextField or a later `type` lands in whatever
|
|
447
|
+
// field still has keyboard focus.
|
|
448
|
+
if let field = resolveFieldByHint(identifier), tryTap(field) { return "ok" }
|
|
449
|
+
return existedButNotHittable ? "not_hittable" : "not_found"
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/// Resolve a text/secure field by accessibility id, label, or placeholder (exact first, then
|
|
453
|
+
/// case-insensitive contains), then by semantic keyword ("password" → first secure field;
|
|
454
|
+
/// "email"/"username" → first plain text field). Subscript queries miss placeholder-named
|
|
455
|
+
/// fields entirely — this is how login forms actually name their fields.
|
|
456
|
+
private func resolveFieldByHint(_ hint: String) -> XCUIElement? {
|
|
457
|
+
let h = hint.lowercased().trimmingCharacters(in: .whitespacesAndNewlines)
|
|
458
|
+
guard !h.isEmpty else { return nil }
|
|
459
|
+
// textViews included deliberately: chat composers / notes / comment boxes are
|
|
460
|
+
// TextViews, not TextFields — without them "Write a message…" is untypeable.
|
|
461
|
+
let fields = app.textFields.allElementsBoundByIndex + app.secureTextFields.allElementsBoundByIndex + app.textViews.allElementsBoundByIndex
|
|
462
|
+
let existing = fields.filter { $0.exists }
|
|
463
|
+
if let exact = existing.first(where: {
|
|
464
|
+
$0.identifier.lowercased() == h || $0.label.lowercased() == h || ($0.placeholderValue ?? "").lowercased() == h
|
|
465
|
+
}) { return exact }
|
|
466
|
+
if let contains = existing.first(where: {
|
|
467
|
+
($0.identifier + " " + $0.label + " " + ($0.placeholderValue ?? "")).lowercased().contains(h)
|
|
468
|
+
}) { return contains }
|
|
469
|
+
if h.contains("password") {
|
|
470
|
+
let secure = app.secureTextFields.firstMatch
|
|
471
|
+
if secure.exists { return secure }
|
|
472
|
+
}
|
|
473
|
+
if h.contains("email") || h.contains("username") {
|
|
474
|
+
let plain = app.textFields.firstMatch
|
|
475
|
+
if plain.exists { return plain }
|
|
476
|
+
}
|
|
477
|
+
// SwiftUI "fake placeholder" pattern: the visible hint ("Write a message…") is an
|
|
478
|
+
// overlay Text and the real field is a NAMELESS TextField underneath (no id, no
|
|
479
|
+
// label, no placeholderValue). Resolve the overlay text, then return the field
|
|
480
|
+
// whose frame overlaps it.
|
|
481
|
+
let overlays = app.staticTexts.allElementsBoundByIndex.filter { $0.exists }
|
|
482
|
+
if let overlay = overlays.first(where: {
|
|
483
|
+
let l = $0.label.lowercased().trimmingCharacters(in: .whitespacesAndNewlines)
|
|
484
|
+
return !l.isEmpty && (l == h || l.contains(h) || h.contains(l))
|
|
485
|
+
}) {
|
|
486
|
+
let zone = overlay.frame.insetBy(dx: -24, dy: -24)
|
|
487
|
+
if let field = existing.first(where: { $0.frame.intersects(zone) }) {
|
|
488
|
+
return field
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
return nil
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
private func fieldDesc(_ f: XCUIElement) -> String {
|
|
495
|
+
let name = !f.identifier.isEmpty ? f.identifier : (!f.label.isEmpty ? f.label : (f.placeholderValue ?? ""))
|
|
496
|
+
let secure = f.elementType == .secureTextField
|
|
497
|
+
if name.isEmpty { return secure ? "secure field" : "text field" }
|
|
498
|
+
return secure ? "\(name) (secure)" : name
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
private func focusedField() -> XCUIElement? {
|
|
502
|
+
let fields = app.textFields.allElementsBoundByIndex + app.secureTextFields.allElementsBoundByIndex + app.textViews.allElementsBoundByIndex
|
|
503
|
+
return fields.first(where: { $0.exists && (($0.value(forKey: "hasKeyboardFocus") as? Bool) ?? false) })
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
private func dismissKeyboardIfPresent() {
|
|
507
|
+
guard app.keyboards.count > 0 else { return }
|
|
508
|
+
// The keyboard's own return/done key is the only dismissal that works everywhere —
|
|
509
|
+
// SwiftUI apps don't resign focus on background taps by default.
|
|
510
|
+
let kb = app.keyboards.firstMatch
|
|
511
|
+
for label in ["Done", "done", "Return", "return", "Go", "go"] {
|
|
512
|
+
let key = kb.buttons[label]
|
|
513
|
+
if key.exists && key.isHittable {
|
|
514
|
+
key.tap()
|
|
515
|
+
Thread.sleep(forTimeInterval: 0.4)
|
|
516
|
+
break
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
if app.keyboards.count == 0 { return }
|
|
520
|
+
// Fallbacks: background tap (UIKit apps wired to endEditing), then interactive swipe-dismiss.
|
|
521
|
+
app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.08)).tap()
|
|
522
|
+
Thread.sleep(forTimeInterval: 0.4)
|
|
523
|
+
if app.keyboards.count > 0 {
|
|
524
|
+
app.swipeDown()
|
|
525
|
+
Thread.sleep(forTimeInterval: 0.4)
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
private func sessionWaitFor(_ target: String, timeoutMs: Int) -> Bool {
|
|
530
|
+
guard !target.isEmpty else { return false }
|
|
531
|
+
let deadline = Date().addingTimeInterval(Double(timeoutMs) / 1000.0)
|
|
532
|
+
repeat {
|
|
533
|
+
let queries: [XCUIElementQuery] = [app.buttons, app.staticTexts, app.cells, app.links, app.switches, app.textFields, app.otherElements]
|
|
534
|
+
for q in queries where q[target].exists { return true }
|
|
535
|
+
if app.descendants(matching: .any).matching(NSPredicate(format: "label == %@ OR label CONTAINS[c] %@", target, target)).firstMatch.exists { return true }
|
|
536
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
537
|
+
} while Date() < deadline
|
|
538
|
+
return false
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/// Which field the last successful sessionType landed in — reported back to the client so
|
|
542
|
+
/// mis-targeting is visible immediately instead of discovered screenshots later.
|
|
543
|
+
private var lastTypedInto = ""
|
|
544
|
+
|
|
545
|
+
@discardableResult
|
|
546
|
+
private func sessionType(_ text: String, id: String?) -> Bool {
|
|
547
|
+
lastTypedInto = ""
|
|
548
|
+
// Explicit target: accessibility id/label subscripts, then placeholder/semantic resolution.
|
|
549
|
+
// An explicit target that matches NOTHING must fail loudly — silently typing into the
|
|
550
|
+
// still-focused field is exactly how passwords end up appended to the email box.
|
|
551
|
+
// Always REPLACE, never append (Playwright fill() semantics): agents retry failed
|
|
552
|
+
// submits, and append-on-retry turns "Lemonade" into "LemonadeLemonade" — a malformed
|
|
553
|
+
// credential the server rejects with no visible cause.
|
|
554
|
+
if let id = id, !id.isEmpty {
|
|
555
|
+
for field in [app.textFields[id], app.secureTextFields[id], app.textViews[id]] where field.exists {
|
|
556
|
+
replaceText(on: field, with: text); lastTypedInto = fieldDesc(field); return true
|
|
557
|
+
}
|
|
558
|
+
if let field = resolveFieldByHint(id) {
|
|
559
|
+
replaceText(on: field, with: text); lastTypedInto = fieldDesc(field); return true
|
|
560
|
+
}
|
|
561
|
+
return false
|
|
562
|
+
}
|
|
563
|
+
// No target: type into whatever field currently has keyboard focus — this respects a prior
|
|
564
|
+
// tap (e.g. tap the password field by coordinate, then type) — and report which field that is.
|
|
565
|
+
if app.keyboards.firstMatch.exists {
|
|
566
|
+
if let focused = focusedField() {
|
|
567
|
+
lastTypedInto = fieldDesc(focused)
|
|
568
|
+
replaceText(on: focused, with: text)
|
|
569
|
+
return true
|
|
570
|
+
}
|
|
571
|
+
app.typeText(text)
|
|
572
|
+
lastTypedInto = "focused field"
|
|
573
|
+
return true
|
|
574
|
+
}
|
|
575
|
+
// Nothing focused and no usable id — last resort: the first text field or text view.
|
|
576
|
+
for first in [app.textFields.firstMatch, app.textViews.firstMatch] where first.exists {
|
|
577
|
+
replaceText(on: first, with: text); lastTypedInto = fieldDesc(first); return true
|
|
578
|
+
}
|
|
579
|
+
return false
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/// One-call login: find the form, fill both fields, submit, verify — all harness-side.
|
|
583
|
+
/// Agents must never do this step-by-step: iOS clears a secure field whenever editing
|
|
584
|
+
/// re-begins, so any multi-step flow (type → dismiss keyboard → tap) can silently wipe
|
|
585
|
+
/// the password it just typed.
|
|
586
|
+
private func sessionLogin(email: String, password: String) -> (status: String, detail: String) {
|
|
587
|
+
waitForUIStability(timeout: 2.0)
|
|
588
|
+
let textFields = app.textFields.allElementsBoundByIndex.filter { $0.exists && $0.frame.width > 0 }
|
|
589
|
+
let secureFields = app.secureTextFields.allElementsBoundByIndex.filter { $0.exists && $0.frame.width > 0 }
|
|
590
|
+
let emailField = textFields.first { f in
|
|
591
|
+
let hint = (f.identifier + " " + (f.placeholderValue ?? "") + " " + f.label).lowercased()
|
|
592
|
+
return hint.contains("email") || hint.contains("e-mail") || hint.contains("user")
|
|
593
|
+
} ?? (secureFields.isEmpty ? nil : textFields.first)
|
|
594
|
+
guard let emailF = emailField else { return ("no_login_form", "no email/username field visible") }
|
|
595
|
+
guard let passF = secureFields.first else { return ("no_login_form", "no password (secure) field visible") }
|
|
596
|
+
|
|
597
|
+
replaceText(on: emailF, with: email)
|
|
598
|
+
replaceText(on: passF, with: password)
|
|
599
|
+
|
|
600
|
+
func formGone(within seconds: TimeInterval) -> Bool {
|
|
601
|
+
let deadline = Date().addingTimeInterval(seconds)
|
|
602
|
+
while Date() < deadline {
|
|
603
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
604
|
+
if !(passF.exists && emailF.exists) {
|
|
605
|
+
waitForAnimationsToSettle()
|
|
606
|
+
// iOS offers to save the password after a successful sign-in — never wanted
|
|
607
|
+
// in QA; decline so login lands on the app's real post-auth screen.
|
|
608
|
+
for label in ["Not Now", "Not now", "Never", "Never for This Website"] {
|
|
609
|
+
let b = app.buttons[label]
|
|
610
|
+
if b.exists && b.isHittable {
|
|
611
|
+
b.tap()
|
|
612
|
+
waitForAnimationsToSettle()
|
|
613
|
+
break
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
return true
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
return false
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
// Return key first: SwiftUI forms usually wire onSubmit to sign-in, and it doubles as
|
|
623
|
+
// the one keyboard dismissal that works everywhere.
|
|
624
|
+
passF.typeText("\n")
|
|
625
|
+
if formGone(within: 4) { return ("ok", "") }
|
|
626
|
+
|
|
627
|
+
func submitButton() -> XCUIElement? {
|
|
628
|
+
let keywords = ["log in", "login", "sign in", "signin", "continue", "submit", "next", "get started"]
|
|
629
|
+
let buttons = app.buttons.allElementsBoundByIndex.filter { $0.exists }
|
|
630
|
+
return buttons.first { b in
|
|
631
|
+
let t = (b.label + " " + b.identifier).lowercased()
|
|
632
|
+
return keywords.contains { t.contains($0) }
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
guard var submit = submitButton() else { return ("submit_not_found", "no login/continue-style button found") }
|
|
636
|
+
if !submit.isHittable {
|
|
637
|
+
dismissKeyboardIfPresent()
|
|
638
|
+
// Dismissal can re-begin editing on the secure field, which wipes it (iOS security
|
|
639
|
+
// behavior). Detect and re-fill before submitting.
|
|
640
|
+
let pv = (passF.value as? String) ?? ""
|
|
641
|
+
if pv.isEmpty || pv == (passF.placeholderValue ?? "§none§") {
|
|
642
|
+
replaceText(on: passF, with: password)
|
|
643
|
+
dismissKeyboardIfPresent()
|
|
644
|
+
}
|
|
645
|
+
submit = submitButton() ?? submit
|
|
646
|
+
}
|
|
647
|
+
guard submit.isHittable else { return ("submit_not_hittable", "login button stays covered by the keyboard") }
|
|
648
|
+
submit.tap()
|
|
649
|
+
|
|
650
|
+
// Success = the form goes away; otherwise collect error-looking text for the agent.
|
|
651
|
+
if formGone(within: 8) { return ("ok", "") }
|
|
652
|
+
let errTexts = app.staticTexts.allElementsBoundByIndex.filter { $0.exists }.map { $0.label }
|
|
653
|
+
.filter { t in
|
|
654
|
+
let l = t.lowercased()
|
|
655
|
+
return l.contains("error") || l.contains("invalid") || l.contains("incorrect") || l.contains("failed")
|
|
656
|
+
|| l.contains("expired") || l.contains("malformed") || l.contains("please enter") || l.contains("wrong")
|
|
657
|
+
}
|
|
658
|
+
return ("still_on_login", errTexts.prefix(2).joined(separator: " | "))
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
private func sessionBack() {
|
|
662
|
+
let backButton = app.navigationBars.buttons.firstMatch
|
|
663
|
+
if backButton.exists && backButton.isHittable {
|
|
664
|
+
backButton.tap()
|
|
665
|
+
} else {
|
|
666
|
+
app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0.5))
|
|
667
|
+
.press(forDuration: 0.05, thenDragTo: app.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5)))
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/// Replays an explicit, config-driven login flow (OCQA_LOGIN_STEPS) before exploration — for the
|
|
672
|
+
/// custom login UIs the heuristic preamble can't parse, which are the #1 reason a real app stays
|
|
673
|
+
/// invisible to AutoTap. Steps are a JSON array of {action: type|tap|wait, target: <id-or-label>,
|
|
674
|
+
/// value?: <text; "$TEST_EMAIL"/"$TEST_PASSWORD" substituted from stored creds>, timeoutMs?}.
|
|
675
|
+
/// Returns true if any step ran (so the caller skips the heuristic preamble). Tolerant: a failed
|
|
676
|
+
/// step is logged but doesn't abort — exploration still proceeds, and the coverage eval reveals
|
|
677
|
+
/// whether login was actually passed (login_present without a wall).
|
|
678
|
+
private func executeLoginSteps(_ json: String, email: String, password: String) -> Bool {
|
|
679
|
+
guard let data = json.data(using: .utf8),
|
|
680
|
+
let steps = (try? JSONSerialization.jsonObject(with: data)) as? [[String: Any]],
|
|
681
|
+
!steps.isEmpty else { return false }
|
|
682
|
+
print("OCQA_STATE:login_replay_started steps=\(steps.count)")
|
|
683
|
+
waitForUIStability(timeout: 2.0)
|
|
684
|
+
for (i, step) in steps.enumerated() {
|
|
685
|
+
let action = (step["action"] as? String ?? "").lowercased()
|
|
686
|
+
let target = step["target"] as? String ?? ""
|
|
687
|
+
let value = (step["value"] as? String ?? "")
|
|
688
|
+
.replacingOccurrences(of: "$TEST_EMAIL", with: email)
|
|
689
|
+
.replacingOccurrences(of: "$TEST_PASSWORD", with: password)
|
|
690
|
+
var status = "ok"
|
|
691
|
+
switch action {
|
|
692
|
+
case "type": status = sessionType(value, id: target) ? "ok" : "field_not_found"
|
|
693
|
+
case "tap": status = sessionTapById(target)
|
|
694
|
+
case "wait":
|
|
695
|
+
let ms = (step["timeoutMs"] as? Int) ?? 5000
|
|
696
|
+
status = sessionWaitFor(target, timeoutMs: ms) ? "ok" : "timeout"
|
|
697
|
+
default: status = "unknown_action"
|
|
698
|
+
}
|
|
699
|
+
print("OCQA_ACTION:{\"type\":\"login_\(escapeJSON(action))\",\"target\":\"\(escapeJSON(target))\",\"step\":\(i + 1),\"status\":\"\(escapeJSON(status))\",\"reason\":\"login_replay\"}")
|
|
700
|
+
waitForAnimationsToSettle()
|
|
701
|
+
}
|
|
702
|
+
print("OCQA_STATE:login_replay_done")
|
|
703
|
+
return true
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
// MARK: - Flow replay (deterministic E2E tests)
|
|
707
|
+
//
|
|
708
|
+
// Replays a recorded/authored Flow (see docs/flows-architecture.md) step-by-step and checks its
|
|
709
|
+
// assertions. Deterministic by construction: fresh launch, wait-for-stability between steps,
|
|
710
|
+
// poll-with-timeout assertions (never instant-checks), and the same id→label→contains selector
|
|
711
|
+
// resolution the session uses. Exact assertions gate pass/fail; `assert_ai` is opt-in and routed
|
|
712
|
+
// host-side (skipped when no judge is configured, so the deterministic core runs everywhere).
|
|
713
|
+
func testReplayFlow() {
|
|
714
|
+
let flowJson = resolve("OCQA_FLOW_JSON")
|
|
715
|
+
guard !flowJson.isEmpty,
|
|
716
|
+
let data = flowJson.data(using: .utf8),
|
|
717
|
+
let flow = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any],
|
|
718
|
+
let steps = flow["steps"] as? [[String: Any]] else {
|
|
719
|
+
print("OCQA_FLOW_RESULT:{\"passed\":false,\"total\":0,\"failed\":0,\"error\":\"no OCQA_FLOW_JSON with steps\"}")
|
|
720
|
+
return
|
|
721
|
+
}
|
|
722
|
+
// Variable substitution: $TEST_EMAIL/$TEST_PASSWORD from creds, plus any OCQA_FLOW_VARS.
|
|
723
|
+
var vars: [String: String] = ["TEST_EMAIL": resolve("OCQA_TEST_EMAIL", fallback: "test@example.com"),
|
|
724
|
+
"TEST_PASSWORD": resolve("OCQA_TEST_PASSWORD", fallback: "TestPass123!")]
|
|
725
|
+
if let vd = flow["vars"] as? [String: Any] {
|
|
726
|
+
for (k, v) in vd { vars[k] = "\(v)" }
|
|
727
|
+
}
|
|
728
|
+
if let extra = (try? JSONSerialization.jsonObject(with: Data(resolve("OCQA_FLOW_VARS").utf8))) as? [String: Any] {
|
|
729
|
+
for (k, v) in extra { vars[k] = "\(v)" }
|
|
730
|
+
}
|
|
731
|
+
let releaseContract = flow["releaseContract"] as? [String: Any]
|
|
732
|
+
let contractName = releaseContract?["name"] as? String ?? ""
|
|
733
|
+
let contractCriticality = releaseContract?["criticality"] as? String ?? ""
|
|
734
|
+
let evidenceKind = contractName.isEmpty ? (flow["kind"] as? String ?? "flow") : "release-contract"
|
|
735
|
+
func subst(_ s: String) -> String {
|
|
736
|
+
var out = s
|
|
737
|
+
for (k, v) in vars { out = out.replacingOccurrences(of: "$\(k)", with: v) }
|
|
738
|
+
return out
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
if !targetBundleId.isEmpty { app.launch() } else { app.launch() }
|
|
742
|
+
waitForUIStability(timeout: 3.0)
|
|
743
|
+
let contractStart = contractName.isEmpty ? "" : " contract=\(escapeJSON(contractName))"
|
|
744
|
+
print("OCQA_FLOW_RESULT:started total=\(steps.count) name=\(escapeJSON(flow["name"] as? String ?? "flow")) kind=\(evidenceKind)\(contractStart)")
|
|
745
|
+
|
|
746
|
+
var failed = 0
|
|
747
|
+
var executed = 0
|
|
748
|
+
var aiIndex = 0
|
|
749
|
+
for (i, raw) in steps.enumerated() {
|
|
750
|
+
let idx = i + 1
|
|
751
|
+
executed = idx
|
|
752
|
+
let taskName = (raw["__tappTask"] as? [String: Any])?["name"] as? String ?? ""
|
|
753
|
+
// A step is either {action: value} sugar or {action:..., target/value/...}. Normalize.
|
|
754
|
+
let (action, step) = normalizeFlowStep(raw)
|
|
755
|
+
let target = subst((step["target"] as? String) ?? "")
|
|
756
|
+
let value = subst((step["value"] as? String) ?? "")
|
|
757
|
+
let timeoutMs = (step["timeoutMs"] as? Int) ?? 6000
|
|
758
|
+
var status = "pass"
|
|
759
|
+
var detail = ""
|
|
760
|
+
|
|
761
|
+
switch action {
|
|
762
|
+
case "tap":
|
|
763
|
+
status = sessionTapById(target) == "ok" ? "pass" : "fail"
|
|
764
|
+
if status == "fail" { detail = "could not tap ‘\(target)’" }
|
|
765
|
+
case "type":
|
|
766
|
+
status = sessionType(value, id: target.isEmpty ? nil : target) ? "pass" : "fail"
|
|
767
|
+
if status == "fail" { detail = "no field ‘\(target)’ to type into" }
|
|
768
|
+
case "swipe":
|
|
769
|
+
switch target.lowercased() { case "down": app.swipeDown(); case "left": app.swipeLeft(); case "right": app.swipeRight(); default: app.swipeUp() }
|
|
770
|
+
case "back":
|
|
771
|
+
_ = tryGoBack()
|
|
772
|
+
case "wait":
|
|
773
|
+
Thread.sleep(forTimeInterval: Double(timeoutMs) / 1000.0)
|
|
774
|
+
case "wait_for":
|
|
775
|
+
status = sessionWaitFor(target, timeoutMs: timeoutMs) ? "pass" : "fail"
|
|
776
|
+
if status == "fail" { detail = "‘\(target)’ never appeared within \(timeoutMs)ms" }
|
|
777
|
+
case "assert_screen":
|
|
778
|
+
let ok = pollUntil(timeoutMs: timeoutMs) { (detectTitle(readUITree(app)) ?? "").caseInsensitiveCompare(value.isEmpty ? target : value) == .orderedSame }
|
|
779
|
+
status = ok ? "pass" : "fail"
|
|
780
|
+
if !ok { detail = "expected screen ‘\(value.isEmpty ? target : value)’, saw ‘\(detectTitle(readUITree(app)) ?? "?")’" }
|
|
781
|
+
case "assert_exists":
|
|
782
|
+
let ok = sessionWaitFor(target, timeoutMs: timeoutMs)
|
|
783
|
+
status = ok ? "pass" : "fail"
|
|
784
|
+
if !ok { detail = "‘\(target)’ not found" }
|
|
785
|
+
case "assert_absent":
|
|
786
|
+
waitForUIStability(timeout: 1.5)
|
|
787
|
+
let present = elementPresent(target)
|
|
788
|
+
status = present ? "fail" : "pass"
|
|
789
|
+
if present { detail = "‘\(target)’ was present but should be absent" }
|
|
790
|
+
case "assert_text":
|
|
791
|
+
let of = subst((step["of"] as? String) ?? target)
|
|
792
|
+
let needle = subst((step["contains"] as? String) ?? value)
|
|
793
|
+
let ok = pollUntil(timeoutMs: timeoutMs) { elementTextContains(of, needle) }
|
|
794
|
+
status = ok ? "pass" : "fail"
|
|
795
|
+
if !ok { detail = "‘\(of)’ did not contain ‘\(needle)’" }
|
|
796
|
+
case "assert_ai":
|
|
797
|
+
aiIndex += 1
|
|
798
|
+
let claim = value.isEmpty ? target : value
|
|
799
|
+
let (aiStatus, aiDetail) = judgeWithAI(index: aiIndex, claim: claim)
|
|
800
|
+
status = aiStatus; detail = aiDetail
|
|
801
|
+
default:
|
|
802
|
+
status = "fail"; detail = "unknown action ‘\(action)’"
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
// App-death is a hard failure for any step (real crash during the flow).
|
|
806
|
+
if app.state != .runningForeground && action != "assert_absent" {
|
|
807
|
+
status = "fail"; detail = "app left the foreground (crash?) during ‘\(action)’"
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
let isAssert = action.hasPrefix("assert_")
|
|
811
|
+
let taskEvidence = taskName.isEmpty ? "" : ",\"task\":\"\(escapeJSON(taskName))\""
|
|
812
|
+
let contractEvidence = contractName.isEmpty ? "" : ",\"contract\":\"\(escapeJSON(contractName))\""
|
|
813
|
+
print("OCQA_FLOW_STEP:{\"index\":\(idx),\"action\":\"\(escapeJSON(action))\",\"target\":\"\(escapeJSON(target.isEmpty ? value : target))\",\"assert\":\(isAssert),\"status\":\"\(status)\",\"detail\":\"\(escapeJSON(detail))\"\(taskEvidence)\(contractEvidence)}")
|
|
814
|
+
if status == "fail" {
|
|
815
|
+
failed += 1
|
|
816
|
+
// A failed step surfaces as a finding, with the same shape QA findings use.
|
|
817
|
+
let screen = detectTitle(readUITree(app)) ?? "Unknown"
|
|
818
|
+
print("OCQA_ISSUE:{\"type\":\"flow_assertion_failed\",\"severity\":\"high\",\"title\":\"\(escapeJSON("Step \(idx) (\(action)) failed: \(detail)"))\",\"screen\":\"\(escapeJSON(screen))\",\"step\":\(idx)}")
|
|
819
|
+
// Stop on the first failure by default — later steps assume earlier ones succeeded.
|
|
820
|
+
if (flow["continueOnFailure"] as? Bool) != true { break }
|
|
821
|
+
}
|
|
822
|
+
waitForAnimationsToSettle()
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
let shot = app.screenshot()
|
|
826
|
+
let att = XCTAttachment(screenshot: shot); att.name = "flow_final"; att.lifetime = .keepAlways; add(att)
|
|
827
|
+
let contractResult = contractName.isEmpty ? "" : ",\"contract\":\"\(escapeJSON(contractName))\",\"criticality\":\"\(escapeJSON(contractCriticality))\""
|
|
828
|
+
print("OCQA_FLOW_RESULT:{\"passed\":\(failed == 0),\"name\":\"\(escapeJSON(flow["name"] as? String ?? "flow"))\",\"kind\":\"\(escapeJSON(evidenceKind))\"\(contractResult),\"total\":\(steps.count),\"executed\":\(executed),\"failed\":\(failed)}")
|
|
829
|
+
if failed > 0 { XCTFail("Flow had \(failed) failed step(s)") }
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
/// Accepts both sugar (`{ tap: "Sign In" }`) and explicit (`{ action: "tap", target: "Sign In" }`).
|
|
833
|
+
private func normalizeFlowStep(_ raw: [String: Any]) -> (String, [String: Any]) {
|
|
834
|
+
if let action = raw["action"] as? String { return (action.lowercased(), raw) }
|
|
835
|
+
// Sugar: the single key is the action; a string value is target, an object is the params.
|
|
836
|
+
for (k, v) in raw where !k.hasPrefix("__") {
|
|
837
|
+
if let s = v as? String { return (k.lowercased(), ["target": s, "value": s]) }
|
|
838
|
+
if let o = v as? [String: Any] {
|
|
839
|
+
var params = o
|
|
840
|
+
// map {field:} → target for type steps
|
|
841
|
+
if let f = o["field"] as? String, params["target"] == nil { params["target"] = f }
|
|
842
|
+
return (k.lowercased(), params)
|
|
843
|
+
}
|
|
844
|
+
return (k.lowercased(), [:])
|
|
845
|
+
}
|
|
846
|
+
return ("noop", [:])
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
private func pollUntil(timeoutMs: Int, _ cond: () -> Bool) -> Bool {
|
|
850
|
+
let deadline = Date().addingTimeInterval(Double(timeoutMs) / 1000.0)
|
|
851
|
+
repeat { if cond() { return true }; Thread.sleep(forTimeInterval: 0.25) } while Date() < deadline
|
|
852
|
+
return false
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
private func elementPresent(_ target: String) -> Bool {
|
|
856
|
+
guard !target.isEmpty else { return false }
|
|
857
|
+
let queries: [XCUIElementQuery] = [app.buttons, app.staticTexts, app.cells, app.links, app.switches, app.textFields, app.otherElements]
|
|
858
|
+
for q in queries where q[target].exists { return true }
|
|
859
|
+
return app.descendants(matching: .any).matching(NSPredicate(format: "label == %@ OR label CONTAINS[c] %@", target, target)).firstMatch.exists
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
private func elementTextContains(_ of: String, _ needle: String) -> Bool {
|
|
863
|
+
let lc = needle.lowercased()
|
|
864
|
+
for el in readUITree(app) {
|
|
865
|
+
let id = el.identifier.trimmingCharacters(in: .whitespaces)
|
|
866
|
+
let label = el.label.trimmingCharacters(in: .whitespaces)
|
|
867
|
+
if id == of || label == of || label.lowercased().contains(of.lowercased()) {
|
|
868
|
+
if el.value.lowercased().contains(lc) || el.label.lowercased().contains(lc) { return true }
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
return false
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/// Routes an `assert_ai` claim to a host-side judge over a file channel (same pattern as vision
|
|
875
|
+
/// escalation). No judge configured ⇒ skipped (not failed), so deterministic flows run anywhere.
|
|
876
|
+
private func judgeWithAI(index: Int, claim: String) -> (String, String) {
|
|
877
|
+
let responsePath = resolve("OCQA_FLOW_AI_RESPONSE_PATH")
|
|
878
|
+
let imageDir = resolve("OCQA_FLOW_AI_IMAGE_DIR")
|
|
879
|
+
guard !responsePath.isEmpty, !imageDir.isEmpty else { return ("skip", "no AI judge configured") }
|
|
880
|
+
try? FileManager.default.createDirectory(atPath: imageDir, withIntermediateDirectories: true)
|
|
881
|
+
let imagePath = (imageDir as NSString).appendingPathComponent("assert-\(index).png")
|
|
882
|
+
guard (try? app.screenshot().pngRepresentation.write(to: URL(fileURLWithPath: imagePath))) != nil else { return ("skip", "screenshot failed") }
|
|
883
|
+
try? FileManager.default.removeItem(atPath: responsePath)
|
|
884
|
+
print("OCQA_FLOW_AI_QUERY:{\"index\":\(index),\"claim\":\"\(escapeJSON(claim))\",\"image\":\"\(escapeJSON(imagePath))\"}")
|
|
885
|
+
let deadline = Date().addingTimeInterval(60)
|
|
886
|
+
while Date() < deadline {
|
|
887
|
+
Thread.sleep(forTimeInterval: 0.4)
|
|
888
|
+
guard let data = try? Data(contentsOf: URL(fileURLWithPath: responsePath)),
|
|
889
|
+
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
|
890
|
+
(obj["index"] as? Int) == index else { continue }
|
|
891
|
+
let pass = (obj["pass"] as? Bool) ?? false
|
|
892
|
+
return (pass ? "pass" : "fail", (obj["reason"] as? String) ?? "")
|
|
893
|
+
}
|
|
894
|
+
return ("skip", "AI judge timed out")
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
// MARK: - Full Autonomous Exploration
|
|
898
|
+
|
|
899
|
+
func testAutonomousExploration() {
|
|
900
|
+
// Autonomous QA is a controlled-state run, not a continuation of whichever screen a
|
|
901
|
+
// developer or previous Tapp invocation left foregrounded. setUp deliberately activates
|
|
902
|
+
// for the single-action/session utilities; reset this full run explicitly and forward the
|
|
903
|
+
// same launch configuration used by deterministic replay.
|
|
904
|
+
if !targetBundleId.isEmpty { app.terminate() }
|
|
905
|
+
app.launchArguments = appLaunchArgs
|
|
906
|
+
app.launchEnvironment = appLaunchEnv
|
|
907
|
+
app.launch()
|
|
908
|
+
_ = app.wait(for: .runningForeground, timeout: 10)
|
|
909
|
+
|
|
910
|
+
let maxActions = self.maxActions
|
|
911
|
+
let timeoutSeconds = Double(self.timeoutSeconds)
|
|
912
|
+
var inputOverrides = (config["OCQA_INPUT_OVERRIDES"] as? [String: String]) ?? [:]
|
|
913
|
+
|
|
914
|
+
// (Crash safety net registered in setUp — covers launch-phase and mid-run deaths.)
|
|
915
|
+
|
|
916
|
+
var visitedStates = Set<String>()
|
|
917
|
+
var stateTransitions: [(from: String, to: String, action: String)] = []
|
|
918
|
+
var actionCounts: [String: Int] = [:]
|
|
919
|
+
var stateActionCounts: [String: Int] = [:] // "stateHash|actionKey" -> count
|
|
920
|
+
// "screenTitle|actionKey" of text fields we've typed into — SCREEN-scoped because anonymous
|
|
921
|
+
// action keys (position buckets) collide across screens (login Password and signup Confirm
|
|
922
|
+
// Password landed in the same bucket, wrongly marking the latter "filled"). Drives
|
|
923
|
+
// form-completion steering's unfilled check; membership is permanent per run (termination).
|
|
924
|
+
var typedFieldKeys = Set<String>()
|
|
925
|
+
// Screens where the one-shot stuck-escape dismiss has already been tried.
|
|
926
|
+
var stuckDismissTried = Set<String>()
|
|
927
|
+
// Trap escapes: a screen whose back/dismiss controls are all dead (e.g. a broken custom
|
|
928
|
+
// Back button with the system gesture disabled) would otherwise strand the WHOLE remaining
|
|
929
|
+
// budget. A relaunch always returns to the root; capped so a relaunch-loop can't eat the run.
|
|
930
|
+
var relaunchEscapesUsed = 0
|
|
931
|
+
var actionCooldownUntilStep: [String: Int] = [:] // actionKey -> next allowed step
|
|
932
|
+
var lastActionKey: String?
|
|
933
|
+
var lastActionFromStateHash: String?
|
|
934
|
+
var previousStateHash: String?
|
|
935
|
+
var repeatedStateCount = 0
|
|
936
|
+
var actionCount = 0
|
|
937
|
+
var issues: [(type: String, severity: String, title: String, desc: String)] = []
|
|
938
|
+
/// De-dupes high-impact findings (failed submits, dead controls) so each is reported once.
|
|
939
|
+
var reportedIssueKeys = Set<String>()
|
|
940
|
+
/// Per-screen scroll-to-discover state — reveals below-the-fold content so off-screen
|
|
941
|
+
/// controls (and the issues on them) are actually reached.
|
|
942
|
+
var screenScrollDepth: [String: Int] = [:]
|
|
943
|
+
var screenScrolledToBottom = Set<String>()
|
|
944
|
+
var screenTitles: [String: String] = [:] // hash -> title
|
|
945
|
+
/// Tracks element keys that appear in multiple distinct screen hashes — likely persistent nav
|
|
946
|
+
var elementScreenPresence: [String: Set<String>] = [:]
|
|
947
|
+
var totalDistinctStates = 0
|
|
948
|
+
var recentStateHashes: [String] = []
|
|
949
|
+
var actionsSinceNewState = 0
|
|
950
|
+
var sameScreenStreak = 0
|
|
951
|
+
var screenTextEntryCount: [String: Int] = [:]
|
|
952
|
+
/// Screen title -> count of submits that left us on the same screen (failed login/form)
|
|
953
|
+
var failedSubmits: [String: Int] = [:]
|
|
954
|
+
/// Set once any sign-in (a submit on a screen with a password field) navigates the user
|
|
955
|
+
/// forward. Suppresses false "auth failed" / loop / unresponsive findings when the explorer
|
|
956
|
+
/// later re-pokes the login form (e.g. after tapping Sign Out).
|
|
957
|
+
var authSucceeded = false
|
|
958
|
+
/// Per-screen-title visit count — drives "don't revisit explored screens" logic
|
|
959
|
+
var screenVisitCount: [String: Int] = [:]
|
|
960
|
+
/// Per-screen-title set of action keys already tried — never repeat an action on the same screen
|
|
961
|
+
var screenActionsTried: [String: Set<String>] = [:]
|
|
962
|
+
/// Remembered transitions: "fromTitle|actionKey" -> destination screen title
|
|
963
|
+
var knownTransitions: [String: String] = [:]
|
|
964
|
+
/// Affordance fingerprint of the destination reached by "fromTitle|actionKey" — the set of
|
|
965
|
+
/// interactable-element labels on the destination screen. Used for template-sibling detection.
|
|
966
|
+
var destFingerprint: [String: String] = [:]
|
|
967
|
+
/// Per hub title: affordance fingerprint -> the distinct action keys that led to it. A hub
|
|
968
|
+
/// with any fingerprint reached by >= 2 keys is a repeating template list (e.g. a list of
|
|
969
|
+
/// structurally-identical detail screens), so its remaining siblings can be deferred.
|
|
970
|
+
var hubKeysByFingerprint: [String: [String: Set<String>]] = [:]
|
|
971
|
+
/// Set after performing an action; resolved on next iteration to populate knownTransitions
|
|
972
|
+
var pendingTransitionFrom: (title: String, actionKey: String, hash: String)? = nil
|
|
973
|
+
/// Tracks which tab bar position (0-4) to try next for rotation
|
|
974
|
+
var nextTabRotation = 0
|
|
975
|
+
/// Count of distinct screen titles discovered so far
|
|
976
|
+
var knownScreenTitles = Set<String>()
|
|
977
|
+
/// Step at which we last forced a tab switch
|
|
978
|
+
var lastTabSwitchStep = 0
|
|
979
|
+
/// Tab-bar tabs already visited by the guaranteed early sweep (by label, or position for
|
|
980
|
+
/// unlabeled tabs). See the "Guaranteed tab sweep" block in the main loop.
|
|
981
|
+
var sweptTabLabels = Set<String>()
|
|
982
|
+
/// Sweep lifecycle: done stops the per-iteration tab-bar query entirely (it costs an XCUI
|
|
983
|
+
/// query per iteration); attempts bounds how long a no-tab-bar app keeps paying for it.
|
|
984
|
+
var tabSweepDone = false
|
|
985
|
+
var tabSweepAttempts = 0
|
|
986
|
+
let startTime = Date()
|
|
987
|
+
|
|
988
|
+
// ---- Interactive mid-run input ----
|
|
989
|
+
// When the host enables it, the harness pauses on the first screen with input fields and
|
|
990
|
+
// waits for the user to supply values (written to OCQA_INPUT_RESPONSE_PATH). Time spent
|
|
991
|
+
// blocked here is tracked in `totalWaitSeconds` and excluded from the exploration timeout.
|
|
992
|
+
var totalWaitSeconds = 0.0
|
|
993
|
+
var promptedScreens = Set<String>()
|
|
994
|
+
var dontAskAgain = false
|
|
995
|
+
var interactiveInputEnabled = resolve("OCQA_INTERACTIVE_INPUT") == "1"
|
|
996
|
+
let inputResponsePath = resolve("OCQA_INPUT_RESPONSE_PATH")
|
|
997
|
+
let inputWaitTimeout = Double(resolve("OCQA_INPUT_WAIT_TIMEOUT", fallback: "30")) ?? 30
|
|
998
|
+
|
|
999
|
+
// ---- In-loop vision escalation (opt-in, off by default) ----
|
|
1000
|
+
// When the a11y tree goes blank/stuck the structural explorer is out of moves. If the host
|
|
1001
|
+
// enabled it, screenshot the screen and ask the host's vision model for the single next tap
|
|
1002
|
+
// (normalized coords) or gesture — the model call stays HOST-side; the harness only requests
|
|
1003
|
+
// via OCQA_VISION_QUERY and executes the reply. Budgeted so cost/latency stay bounded.
|
|
1004
|
+
let visionEscalationEnabled = resolve("OCQA_VISION_ESCALATION") == "1"
|
|
1005
|
+
let visionResponsePath = resolve("OCQA_VISION_RESPONSE_PATH")
|
|
1006
|
+
let visionImageDir = resolve("OCQA_VISION_IMAGE_DIR")
|
|
1007
|
+
let visionEscalationBudget = Int(resolve("OCQA_VISION_BUDGET", fallback: "4")) ?? 4
|
|
1008
|
+
let visionWaitTimeout = Double(resolve("OCQA_VISION_WAIT_TIMEOUT", fallback: "60")) ?? 60
|
|
1009
|
+
var visionEscalationsUsed = 0
|
|
1010
|
+
|
|
1011
|
+
if !targetBundleId.isEmpty {
|
|
1012
|
+
app.activate()
|
|
1013
|
+
} else {
|
|
1014
|
+
app.launch()
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
// Wait for app to settle before the first capture. Uses waitForUIStability (requires TWO
|
|
1018
|
+
// consecutive stable element-count reads, not one) — the same check testReplayFlow uses on
|
|
1019
|
+
// launch. A single-stable-read check can break mid-animation for a launch-time `.sheet`
|
|
1020
|
+
// (e.g. a "Get Started" welcome sheet over a TabView): the sheet's own nav bar isn't in the
|
|
1021
|
+
// tree yet, so the first capture reads the screen underneath instead of the sheet, and
|
|
1022
|
+
// grounding/replay disagree on the entry screen. Requiring two consecutive stable reads
|
|
1023
|
+
// reliably lands after the presentation animation finishes.
|
|
1024
|
+
_ = app.descendants(matching: .any).firstMatch.waitForExistence(timeout: 3)
|
|
1025
|
+
_ = waitForUIStability(timeout: 2.0)
|
|
1026
|
+
|
|
1027
|
+
print("OCQA_STATE:exploration_started max_actions=\(maxActions)")
|
|
1028
|
+
|
|
1029
|
+
let testEmail = resolve("OCQA_TEST_EMAIL")
|
|
1030
|
+
let testPassword = resolve("OCQA_TEST_PASSWORD")
|
|
1031
|
+
|
|
1032
|
+
// --- Explicit login replay (config-driven): a recorded type/tap/wait sequence for custom
|
|
1033
|
+
// login UIs the heuristic preamble below can't parse. When configured it takes precedence. ---
|
|
1034
|
+
let ranExplicitLogin = executeLoginSteps(resolve("OCQA_LOGIN_STEPS"), email: testEmail, password: testPassword)
|
|
1035
|
+
|
|
1036
|
+
// --- Login preamble: if credentials are provided and login fields are visible, log in first ---
|
|
1037
|
+
if !ranExplicitLogin, !testEmail.isEmpty, !testPassword.isEmpty {
|
|
1038
|
+
waitForUIStability(timeout: 2.0) // let app fully settle
|
|
1039
|
+
let allTextFields = app.textFields.allElementsBoundByIndex.filter { $0.exists && $0.frame.width > 0 }
|
|
1040
|
+
let allSecureFields = app.secureTextFields.allElementsBoundByIndex.filter { $0.exists && $0.frame.width > 0 }
|
|
1041
|
+
print("OCQA_STATE:login_preamble_fields textFields=\(allTextFields.count) secureFields=\(allSecureFields.count)")
|
|
1042
|
+
|
|
1043
|
+
let emailField = allTextFields.first { f in
|
|
1044
|
+
let hint = (f.identifier + " " + (f.placeholderValue ?? "") + " " + f.label).lowercased()
|
|
1045
|
+
return hint.contains("email") || hint.contains("e-mail")
|
|
1046
|
+
} ?? (allSecureFields.count > 0 ? allTextFields.first : nil)
|
|
1047
|
+
|
|
1048
|
+
let passwordField = allSecureFields.first
|
|
1049
|
+
?? allTextFields.first { f in
|
|
1050
|
+
let hint = (f.identifier + " " + (f.placeholderValue ?? "") + " " + f.label).lowercased()
|
|
1051
|
+
return hint.contains("password") || hint.contains("passcode")
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
if let emailF = emailField, emailF.exists, let passF = passwordField, passF.exists {
|
|
1055
|
+
// ---- Pause-first at credential surfaces (attended runs must never get surprise
|
|
1056
|
+
// typing). When the host wired interactive input, ask BEFORE the preamble types:
|
|
1057
|
+
// the prompt shows the known email as its default; Submit overrides the values,
|
|
1058
|
+
// Skip skips login entirely, Use-defaults or a 30s timeout (unattended run)
|
|
1059
|
+
// proceeds with the configured credentials — the old silent behavior becomes the
|
|
1060
|
+
// fallback, not the default.
|
|
1061
|
+
var preambleEmail = testEmail
|
|
1062
|
+
var preamblePassword = testPassword
|
|
1063
|
+
var skipPreambleLogin = false
|
|
1064
|
+
if interactiveInputEnabled, !inputResponsePath.isEmpty {
|
|
1065
|
+
let navTitle = app.navigationBars.firstMatch.exists ? app.navigationBars.firstMatch.identifier : ""
|
|
1066
|
+
let promptScreen = navTitle.isEmpty ? "Sign In" : navTitle
|
|
1067
|
+
let emailKey = emailF.identifier.isEmpty ? "email" : emailF.identifier
|
|
1068
|
+
let passKey = passF.identifier.isEmpty ? "password" : passF.identifier
|
|
1069
|
+
let descriptors = [
|
|
1070
|
+
InputDescriptor(key: emailKey, label: "Email", secure: false, placeholder: emailF.placeholderValue ?? ""),
|
|
1071
|
+
InputDescriptor(key: passKey, label: "Password", secure: true, placeholder: passF.placeholderValue ?? ""),
|
|
1072
|
+
]
|
|
1073
|
+
promptedScreens.insert(normalizeKey(promptScreen)) // don't re-ask in the main loop
|
|
1074
|
+
let action = awaitInteractiveInput(
|
|
1075
|
+
requestId: UUID().uuidString,
|
|
1076
|
+
screenTitle: promptScreen,
|
|
1077
|
+
descriptors: descriptors,
|
|
1078
|
+
responsePath: inputResponsePath,
|
|
1079
|
+
waitTimeout: inputWaitTimeout,
|
|
1080
|
+
overrides: &inputOverrides,
|
|
1081
|
+
totalWaitSeconds: &totalWaitSeconds,
|
|
1082
|
+
dontAskAgain: &dontAskAgain,
|
|
1083
|
+
interactiveEnabled: &interactiveInputEnabled
|
|
1084
|
+
)
|
|
1085
|
+
if action == "skip" {
|
|
1086
|
+
skipPreambleLogin = true
|
|
1087
|
+
print("OCQA_STATE:login_preamble_skipped_by_user")
|
|
1088
|
+
} else if action == "submit" {
|
|
1089
|
+
let screenKey = normalizeKey(promptScreen)
|
|
1090
|
+
if let v = inputOverrides["screen:\(screenKey)|\(emailKey)"] ?? inputOverrides[emailKey], !v.isEmpty {
|
|
1091
|
+
preambleEmail = v
|
|
1092
|
+
}
|
|
1093
|
+
if let v = inputOverrides["screen:\(screenKey)|\(passKey)"] ?? inputOverrides[passKey], !v.isEmpty {
|
|
1094
|
+
preamblePassword = v
|
|
1095
|
+
}
|
|
1096
|
+
} // "defaults" / "dont_ask" / "timeout" → proceed with the configured creds
|
|
1097
|
+
}
|
|
1098
|
+
|
|
1099
|
+
if !skipPreambleLogin {
|
|
1100
|
+
print("OCQA_STATE:login_preamble_attempting")
|
|
1101
|
+
emailF.tap()
|
|
1102
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1103
|
+
emailF.typeText(preambleEmail)
|
|
1104
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1105
|
+
|
|
1106
|
+
passF.tap()
|
|
1107
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1108
|
+
passF.typeText(preamblePassword)
|
|
1109
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1110
|
+
|
|
1111
|
+
// Dismiss keyboard
|
|
1112
|
+
let keyboard = app.keyboards.firstMatch
|
|
1113
|
+
if keyboard.exists {
|
|
1114
|
+
let aboveKeyboard = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.3))
|
|
1115
|
+
aboveKeyboard.tap()
|
|
1116
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
// Find and tap login/sign-in button
|
|
1120
|
+
let loginLabels = ["Log In", "Login", "Sign In", "Sign in", "log in", "LOG IN", "SIGN IN"]
|
|
1121
|
+
var tappedLogin = false
|
|
1122
|
+
for label in loginLabels {
|
|
1123
|
+
let btn = app.buttons[label]
|
|
1124
|
+
if btn.exists && btn.isHittable {
|
|
1125
|
+
btn.tap()
|
|
1126
|
+
tappedLogin = true
|
|
1127
|
+
break
|
|
1128
|
+
}
|
|
1129
|
+
let st = app.staticTexts[label]
|
|
1130
|
+
if st.exists && st.isHittable {
|
|
1131
|
+
st.tap()
|
|
1132
|
+
tappedLogin = true
|
|
1133
|
+
break
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
if tappedLogin {
|
|
1137
|
+
print("OCQA_STATE:login_preamble_submitted")
|
|
1138
|
+
Thread.sleep(forTimeInterval: 2.0)
|
|
1139
|
+
waitForUIStability(timeout: 4.0)
|
|
1140
|
+
} else {
|
|
1141
|
+
// Two-step auth: try entering email + tapping Continue, then handle password on next screen
|
|
1142
|
+
let continueLabels = ["Continue", "Next", "Submit", "Proceed", "Get Started"]
|
|
1143
|
+
var tappedContinue = false
|
|
1144
|
+
for label in continueLabels {
|
|
1145
|
+
let btn = app.buttons[label]
|
|
1146
|
+
if btn.exists && btn.isHittable {
|
|
1147
|
+
btn.tap()
|
|
1148
|
+
tappedContinue = true
|
|
1149
|
+
break
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
if tappedContinue {
|
|
1153
|
+
print("OCQA_STATE:login_preamble_continue_tapped")
|
|
1154
|
+
Thread.sleep(forTimeInterval: 1.5)
|
|
1155
|
+
// Now look for password field on the next screen
|
|
1156
|
+
let passFieldAfter = app.secureTextFields.firstMatch
|
|
1157
|
+
if passFieldAfter.exists {
|
|
1158
|
+
passFieldAfter.tap()
|
|
1159
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1160
|
+
passFieldAfter.typeText(preamblePassword)
|
|
1161
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1162
|
+
for label in loginLabels + continueLabels {
|
|
1163
|
+
let btn = app.buttons[label]
|
|
1164
|
+
if btn.exists && btn.isHittable {
|
|
1165
|
+
btn.tap()
|
|
1166
|
+
print("OCQA_STATE:login_preamble_two_step_submitted")
|
|
1167
|
+
Thread.sleep(forTimeInterval: 2.0)
|
|
1168
|
+
waitForUIStability(timeout: 4.0)
|
|
1169
|
+
break
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
} else {
|
|
1174
|
+
print("OCQA_STATE:login_preamble_no_submit_button")
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
// OTP / verification code step — look for numeric code fields after login submission
|
|
1179
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1180
|
+
let otpLabels = ["code", "otp", "verification", "passcode", "one-time", "pin"]
|
|
1181
|
+
let otpCandidate = app.textFields.allElementsBoundByIndex.first { f in
|
|
1182
|
+
let hint = (f.identifier + " " + (f.placeholderValue ?? "") + " " + f.label).lowercased()
|
|
1183
|
+
return otpLabels.contains { hint.contains($0) }
|
|
1184
|
+
}
|
|
1185
|
+
if let otpField = otpCandidate, otpField.exists, otpField.isHittable {
|
|
1186
|
+
let testOTP = resolve("OCQA_TEST_OTP", fallback: "123456")
|
|
1187
|
+
otpField.tap()
|
|
1188
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1189
|
+
otpField.typeText(testOTP)
|
|
1190
|
+
print("OCQA_STATE:otp_entered")
|
|
1191
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1192
|
+
for label in ["Verify", "Continue", "Submit", "Confirm"] {
|
|
1193
|
+
let btn = app.buttons[label]
|
|
1194
|
+
if btn.exists && btn.isHittable { btn.tap(); break }
|
|
1195
|
+
}
|
|
1196
|
+
Thread.sleep(forTimeInterval: 2.0)
|
|
1197
|
+
}
|
|
1198
|
+
} // !skipPreambleLogin
|
|
1199
|
+
}
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
// Early-death guard: an app that dies at launch or during the login preamble must
|
|
1203
|
+
// surface as a CRITICAL crash finding — not as an XCTest "Failed to resolve query"
|
|
1204
|
+
// error with zero markers (which reports 0 findings and, against a baseline, would
|
|
1205
|
+
// sail through a regression gate). Found via corpus bug-seeding: a state-restoring
|
|
1206
|
+
// app that crashes in its restored screen dies here, before the loop's crash checks.
|
|
1207
|
+
if app.state != .runningForeground {
|
|
1208
|
+
_ = app.wait(for: .runningForeground, timeout: 5)
|
|
1209
|
+
}
|
|
1210
|
+
if app.state != .runningForeground {
|
|
1211
|
+
print("OCQA_ISSUE:{\"type\":\"crash\",\"severity\":\"critical\",\"title\":\"App crashed at launch/startup\",\"screen\":\"Launch\",\"step\":0}")
|
|
1212
|
+
didEmitComplete = true
|
|
1213
|
+
print("OCQA_COMPLETE:{\"actions\":0,\"states\":0,\"issues\":1,\"screens\":\"\",\"outcome\":\"crash_at_launch\"}")
|
|
1214
|
+
return
|
|
1215
|
+
}
|
|
1216
|
+
|
|
1217
|
+
// Trigger the interruption monitor on any pending system alerts
|
|
1218
|
+
app.tap()
|
|
1219
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1220
|
+
|
|
1221
|
+
// Record the TRUE initial screen (e.g. a "Get Started" welcome sheet) before
|
|
1222
|
+
// navigateToRootScreen() below auto-dismisses it. Without this, grounding (AI-generate,
|
|
1223
|
+
// coverage) never sees the app's real launch screen or its dismiss control (e.g. "Continue")
|
|
1224
|
+
// — it only sees the post-dismiss root ("Dashboard") — while a fresh replay of a
|
|
1225
|
+
// Flow (no auto-dismiss there; see testReplayFlow) DOES see the undismissed sheet, so a
|
|
1226
|
+
// generated flow's first assertion mismatched. Emitting it here as a normal OCQA_STATE makes
|
|
1227
|
+
// it the grounding's startScreen with its real tap targets, same shape a recorded Flow sees.
|
|
1228
|
+
let initialElements = readUITree(app)
|
|
1229
|
+
let initialTitle = detectTitle(initialElements) ?? "Unknown"
|
|
1230
|
+
let initialInputs = detectInputDescriptors(in: initialElements)
|
|
1231
|
+
let initialInteractable = initialElements.filter { $0.isEnabled && isInteractable($0.type) }
|
|
1232
|
+
let initialRole = classifyScreenRole(title: initialTitle, elements: initialElements, inputs: initialInputs, interactable: initialInteractable)
|
|
1233
|
+
let initialSummary = describeScreen(title: initialTitle, role: initialRole, elements: initialElements, inputs: initialInputs, interactable: initialInteractable)
|
|
1234
|
+
let initialAtext = visionTextInventory(initialElements).map { "\"\(escapeJSON($0))\"" }.joined(separator: ",")
|
|
1235
|
+
let initialInputJson = initialInputs.map { descriptor in
|
|
1236
|
+
"{\"key\":\"\(escapeJSON(descriptor.key))\",\"label\":\"\(escapeJSON(descriptor.label))\",\"secure\":\(descriptor.secure ? "true" : "false"),\"placeholder\":\"\(escapeJSON(descriptor.placeholder))\"}"
|
|
1237
|
+
}.joined(separator: ",")
|
|
1238
|
+
let initialControlsJson = mapControlsJSON(initialElements)
|
|
1239
|
+
print("OCQA_STATE:{\"screen\":\"\(escapeJSON(initialTitle))\",\"hash\":\"\(computeHash(initialElements))\",\"elements\":\(initialElements.count),\"action\":0,\"role\":\"\(escapeJSON(initialRole))\",\"summary\":\"\(escapeJSON(initialSummary))\",\"settled\":\(isScreenSettled() ? "true" : "false"),\"atext\":[\(initialAtext)],\"inputs\":[\(initialInputJson)],\"controls\":[\(initialControlsJson)]}")
|
|
1240
|
+
|
|
1241
|
+
navigateToRootScreen(actionCount: &actionCount)
|
|
1242
|
+
|
|
1243
|
+
// The first state is the true customer launch surface. Directed replay begins after
|
|
1244
|
+
// deterministic root normalization, so persist that separate navigation anchor in the
|
|
1245
|
+
// shared UI Map instead of pretending onboarding was skipped by a graph edge.
|
|
1246
|
+
let navigationRootElements = readUITree(app)
|
|
1247
|
+
let navigationRootTitle = detectTitle(navigationRootElements) ?? "Unknown"
|
|
1248
|
+
let navigationRootInputs = detectInputDescriptors(in: navigationRootElements)
|
|
1249
|
+
let navigationRootInteractable = navigationRootElements.filter { $0.isEnabled && isInteractable($0.type) }
|
|
1250
|
+
let navigationRootRole = classifyScreenRole(title: navigationRootTitle, elements: navigationRootElements, inputs: navigationRootInputs, interactable: navigationRootInteractable)
|
|
1251
|
+
let navigationRootControls = mapControlsJSON(navigationRootElements)
|
|
1252
|
+
print("OCQA_NAVIGATION_ROOT:{\"screen\":\"\(escapeJSON(navigationRootTitle))\",\"role\":\"\(escapeJSON(navigationRootRole))\",\"controls\":[\(navigationRootControls)]}")
|
|
1253
|
+
|
|
1254
|
+
// ---- Directed (targeted) exploration ----
|
|
1255
|
+
// Beeline from the root to the requested screen as fast as possible by following the route
|
|
1256
|
+
// (control labels learned from prior runs), then fall into the normal loop to explore from
|
|
1257
|
+
// there. Tab rotation is disabled below in directed mode so we stay in the target's area.
|
|
1258
|
+
if !targetScreen.isEmpty {
|
|
1259
|
+
beelineToTarget(target: targetScreen, route: route, actionCount: &actionCount, maxActions: maxActions)
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
while actionCount < maxActions {
|
|
1263
|
+
// Subtract time spent paused for interactive input so human typing never eats the budget.
|
|
1264
|
+
if Date().timeIntervalSince(startTime) - totalWaitSeconds > timeoutSeconds {
|
|
1265
|
+
print("OCQA_STATE:timeout_reached")
|
|
1266
|
+
break
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
// ---- Tab rotation: move to a new tab only once the current tab has gone STALE ----
|
|
1270
|
+
// (≥ tabRotationInterval actions on this tab AND a short drought of new screens). This
|
|
1271
|
+
// lets the explorer fully exhaust a tab's screens — including nested NavigationLinks —
|
|
1272
|
+
// before wandering off, which improves real coverage and makes deep screens reliably
|
|
1273
|
+
// reachable. The previous fixed-interval rotation (plus an early warmup sweep) abandoned
|
|
1274
|
+
// rich screens before their links were visited.
|
|
1275
|
+
let tabRotationInterval = 6
|
|
1276
|
+
if targetScreen.isEmpty // directed mode stays in the target's area — no tab rotation
|
|
1277
|
+
&& actionCount > 0
|
|
1278
|
+
&& (actionCount - lastTabSwitchStep) >= tabRotationInterval
|
|
1279
|
+
&& actionsSinceNewState >= 3
|
|
1280
|
+
&& nextTabRotation < 5 {
|
|
1281
|
+
let rotationElements = readUITree(app).filter { $0.isEnabled && $0.isHittable && isInteractable($0.type) }
|
|
1282
|
+
guard hasVisibleGlobalNavigation(rotationElements, screenBounds: screenBounds) else {
|
|
1283
|
+
lastTabSwitchStep = actionCount
|
|
1284
|
+
continue
|
|
1285
|
+
}
|
|
1286
|
+
// Skip position 0 (home tab) since we usually start there — try other tabs first
|
|
1287
|
+
let rotationOrder = [1, 2, 3, 4, 0]
|
|
1288
|
+
if nextTabRotation < rotationOrder.count {
|
|
1289
|
+
let tabIdx = rotationOrder[nextTabRotation]
|
|
1290
|
+
// First go back to root of current tab
|
|
1291
|
+
for _ in 0..<5 {
|
|
1292
|
+
if tryGoBack() {
|
|
1293
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1294
|
+
} else {
|
|
1295
|
+
break
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
tapTab(atRotationIndex: tabIdx)
|
|
1299
|
+
nextTabRotation += 1
|
|
1300
|
+
lastTabSwitchStep = actionCount
|
|
1301
|
+
actionCount += 1
|
|
1302
|
+
print("OCQA_ACTION:{\"type\":\"tab_rotation\",\"tab_index\":\(tabIdx),\"step\":\(actionCount),\"narrative\":\"\(escapeJSON(recoveryNarrative("tab_rotation", screen: "")))\"}")
|
|
1303
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1304
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1305
|
+
continue
|
|
1306
|
+
}
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
var elements = readUITree(app)
|
|
1310
|
+
if elements.isEmpty {
|
|
1311
|
+
// App may have gone to background, crashed, or be unresponsive.
|
|
1312
|
+
print("OCQA_STATE:empty_tree_reactivating step=\(actionCount)")
|
|
1313
|
+
app.activate()
|
|
1314
|
+
Thread.sleep(forTimeInterval: 2.0)
|
|
1315
|
+
elements = readUITree(app)
|
|
1316
|
+
if elements.isEmpty {
|
|
1317
|
+
print("OCQA_STATE:empty_tree_after_retry step=\(actionCount)")
|
|
1318
|
+
// A dead app (didn't come back on reactivate) is a CRASH — often from an ASYNC
|
|
1319
|
+
// failure a few hundred ms after the action (e.g. a background captcha fetch),
|
|
1320
|
+
// so it surfaces here at the next tree read rather than at the early crash check.
|
|
1321
|
+
// Report it against the action that led here instead of breaking silently. (Real:
|
|
1322
|
+
// Wikipedia's WMFCaptchaViewController.refreshImage assertionFailure crashes the
|
|
1323
|
+
// app during login — a genuine production crash AutoTap must surface.)
|
|
1324
|
+
if app.state != .runningForeground {
|
|
1325
|
+
let where_ = pendingTransitionFrom?.title ?? "the previous screen"
|
|
1326
|
+
let crashKey = "crash-async:\(where_)"
|
|
1327
|
+
if !reportedIssueKeys.contains(crashKey) {
|
|
1328
|
+
reportedIssueKeys.insert(crashKey)
|
|
1329
|
+
issues.append((type: "crash", severity: "critical", title: "App crashed during \(where_)",
|
|
1330
|
+
desc: "The app terminated and did not recover after the action on '\(where_)' — likely an unhandled error or assertion failure (possibly from an async task started by that action)."))
|
|
1331
|
+
print("OCQA_ISSUE:{\"type\":\"crash\",\"severity\":\"critical\",\"title\":\"\(escapeJSON("App crashed during \(where_)"))\",\"screen\":\"\(escapeJSON(where_))\",\"step\":\(actionCount)}")
|
|
1332
|
+
}
|
|
1333
|
+
}
|
|
1334
|
+
break
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
let stateHash = computeHash(elements)
|
|
1338
|
+
let screenTitle = detectTitle(elements)
|
|
1339
|
+
let titleStr = screenTitle ?? "Unknown"
|
|
1340
|
+
|
|
1341
|
+
// Resolve pending transition from previous action — now emit with real destination
|
|
1342
|
+
if let pending = pendingTransitionFrom {
|
|
1343
|
+
knownTransitions["\(pending.title)|\(pending.actionKey)"] = titleStr
|
|
1344
|
+
pendingTransitionFrom = nil
|
|
1345
|
+
if pending.hash != stateHash {
|
|
1346
|
+
// Real navigation: record the destination's affordance fingerprint under the hub
|
|
1347
|
+
// it was reached from, so a hub that reaches >= 2 same-fingerprint screens can be
|
|
1348
|
+
// detected as a repeating template list. Only destinations that have real content
|
|
1349
|
+
// affordances participate — content-less screens (just text / a spinner) are never
|
|
1350
|
+
// treated as a template, so they (and any finding in their text) are never skipped.
|
|
1351
|
+
let (fp, contentCount) = affordanceFingerprint(elements)
|
|
1352
|
+
if contentCount >= 1 {
|
|
1353
|
+
destFingerprint["\(pending.title)|\(pending.actionKey)"] = fp
|
|
1354
|
+
hubKeysByFingerprint[pending.title, default: [:]][fp, default: []].insert(pending.actionKey)
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
let escapedFrom = escapeJSON(pending.title)
|
|
1358
|
+
let escapedTo = escapeJSON(titleStr)
|
|
1359
|
+
let escapedAct = escapeJSON(pending.actionKey)
|
|
1360
|
+
print("OCQA_TRANSITION_RESOLVED:{\"from\":\"\(escapedFrom)\",\"fromHash\":\"\(pending.hash)\",\"to\":\"\(escapedTo)\",\"toHash\":\"\(stateHash)\",\"action\":\"\(escapedAct)\"}")
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
screenVisitCount[titleStr, default: 0] += 1
|
|
1364
|
+
knownScreenTitles.insert(titleStr)
|
|
1365
|
+
|
|
1366
|
+
// If the last action returned us to the same state, cooldown that action key.
|
|
1367
|
+
if let lastActionKey, let lastFrom = lastActionFromStateHash, lastFrom == stateHash {
|
|
1368
|
+
let currentCooldown = actionCooldownUntilStep[lastActionKey] ?? 0
|
|
1369
|
+
actionCooldownUntilStep[lastActionKey] = max(currentCooldown, actionCount + 18)
|
|
1370
|
+
}
|
|
1371
|
+
|
|
1372
|
+
if let title = screenTitle {
|
|
1373
|
+
screenTitles[stateHash] = title
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1376
|
+
// Track which elements appear on which screens (for persistent-nav detection)
|
|
1377
|
+
if !visitedStates.contains(stateHash) {
|
|
1378
|
+
totalDistinctStates += 1
|
|
1379
|
+
actionsSinceNewState = 0
|
|
1380
|
+
for el in elements where el.isEnabled && isInteractable(el.type) {
|
|
1381
|
+
let key = actionKey(for: el)
|
|
1382
|
+
elementScreenPresence[key, default: []].insert(stateHash)
|
|
1383
|
+
}
|
|
1384
|
+
} else {
|
|
1385
|
+
actionsSinceNewState += 1
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
recentStateHashes.append(stateHash)
|
|
1389
|
+
if recentStateHashes.count > 12 {
|
|
1390
|
+
recentStateHashes.removeFirst(recentStateHashes.count - 12)
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
if previousStateHash == stateHash {
|
|
1394
|
+
repeatedStateCount += 1
|
|
1395
|
+
} else {
|
|
1396
|
+
repeatedStateCount = 0
|
|
1397
|
+
}
|
|
1398
|
+
if let previousHash = previousStateHash,
|
|
1399
|
+
let previousTitle = screenTitles[previousHash],
|
|
1400
|
+
previousTitle == titleStr {
|
|
1401
|
+
sameScreenStreak += 1
|
|
1402
|
+
} else {
|
|
1403
|
+
sameScreenStreak = 0
|
|
1404
|
+
}
|
|
1405
|
+
previousStateHash = stateHash
|
|
1406
|
+
|
|
1407
|
+
visitedStates.insert(stateHash)
|
|
1408
|
+
|
|
1409
|
+
// Bail out only when we're genuinely making no progress — i.e. not discovering new
|
|
1410
|
+
// states. A stable, correct screen title alone is NOT stuck (a rich screen can have
|
|
1411
|
+
// many controls worth tapping), so the title-streak path also requires a new-state drought.
|
|
1412
|
+
if actionsSinceNewState >= 14 || (sameScreenStreak >= 10 && actionsSinceNewState >= 5) {
|
|
1413
|
+
let escaped = escapeJSON(titleStr)
|
|
1414
|
+
print("OCQA_STATE:stuck_same_screen screen=\(escaped) streak=\(sameScreenStreak) actions_without_new_state=\(actionsSinceNewState) step=\(actionCount)")
|
|
1415
|
+
// Deterministic escape first: a modal sheet (booking calendar, picker) traps the
|
|
1416
|
+
// explorer with value-selection controls that never yield a new state — a stuck
|
|
1417
|
+
// break here ends the WHOLE RUN (observed: died at 17/100 actions inside a real
|
|
1418
|
+
// app's authenticated area). Try dismissing the screen once before bailing.
|
|
1419
|
+
if !stuckDismissTried.contains(titleStr) {
|
|
1420
|
+
stuckDismissTried.insert(titleStr)
|
|
1421
|
+
if tryGoBack() {
|
|
1422
|
+
actionCount += 1
|
|
1423
|
+
print("OCQA_ACTION:{\"type\":\"back\",\"reason\":\"stuck_escape\",\"step\":\(actionCount),\"screen\":\"\(escaped)\",\"narrative\":\"\(escapeJSON("Dismissing the \(titleStr) screen — nothing here leads anywhere new."))\"}")
|
|
1424
|
+
actionsSinceNewState = 0
|
|
1425
|
+
sameScreenStreak = 0
|
|
1426
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1427
|
+
continue
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
// Last resort before bailing: ask vision for one directed move (if enabled + budget).
|
|
1431
|
+
// A win resets the drought counter and keeps exploration alive on a screen the
|
|
1432
|
+
// structural heuristics have exhausted.
|
|
1433
|
+
if visionEscalationEnabled, visionEscalationsUsed < visionEscalationBudget,
|
|
1434
|
+
!visionResponsePath.isEmpty,
|
|
1435
|
+
visionEscalate(app: app, screenTitle: titleStr, reason: "stuck_same_screen",
|
|
1436
|
+
actionCount: &actionCount, usedCount: &visionEscalationsUsed,
|
|
1437
|
+
responsePath: visionResponsePath, imageDir: visionImageDir,
|
|
1438
|
+
waitTimeout: visionWaitTimeout) {
|
|
1439
|
+
actionsSinceNewState = 0
|
|
1440
|
+
sameScreenStreak = 0
|
|
1441
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1442
|
+
continue
|
|
1443
|
+
}
|
|
1444
|
+
// Deterministic trap escape: dismiss failed (or already tried) and we're still
|
|
1445
|
+
// pinned to this screen — relaunch to the root and keep exploring instead of
|
|
1446
|
+
// ending the whole run. The novelty bias then steers toward unexplored screens.
|
|
1447
|
+
if relaunchEscapesUsed < 2, actionCount < maxActions {
|
|
1448
|
+
relaunchEscapesUsed += 1
|
|
1449
|
+
actionCount += 1
|
|
1450
|
+
print("OCQA_ACTION:{\"type\":\"relaunch\",\"reason\":\"trap_escape\",\"step\":\(actionCount),\"screen\":\"\(escaped)\",\"narrative\":\"\(escapeJSON("The \(titleStr) screen has no working way back — relaunching the app to continue exploring elsewhere."))\"}")
|
|
1451
|
+
app.terminate()
|
|
1452
|
+
Thread.sleep(forTimeInterval: 1.0)
|
|
1453
|
+
app.launch()
|
|
1454
|
+
_ = app.wait(for: .runningForeground, timeout: 10)
|
|
1455
|
+
Thread.sleep(forTimeInterval: 1.0)
|
|
1456
|
+
actionsSinceNewState = 0
|
|
1457
|
+
sameScreenStreak = 0
|
|
1458
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1459
|
+
continue
|
|
1460
|
+
}
|
|
1461
|
+
break
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
let detectedInputs = detectInputDescriptors(in: elements)
|
|
1465
|
+
let inputJsonArray = detectedInputs.map { descriptor in
|
|
1466
|
+
"{\"key\":\"\(escapeJSON(descriptor.key))\",\"label\":\"\(escapeJSON(descriptor.label))\",\"secure\":\(descriptor.secure ? "true" : "false"),\"placeholder\":\"\(escapeJSON(descriptor.placeholder))\"}"
|
|
1467
|
+
}.joined(separator: ",")
|
|
1468
|
+
let interactableForSummary = elements.filter { $0.isEnabled && isInteractable($0.type) }
|
|
1469
|
+
let screenRole = classifyScreenRole(title: titleStr, elements: elements, inputs: detectedInputs, interactable: interactableForSummary)
|
|
1470
|
+
let screenSummary = describeScreen(title: titleStr, role: screenRole, elements: elements, inputs: detectedInputs, interactable: interactableForSummary)
|
|
1471
|
+
|
|
1472
|
+
// Emit screen state. `settled` marks a resting screenshot (no keyboard/menu/sheet up) so
|
|
1473
|
+
// the post-run vision pass can prefer un-ambiguous captures (see isScreenSettled). `atext`
|
|
1474
|
+
// is the fuller a11y text inventory that grounds the vision reviewer (see visionTextInventory).
|
|
1475
|
+
let escapedTitle = escapeJSON(titleStr)
|
|
1476
|
+
let settled = isScreenSettled()
|
|
1477
|
+
let atextJson = visionTextInventory(elements).map { "\"\(escapeJSON($0))\"" }.joined(separator: ",")
|
|
1478
|
+
let controlsJson = mapControlsJSON(elements)
|
|
1479
|
+
print("OCQA_STATE:{\"screen\":\"\(escapedTitle)\",\"hash\":\"\(stateHash)\",\"elements\":\(elements.count),\"action\":\(actionCount),\"role\":\"\(escapeJSON(screenRole))\",\"summary\":\"\(escapeJSON(screenSummary))\",\"settled\":\(settled ? "true" : "false"),\"atext\":[\(atextJson)],\"inputs\":[\(inputJsonArray)],\"controls\":[\(controlsJson)]}")
|
|
1480
|
+
|
|
1481
|
+
// ---- Persistence probe: on a fresh RE-ARRIVAL at a screen, fields we previously
|
|
1482
|
+
// typed into (and verified visible in the a11y value) should still hold their value.
|
|
1483
|
+
// An empty field here means the entered state was silently lost on navigation —
|
|
1484
|
+
// the "value does not persist" bug class. Only fires on arrival from a DIFFERENT
|
|
1485
|
+
// screen (same-screen re-reads can't have lost state to navigation).
|
|
1486
|
+
if lastProbeTitle != titleStr, lastProbeTitle != "" {
|
|
1487
|
+
for (memKey, typed) in typedFieldMemory {
|
|
1488
|
+
let parts = memKey.split(separator: "|", maxSplits: 1).map(String.init)
|
|
1489
|
+
guard parts.count == 2, parts[0] == titleStr, !reportedPersistenceKeys.contains(memKey) else { continue }
|
|
1490
|
+
let fieldKey = parts[1]
|
|
1491
|
+
guard let el = elements.first(where: {
|
|
1492
|
+
($0.type.contains("TextField") || $0.type.contains("rawValue: 49") || $0.type.contains("TextView") || $0.type.contains("rawValue: 52"))
|
|
1493
|
+
&& ($0.identifier == fieldKey || $0.label == fieldKey)
|
|
1494
|
+
}) else { continue }
|
|
1495
|
+
let current = el.value.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1496
|
+
let ph = el.xcElement?.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
|
1497
|
+
if current.isEmpty || current == ph {
|
|
1498
|
+
reportedPersistenceKeys.insert(memKey)
|
|
1499
|
+
let t = "Entered value did not persist: '\(fieldKey)' on \(titleStr)"
|
|
1500
|
+
print("OCQA_ISSUE:{\"type\":\"state_persistence\",\"severity\":\"medium\",\"title\":\"\(escapeJSON(t))\",\"screen\":\"\(escapedTitle)\",\"control\":\"\(escapeJSON(fieldKey))\",\"step\":\(actionCount),\"desc\":\"Typed '\(escapeJSON(typed))' into this field earlier in the run; after navigating away and returning, the field is empty — entered state was silently lost.\"}")
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1503
|
+
}
|
|
1504
|
+
lastProbeTitle = titleStr
|
|
1505
|
+
|
|
1506
|
+
// ---- Keyboard occlusion: a keyboard-covered control DROPS OUT of the a11y tree
|
|
1507
|
+
// entirely (measured: with the keyboard up, a bottom-pinned Submit vanished from the
|
|
1508
|
+
// read; geometry/isHittable never see it). So the signal is disappearance: remember
|
|
1509
|
+
// each screen's primary-action buttons from keyboard-DOWN reads; if the keyboard is
|
|
1510
|
+
// up and a remembered primary button is gone from the tree, it's covered — the
|
|
1511
|
+
// "keyboard covers the button" class (missing keyboard avoidance). Screens WITH
|
|
1512
|
+
// avoidance keep the button visible/present while typing (clean-variant proof).
|
|
1513
|
+
let primaryWords = ["submit", "save", "send", "sign in", "log in", "continue", "post", "confirm"]
|
|
1514
|
+
let currentPrimaryButtons = Set(elements.compactMap { el -> String? in
|
|
1515
|
+
guard el.type.contains("Button") || el.type.contains("rawValue: 9") else { return nil }
|
|
1516
|
+
let lbl = el.label.lowercased()
|
|
1517
|
+
return primaryWords.contains(where: { lbl.contains($0) }) ? el.label : nil
|
|
1518
|
+
})
|
|
1519
|
+
let kb = app.keyboards.firstMatch
|
|
1520
|
+
if kb.exists {
|
|
1521
|
+
for remembered in (settledPrimaryButtons[titleStr] ?? []) where !currentPrimaryButtons.contains(remembered) {
|
|
1522
|
+
let occKey = "kbocc:\(titleStr)|\(remembered)"
|
|
1523
|
+
guard !reportedIssueKeys.contains(occKey) else { continue }
|
|
1524
|
+
reportedIssueKeys.insert(occKey)
|
|
1525
|
+
let t = "Keyboard covers the '\(remembered)' button"
|
|
1526
|
+
issues.append((type: "keyboard_occlusion", severity: "medium", title: t,
|
|
1527
|
+
desc: "While typing on '\(titleStr)', the '\(remembered)' button disappears under the keyboard and cannot be tapped — the screen lacks keyboard avoidance."))
|
|
1528
|
+
print("OCQA_ISSUE:{\"type\":\"keyboard_occlusion\",\"severity\":\"medium\",\"title\":\"\(escapeJSON(t))\",\"screen\":\"\(escapedTitle)\",\"control\":\"\(escapeJSON(remembered))\",\"step\":\(actionCount)}")
|
|
1529
|
+
}
|
|
1530
|
+
} else if !currentPrimaryButtons.isEmpty {
|
|
1531
|
+
settledPrimaryButtons[titleStr, default: []].formUnion(currentPrimaryButtons)
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
// ---- Interactive input: pause ONLY on credential/login forms, where a real value
|
|
1535
|
+
// genuinely matters and a default would be wrong. Optional in-app fields (search,
|
|
1536
|
+
// "special instructions", notes, etc.) get a smart default during normal action
|
|
1537
|
+
// selection and exploration keeps moving — autonomous runs must never freeze waiting for
|
|
1538
|
+
// input that an unattended user can't provide. (This is what caused multi-minute stalls
|
|
1539
|
+
// on detail screens like RestaurantDemo's item pages.)
|
|
1540
|
+
let promptKey = normalizeKey(titleStr)
|
|
1541
|
+
let needsRealInput = detectedInputs.contains { $0.secure } || screenRole == "login" || screenRole == "signup"
|
|
1542
|
+
if interactiveInputEnabled, !dontAskAgain, !inputResponsePath.isEmpty,
|
|
1543
|
+
needsRealInput, !promptedScreens.contains(promptKey),
|
|
1544
|
+
detectedInputs.contains(where: { hasNoOverride(key: $0.key, screen: titleStr, in: inputOverrides) }) {
|
|
1545
|
+
promptedScreens.insert(promptKey) // mark before waiting so we never double-ask
|
|
1546
|
+
// Prompt title: on unlabeled login screens, detectTitle can only grab hero copy
|
|
1547
|
+
// ("Personal training.") — prose is a worse prompt header than the screen's role.
|
|
1548
|
+
let titleIsProse = titleStr.hasSuffix(".") || titleStr.hasSuffix("!") || titleStr.split(separator: " ").count > 4
|
|
1549
|
+
let promptTitle = titleIsProse
|
|
1550
|
+
? (screenRole == "signup" ? "Sign Up" : "Sign In")
|
|
1551
|
+
: titleStr
|
|
1552
|
+
awaitInteractiveInput(
|
|
1553
|
+
requestId: UUID().uuidString,
|
|
1554
|
+
screenTitle: titleStr,
|
|
1555
|
+
displayTitle: promptTitle,
|
|
1556
|
+
descriptors: detectedInputs,
|
|
1557
|
+
responsePath: inputResponsePath,
|
|
1558
|
+
waitTimeout: inputWaitTimeout,
|
|
1559
|
+
overrides: &inputOverrides,
|
|
1560
|
+
totalWaitSeconds: &totalWaitSeconds,
|
|
1561
|
+
dontAskAgain: &dontAskAgain,
|
|
1562
|
+
interactiveEnabled: &interactiveInputEnabled
|
|
1563
|
+
)
|
|
1564
|
+
}
|
|
1565
|
+
|
|
1566
|
+
// Screenshot
|
|
1567
|
+
let screenshot = app.screenshot()
|
|
1568
|
+
let attachment = XCTAttachment(screenshot: screenshot)
|
|
1569
|
+
attachment.name = "state_\(actionCount)_\(titleStr.replacingOccurrences(of: " ", with: "_"))"
|
|
1570
|
+
attachment.lifetime = .keepAlways
|
|
1571
|
+
add(attachment)
|
|
1572
|
+
|
|
1573
|
+
// ---- Error / failure surface detection ----
|
|
1574
|
+
// A visible error/failure message is a high-impact signal that something broke.
|
|
1575
|
+
if let errorText = detectErrorSurface(elements) {
|
|
1576
|
+
let short = String(errorText.prefix(60))
|
|
1577
|
+
let errKey = "error:\(titleStr)|\(short.lowercased())"
|
|
1578
|
+
if !reportedIssueKeys.contains(errKey) {
|
|
1579
|
+
reportedIssueKeys.insert(errKey)
|
|
1580
|
+
let issueTitle = "Error message shown: \(short)"
|
|
1581
|
+
issues.append((type: "error_surface", severity: "high", title: issueTitle, desc: "A failure/error message is visible on '\(titleStr)': \(short)"))
|
|
1582
|
+
print("OCQA_ISSUE:{\"type\":\"error_surface\",\"severity\":\"high\",\"title\":\"\(escapeJSON(issueTitle))\",\"screen\":\"\(escapedTitle)\",\"step\":\(actionCount)}")
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
|
|
1586
|
+
// ---- Stuck-loading / hang detection ----
|
|
1587
|
+
// A loading indicator that never resolves is a hang or a silently-failed load. Poll
|
|
1588
|
+
// briefly; DemoApp's 1.2s dashboard spinner resolves and is not flagged.
|
|
1589
|
+
// Only when the spinner IS the screen: a loading indicator alongside substantial
|
|
1590
|
+
// content is an infinite-scroll/pagination loader, not a hang (observed FP on a
|
|
1591
|
+
// content-feed app: post detail + replies-loading spinner flagged app_hang HIGH).
|
|
1592
|
+
let visibleTextCount = elements.filter { isStaticTextType($0.type) && normalizeVisibleText($0.label).count >= 3 }.count
|
|
1593
|
+
if screenVisitCount[titleStr] ?? 0 <= 1, visibleTextCount <= 4,
|
|
1594
|
+
app.activityIndicators.firstMatch.exists || app.progressIndicators.firstMatch.exists {
|
|
1595
|
+
let loadingKey = "loading:\(titleStr)"
|
|
1596
|
+
if !reportedIssueKeys.contains(loadingKey) {
|
|
1597
|
+
var resolved = false
|
|
1598
|
+
let deadline = Date().addingTimeInterval(8.0)
|
|
1599
|
+
while Date() < deadline {
|
|
1600
|
+
Thread.sleep(forTimeInterval: 1.0)
|
|
1601
|
+
if !(app.activityIndicators.firstMatch.exists || app.progressIndicators.firstMatch.exists) {
|
|
1602
|
+
resolved = true
|
|
1603
|
+
break
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
if !resolved {
|
|
1607
|
+
reportedIssueKeys.insert(loadingKey)
|
|
1608
|
+
let issueTitle = "Screen stuck loading: \(titleStr)"
|
|
1609
|
+
issues.append((type: "app_hang", severity: "high", title: issueTitle, desc: "A loading indicator on '\(titleStr)' did not resolve after 8s — likely a hang or a silently failed load."))
|
|
1610
|
+
print("OCQA_ISSUE:{\"type\":\"app_hang\",\"severity\":\"high\",\"title\":\"\(escapeJSON(issueTitle))\",\"screen\":\"\(escapedTitle)\",\"step\":\(actionCount)}")
|
|
1611
|
+
}
|
|
1612
|
+
}
|
|
1613
|
+
}
|
|
1614
|
+
|
|
1615
|
+
// ---- Carousel handling ----
|
|
1616
|
+
// A page indicator means the screen advances by SWIPING, not tapping. The only tappable
|
|
1617
|
+
// control is often a Back button, so the explorer would otherwise leave after page one.
|
|
1618
|
+
// Swipe through the pages (bounded) so every page is observed, then fall through to
|
|
1619
|
+
// normal exploration once content stops changing.
|
|
1620
|
+
if app.pageIndicators.firstMatch.exists {
|
|
1621
|
+
let swipeKey = "carousel:\(titleStr)"
|
|
1622
|
+
let swipesDone = actionCounts[swipeKey] ?? 0
|
|
1623
|
+
if swipesDone < 6 {
|
|
1624
|
+
swipeScreenLeft()
|
|
1625
|
+
actionCounts[swipeKey, default: 0] += 1
|
|
1626
|
+
actionCount += 1
|
|
1627
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1628
|
+
let afterElements = readUITree(app)
|
|
1629
|
+
let changed = !afterElements.isEmpty && computeHash(afterElements) != stateHash
|
|
1630
|
+
let carouselName = titleStr.isEmpty || titleStr == "Unknown" ? "the carousel" : "the \(titleStr) carousel"
|
|
1631
|
+
print("OCQA_ACTION:{\"type\":\"swipe\",\"direction\":\"left\",\"reason\":\"carousel_page\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON("Swiping through \(carouselName) to the next page."))\"}")
|
|
1632
|
+
if !changed { actionCounts[swipeKey] = 999 } // reached the last page
|
|
1633
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1634
|
+
continue
|
|
1635
|
+
}
|
|
1636
|
+
}
|
|
1637
|
+
|
|
1638
|
+
// Exclude on-screen keyboard keys from the candidate pool — we never want to "explore"
|
|
1639
|
+
// by pressing individual keys (that's how we ended up tapping "shift").
|
|
1640
|
+
let kbFrame = keyboardFrame()
|
|
1641
|
+
let interactable = elements.filter {
|
|
1642
|
+
$0.isEnabled && $0.isHittable && isInteractable($0.type) && !isExternalLink($0)
|
|
1643
|
+
&& !(kbFrame.height > 0 && $0.frame.midY >= kbFrame.minY && !isTextField($0.type))
|
|
1644
|
+
}
|
|
1645
|
+
let globalNavElements = interactable.filter { isLikelyGlobalNavigation($0, screenBounds: screenBounds) }
|
|
1646
|
+
let nonGlobalCandidates = interactable.filter { !isLikelyGlobalNavigation($0, screenBounds: screenBounds) }
|
|
1647
|
+
let candidatePool = nonGlobalCandidates.isEmpty ? interactable : nonGlobalCandidates
|
|
1648
|
+
|
|
1649
|
+
// If keyboard occludes a likely submit action (Continue / Sign Up / Next), dismiss it first.
|
|
1650
|
+
if shouldDismissKeyboardForSubmit(elements: elements) {
|
|
1651
|
+
dismissKeyboardIfNeeded()
|
|
1652
|
+
actionCount += 1
|
|
1653
|
+
print("OCQA_ACTION:{\"type\":\"keyboard_dismiss\",\"reason\":\"reveal_submit\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("keyboard_dismiss", screen: titleStr)))\"}")
|
|
1654
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1655
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1656
|
+
continue
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
// ---- Blank-screen detection ----
|
|
1660
|
+
// Distinguish between "no a11y labels / custom UI" vs genuinely empty.
|
|
1661
|
+
if elements.count < 5 && interactable.count == 0 {
|
|
1662
|
+
let blankKey = "blank:\(titleStr)"
|
|
1663
|
+
let blankCount = (actionCounts[blankKey] ?? 0) + 1
|
|
1664
|
+
actionCounts[blankKey] = blankCount
|
|
1665
|
+
if blankCount == 1 {
|
|
1666
|
+
issues.append((type: "blank_screen", severity: "medium", title: "Blank or inaccessible screen: \(titleStr)", desc: "Screen has \(elements.count) elements, none interactable"))
|
|
1667
|
+
print("OCQA_ISSUE:{\"type\":\"blank_screen\",\"severity\":\"medium\",\"title\":\"Blank or inaccessible screen\",\"screen\":\"\(escapedTitle)\",\"element_count\":\(elements.count),\"step\":\(actionCount)}")
|
|
1668
|
+
}
|
|
1669
|
+
// Before giving up on a blank/inaccessible screen, escalate to vision (if enabled) —
|
|
1670
|
+
// the a11y tree is empty but a real user could still see and tap something. A single
|
|
1671
|
+
// vision-directed action can unstick a custom-drawn/canvas UI the structural pass is
|
|
1672
|
+
// blind to. Bounded by budget; on any non-action reply we fall through to conclude.
|
|
1673
|
+
if visionEscalationEnabled, visionEscalationsUsed < visionEscalationBudget,
|
|
1674
|
+
!visionResponsePath.isEmpty, blankCount >= 2 {
|
|
1675
|
+
if visionEscalate(app: app, screenTitle: titleStr, reason: "blank_surface",
|
|
1676
|
+
actionCount: &actionCount, usedCount: &visionEscalationsUsed,
|
|
1677
|
+
responsePath: visionResponsePath, imageDir: visionImageDir,
|
|
1678
|
+
waitTimeout: visionWaitTimeout) {
|
|
1679
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1680
|
+
continue
|
|
1681
|
+
}
|
|
1682
|
+
}
|
|
1683
|
+
// If consistently blank for 3+ consecutive reads on same screen, treat as limited-surface
|
|
1684
|
+
if blankCount >= 3 && sameScreenStreak >= 2 {
|
|
1685
|
+
print("OCQA_ISSUE:{\"type\":\"limited_surface\",\"severity\":\"high\",\"title\":\"Limited interaction surface\",\"screen\":\"\(escapedTitle)\",\"desc\":\"App surface not accessible via standard accessibility APIs\",\"step\":\(actionCount)}")
|
|
1686
|
+
didEmitComplete = true
|
|
1687
|
+
print("OCQA_COMPLETE:{\"actions\":\(actionCount),\"states\":\(visitedStates.count),\"issues\":\(issues.count + 1),\"screens\":\"\",\"outcome\":\"limited_surface\"}")
|
|
1688
|
+
return
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
|
|
1692
|
+
// ---- Navigation-loop detection ----
|
|
1693
|
+
// Check if recentStateHashes has a repeating cycle of length 2 or 3
|
|
1694
|
+
if recentStateHashes.count >= 6 {
|
|
1695
|
+
let recent = recentStateHashes
|
|
1696
|
+
let hasLoop2 = recent.count >= 4 &&
|
|
1697
|
+
recent[recent.count - 1] == recent[recent.count - 3] &&
|
|
1698
|
+
recent[recent.count - 2] == recent[recent.count - 4]
|
|
1699
|
+
let hasLoop3 = recent.count >= 6 &&
|
|
1700
|
+
recent[recent.count - 1] == recent[recent.count - 4] &&
|
|
1701
|
+
recent[recent.count - 2] == recent[recent.count - 5] &&
|
|
1702
|
+
recent[recent.count - 3] == recent[recent.count - 6]
|
|
1703
|
+
if (hasLoop2 || hasLoop3) && !(authSucceeded && detectedInputs.contains { $0.secure }) {
|
|
1704
|
+
let loopKey = "nav_loop:\(titleStr)"
|
|
1705
|
+
if actionCounts[loopKey] == nil {
|
|
1706
|
+
let period = hasLoop2 ? 2 : 3
|
|
1707
|
+
issues.append((type: "navigation_loop", severity: "low", title: "Navigation loop detected (period \(period))", desc: "Exploration is cycling between the same \(period) screens"))
|
|
1708
|
+
print("OCQA_ISSUE:{\"type\":\"navigation_loop\",\"severity\":\"low\",\"title\":\"Navigation loop\",\"screen\":\"\(escapedTitle)\",\"period\":\(period),\"step\":\(actionCount)}")
|
|
1709
|
+
actionCounts[loopKey] = 1
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
|
|
1714
|
+
// ---- Unresponsive-element detection ----
|
|
1715
|
+
// Skip when we're merely re-poking a login screen we've already passed (Sign Out → re-login
|
|
1716
|
+
// churn) — that's an exploration artifact, not a frozen/broken screen.
|
|
1717
|
+
if repeatedStateCount >= 5 && !(authSucceeded && detectedInputs.contains { $0.secure }) {
|
|
1718
|
+
let unrespKey = "unresponsive:\(titleStr)"
|
|
1719
|
+
if actionCounts[unrespKey] == nil {
|
|
1720
|
+
issues.append((type: "unresponsive_element", severity: "medium", title: "Unresponsive UI on \(titleStr)", desc: "Actions are not changing app state — possible frozen or broken screen"))
|
|
1721
|
+
print("OCQA_ISSUE:{\"type\":\"unresponsive_element\",\"severity\":\"medium\",\"title\":\"Unresponsive UI\",\"screen\":\"\(escapedTitle)\",\"repeated_state_count\":\(repeatedStateCount),\"step\":\(actionCount)}")
|
|
1722
|
+
actionCounts[unrespKey] = 1
|
|
1723
|
+
}
|
|
1724
|
+
}
|
|
1725
|
+
|
|
1726
|
+
if interactable.count < 3 {
|
|
1727
|
+
print("OCQA_STATE:low_interactable screen=\(escapedTitle) total=\(elements.count) interactable=\(interactable.count) global=\(globalNavElements.count) nonGlobal=\(nonGlobalCandidates.count)")
|
|
1728
|
+
}
|
|
1729
|
+
|
|
1730
|
+
// ---- Dead end ----
|
|
1731
|
+
if candidatePool.isEmpty {
|
|
1732
|
+
let deadEndKey = "deadEnd:\(titleStr)"
|
|
1733
|
+
let deadEndCount = actionCounts[deadEndKey] ?? 0
|
|
1734
|
+
actionCounts[deadEndKey, default: 0] += 1
|
|
1735
|
+
|
|
1736
|
+
// If we've been stuck on this dead-end screen 3+ times, use blind tab escape
|
|
1737
|
+
if deadEndCount >= 3 {
|
|
1738
|
+
let tabBarY = screenBounds.height > 0 ? screenBounds.height - 30 : 820.0
|
|
1739
|
+
let screenW = screenBounds.width > 0 ? screenBounds.width : 402.0
|
|
1740
|
+
let tabPositions: [CGFloat] = [0.12, 0.31, 0.5, 0.69, 0.88]
|
|
1741
|
+
let tabTryKey = "deadEndTab:\(titleStr)"
|
|
1742
|
+
let tabIndex = actionCounts[tabTryKey] ?? 0
|
|
1743
|
+
if tabIndex < tabPositions.count {
|
|
1744
|
+
let xPos = screenW * tabPositions[tabIndex]
|
|
1745
|
+
let coord = app.coordinate(withNormalizedOffset: .zero)
|
|
1746
|
+
.withOffset(CGVector(dx: xPos, dy: tabBarY))
|
|
1747
|
+
coord.tap()
|
|
1748
|
+
actionCounts[tabTryKey, default: 0] += 1
|
|
1749
|
+
actionCount += 1
|
|
1750
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"target\":\"tab_bar_pos_\(tabIndex)\",\"reason\":\"dead_end_tab_escape\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("dead_end_tab_escape", screen: titleStr)))\"}")
|
|
1751
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1752
|
+
continue
|
|
1753
|
+
}
|
|
1754
|
+
// All tab positions tried — truly stuck
|
|
1755
|
+
print("OCQA_STATE:truly_stuck_dead_end screen=\(escapedTitle) step=\(actionCount)")
|
|
1756
|
+
emitNavigationTrap(titleStr: titleStr, escapedTitle: escapedTitle, step: actionCount, reported: &reportedIssueKeys, issues: &issues)
|
|
1757
|
+
break
|
|
1758
|
+
}
|
|
1759
|
+
|
|
1760
|
+
let issueTitle = "Dead end: \(titleStr)"
|
|
1761
|
+
issues.append((type: "dead_end", severity: "medium", title: issueTitle, desc: "No interactable elements found"))
|
|
1762
|
+
print("OCQA_ISSUE:{\"type\":\"dead_end\",\"severity\":\"medium\",\"title\":\"\(escapedTitle)\",\"screen\":\"\(escapedTitle)\",\"step\":\(actionCount)}")
|
|
1763
|
+
|
|
1764
|
+
// tryGoBack does swipe-down as its last resort (sheet dismiss)
|
|
1765
|
+
let preBackTitle = titleStr
|
|
1766
|
+
let backWorked = tryGoBack()
|
|
1767
|
+
actionCount += 1
|
|
1768
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
1769
|
+
let postElements = readUITree(app)
|
|
1770
|
+
let postTitle = detectTitle(postElements) ?? "Unknown"
|
|
1771
|
+
if (backWorked || postTitle != preBackTitle) && postTitle != preBackTitle {
|
|
1772
|
+
print("OCQA_ACTION:{\"type\":\"back\",\"reason\":\"dead_end_escape\",\"from\":\"\(escapedTitle)\",\"to\":\"\(escapeJSON(postTitle))\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("back_dead_end", screen: titleStr, to: postTitle)))\"}")
|
|
1773
|
+
continue
|
|
1774
|
+
}
|
|
1775
|
+
// Swipe right (back gesture) as another option
|
|
1776
|
+
let swipeStart = app.coordinate(withNormalizedOffset: CGVector(dx: 0.02, dy: 0.5))
|
|
1777
|
+
let swipeEnd = app.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
|
|
1778
|
+
swipeStart.press(forDuration: 0.05, thenDragTo: swipeEnd)
|
|
1779
|
+
actionCount += 1
|
|
1780
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1781
|
+
let postSwipeElements = readUITree(app)
|
|
1782
|
+
let postSwipeTitle = detectTitle(postSwipeElements) ?? "Unknown"
|
|
1783
|
+
if postSwipeTitle != preBackTitle {
|
|
1784
|
+
print("OCQA_ACTION:{\"type\":\"swipe_back\",\"reason\":\"dead_end_escape\",\"to\":\"\(escapeJSON(postSwipeTitle))\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("swipe_back", screen: titleStr, to: postSwipeTitle)))\"}")
|
|
1785
|
+
continue
|
|
1786
|
+
}
|
|
1787
|
+
continue
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
// ---- Guaranteed tab sweep (coverage floor for tab-bar apps) ----
|
|
1791
|
+
// The explorer prioritizes in-screen content, so a deep first tab can consume the
|
|
1792
|
+
// whole action budget before another tab is ever tapped (measured: DemoApp's Settings
|
|
1793
|
+
// tab was never entered across 7 separate 45-action runs — mutation-recall benchmark).
|
|
1794
|
+
// Visit each tab once, early, so every tab ROOT gets registered and issue-scanned;
|
|
1795
|
+
// depth exploration then continues as before. One tab per iteration, so the normal
|
|
1796
|
+
// state capture/issue detection runs between sweep taps. Sheets/modals covering the
|
|
1797
|
+
// tab bar make the buttons non-hittable, which naturally defers the sweep — the
|
|
1798
|
+
// attempts bound keeps a no-tab-bar app from paying the query cost forever.
|
|
1799
|
+
// app.state is a cheap local check; XCUI element queries against a dead/crashed app
|
|
1800
|
+
// can stall for long timeouts per call — never pay that just to look for tabs.
|
|
1801
|
+
// Directed PR exploration has already spent a reviewed map path reaching one changed
|
|
1802
|
+
// surface. Keep the remaining bounded budget there; an early global tab sweep would
|
|
1803
|
+
// immediately abandon the target and make "observed" mean only that it flashed by.
|
|
1804
|
+
if targetScreen.isEmpty && !tabSweepDone && app.state == .runningForeground {
|
|
1805
|
+
let sweepTabs = app.tabBars.buttons.allElementsBoundByIndex
|
|
1806
|
+
.filter { $0.exists && $0.isHittable }
|
|
1807
|
+
.sorted { $0.frame.midX < $1.frame.midX }
|
|
1808
|
+
if sweepTabs.count >= 2 {
|
|
1809
|
+
func sweepKey(_ el: XCUIElement, _ idx: Int) -> String {
|
|
1810
|
+
let label = el.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
1811
|
+
return label.isEmpty ? "tab_idx_\(idx)" : label
|
|
1812
|
+
}
|
|
1813
|
+
for (i, tab) in sweepTabs.enumerated() where tab.isSelected {
|
|
1814
|
+
sweptTabLabels.insert(sweepKey(tab, i))
|
|
1815
|
+
}
|
|
1816
|
+
if let (idx, target) = sweepTabs.enumerated().first(where: { i, el in
|
|
1817
|
+
!el.isSelected && !sweptTabLabels.contains(sweepKey(el, i))
|
|
1818
|
+
}) {
|
|
1819
|
+
let key = sweepKey(target, idx)
|
|
1820
|
+
sweptTabLabels.insert(key)
|
|
1821
|
+
// A tab sweep is a real semantic navigation action. Preserve it in the
|
|
1822
|
+
// same pending-transition channel as ordinary taps so the UI Map can
|
|
1823
|
+
// connect the current screen to the tab destination. In particular, this
|
|
1824
|
+
// may be the run's final allowed action; the terminal observation below
|
|
1825
|
+
// resolves it instead of leaving a disconnected destination node.
|
|
1826
|
+
pendingTransitionFrom = (title: titleStr, actionKey: key, hash: stateHash)
|
|
1827
|
+
target.tap()
|
|
1828
|
+
actionCount += 1
|
|
1829
|
+
lastTabSwitchStep = actionCount
|
|
1830
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"target\":\"\(escapeJSON(key))\",\"reason\":\"tab_sweep\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON("Visiting the “\(key)” tab to map the app's main sections"))\"}")
|
|
1831
|
+
Thread.sleep(forTimeInterval: 0.8)
|
|
1832
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1833
|
+
continue
|
|
1834
|
+
}
|
|
1835
|
+
tabSweepDone = true // every visible tab visited once — stop querying
|
|
1836
|
+
} else {
|
|
1837
|
+
tabSweepAttempts += 1
|
|
1838
|
+
if tabSweepAttempts >= 40 { tabSweepDone = true } // no (usable) tab bar
|
|
1839
|
+
}
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
// ---- Screen-title-aware DFS action selection ----
|
|
1843
|
+
// Only consider actions not yet tried on this screen title
|
|
1844
|
+
let triedHere = screenActionsTried[titleStr] ?? []
|
|
1845
|
+
let freshCandidates = candidatePool.filter { !triedHere.contains(actionKey(for: $0)) }
|
|
1846
|
+
|
|
1847
|
+
// Prioritize actions leading to UNDISCOVERED screens, then others
|
|
1848
|
+
let newScreenCandidates = freshCandidates.filter { el in
|
|
1849
|
+
let key = actionKey(for: el)
|
|
1850
|
+
if let destTitle = knownTransitions["\(titleStr)|\(key)"] {
|
|
1851
|
+
// Known destination — only try if we haven't visited it many times
|
|
1852
|
+
return (screenVisitCount[destTitle] ?? 0) < 12
|
|
1853
|
+
}
|
|
1854
|
+
// Unknown destination — always try first!
|
|
1855
|
+
return true
|
|
1856
|
+
}
|
|
1857
|
+
|
|
1858
|
+
var activeCandidates = newScreenCandidates.isEmpty ? freshCandidates : newScreenCandidates
|
|
1859
|
+
|
|
1860
|
+
// ---- Template-sibling deferral (affordance fingerprint) ----
|
|
1861
|
+
// If this hub reached >= 2 destinations with the SAME affordance fingerprint (the set of
|
|
1862
|
+
// interactable-element labels — *what you can do* on a screen), it's a repeating template
|
|
1863
|
+
// list, e.g. a list of structurally-identical detail screens. Defer its same-template
|
|
1864
|
+
// links — proven-redundant ones and unvisited siblings predicted to match — so the budget
|
|
1865
|
+
// goes to novel screens first. This only REORDERS: ways to leave are never deferred, and
|
|
1866
|
+
// deferred links stay in the pool (visited if nothing novel remains), so per-screen issue
|
|
1867
|
+
// detection still runs on every screen actually visited — no finding is suppressed.
|
|
1868
|
+
// (Affordance, not structural: validated that this collapses identical siblings while
|
|
1869
|
+
// keeping screens whose actions differ — e.g. an error screen's "Retry" — distinct.)
|
|
1870
|
+
//
|
|
1871
|
+
// Scoped to DIRECTED exploration (a beeline-to-a-screen run, where tab-rotation is off and
|
|
1872
|
+
// the goal is to thoroughly cover one area): there, skipping redundant siblings reaches
|
|
1873
|
+
// more of the target area. In broad autonomous mode it's left off — A/B testing showed the
|
|
1874
|
+
// freed budget there just re-treads already-explored tabs rather than reaching novel
|
|
1875
|
+
// screens, so it only lowered the distinct-screen count without benefit.
|
|
1876
|
+
let hubTemplateFps = targetScreen.isEmpty
|
|
1877
|
+
? Set<String>()
|
|
1878
|
+
: Set((hubKeysByFingerprint[titleStr] ?? [:]).filter { $0.value.count >= 2 }.keys)
|
|
1879
|
+
if !hubTemplateFps.isEmpty {
|
|
1880
|
+
let novelCandidates = activeCandidates.filter { el in
|
|
1881
|
+
if isNavBackButton(el) || isLikelyGlobalNavigation(el, screenBounds: screenBounds) { return true }
|
|
1882
|
+
let key = actionKey(for: el)
|
|
1883
|
+
if let fp = destFingerprint["\(titleStr)|\(key)"] {
|
|
1884
|
+
return !hubTemplateFps.contains(fp) // proven same-template destination — defer
|
|
1885
|
+
}
|
|
1886
|
+
return false // unvisited sibling on a confirmed template hub — predict redundant, defer
|
|
1887
|
+
}
|
|
1888
|
+
if !novelCandidates.isEmpty { activeCandidates = novelCandidates }
|
|
1889
|
+
}
|
|
1890
|
+
|
|
1891
|
+
// Allow one text entry per distinct field on the screen so multi-field forms
|
|
1892
|
+
// (e.g. Create Account with 4 fields) get every field filled — not just the first
|
|
1893
|
+
// two. Capped at 8 to avoid runaway re-entry on pathological screens. The
|
|
1894
|
+
// prefer-pending-fields logic below keeps us moving to empty fields each time.
|
|
1895
|
+
let textFieldCap = max(2, min(detectedInputs.count, 8))
|
|
1896
|
+
let textEntriesHere = screenTextEntryCount[titleStr, default: 0]
|
|
1897
|
+
if textEntriesHere >= textFieldCap {
|
|
1898
|
+
activeCandidates = activeCandidates.filter { !isTextField($0.type) }
|
|
1899
|
+
}
|
|
1900
|
+
|
|
1901
|
+
// Prefer text fields that still need input before revisiting already-filled fields.
|
|
1902
|
+
let pendingTextFields = activeCandidates.filter { isTextField($0.type) && elementNeedsInput($0, in: app) }
|
|
1903
|
+
if !pendingTextFields.isEmpty {
|
|
1904
|
+
activeCandidates = pendingTextFields
|
|
1905
|
+
} else if detectedInputs.count >= 2 {
|
|
1906
|
+
// Form likely complete on this screen: prioritize submit/continue controls next.
|
|
1907
|
+
let submitCandidates = activeCandidates.filter { isLikelySubmitControl($0) }
|
|
1908
|
+
if !submitCandidates.isEmpty {
|
|
1909
|
+
activeCandidates = submitCandidates
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
|
|
1913
|
+
if (screenVisitCount[titleStr] ?? 0) >= 2 {
|
|
1914
|
+
let exploratoryCandidates = activeCandidates.filter { isLikelyExploratoryNavigation($0) }
|
|
1915
|
+
if !exploratoryCandidates.isEmpty {
|
|
1916
|
+
activeCandidates = exploratoryCandidates
|
|
1917
|
+
} else {
|
|
1918
|
+
let namedCandidates = activeCandidates.filter { !isAnonymousControl($0) }
|
|
1919
|
+
if !namedCandidates.isEmpty {
|
|
1920
|
+
activeCandidates = namedCandidates
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
|
|
1925
|
+
if (screenVisitCount[titleStr] ?? 0) >= 3 {
|
|
1926
|
+
let dismissCandidates = activeCandidates.filter { isLikelyDismissControl($0) }
|
|
1927
|
+
if !dismissCandidates.isEmpty {
|
|
1928
|
+
activeCandidates = dismissCandidates
|
|
1929
|
+
}
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
// If a form on this screen already failed to submit (e.g. bad login), stop
|
|
1933
|
+
// re-typing into its fields and re-submitting. Prefer anything that navigates
|
|
1934
|
+
// away — links like "Sign Up"/"Forgot password", or other tabs.
|
|
1935
|
+
if (failedSubmits[titleStr] ?? 0) >= 1 {
|
|
1936
|
+
let escapeCandidates = activeCandidates.filter { !isTextField($0.type) && !isLikelySubmitControl($0) }
|
|
1937
|
+
if !escapeCandidates.isEmpty {
|
|
1938
|
+
activeCandidates = escapeCandidates
|
|
1939
|
+
}
|
|
1940
|
+
}
|
|
1941
|
+
|
|
1942
|
+
// Prefer real CONTENT over ways to LEAVE the screen (back / global nav). On a scrollable
|
|
1943
|
+
// screen whose only remaining options are to leave, first scroll to reveal below-the-fold
|
|
1944
|
+
// content — so off-screen controls (and the issues on them) are actually reached, instead
|
|
1945
|
+
// of bailing out the moment the visible content is exhausted.
|
|
1946
|
+
let contentCandidates = activeCandidates.filter {
|
|
1947
|
+
!isNavBackButton($0) && !isLikelyGlobalNavigation($0, screenBounds: screenBounds)
|
|
1948
|
+
}
|
|
1949
|
+
if contentCandidates.isEmpty, !activeCandidates.isEmpty,
|
|
1950
|
+
isScrollableScreen(), !screenScrolledToBottom.contains(titleStr),
|
|
1951
|
+
(screenScrollDepth[titleStr] ?? 0) < 8 {
|
|
1952
|
+
let beforeSig = contentSignature(elements)
|
|
1953
|
+
performScroll(in: app, upward: true) // swipe up = reveal content further down
|
|
1954
|
+
screenScrollDepth[titleStr, default: 0] += 1
|
|
1955
|
+
actionCount += 1
|
|
1956
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1957
|
+
let afterEls = readUITree(app)
|
|
1958
|
+
if afterEls.isEmpty || contentSignature(afterEls) == beforeSig {
|
|
1959
|
+
screenScrolledToBottom.insert(titleStr) // nothing new revealed — reached bottom
|
|
1960
|
+
}
|
|
1961
|
+
print("OCQA_ACTION:{\"type\":\"scroll\",\"direction\":\"down\",\"reason\":\"discover_below_fold\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("scroll_reveal", screen: titleStr)))\"}")
|
|
1962
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1963
|
+
continue
|
|
1964
|
+
}
|
|
1965
|
+
if !contentCandidates.isEmpty {
|
|
1966
|
+
activeCandidates = contentCandidates
|
|
1967
|
+
}
|
|
1968
|
+
|
|
1969
|
+
// No untried actions on this screen — escape
|
|
1970
|
+
if activeCandidates.isEmpty {
|
|
1971
|
+
// Try scrolling to reveal hidden content (once per direction per screen)
|
|
1972
|
+
if !triedHere.contains("scroll:up") {
|
|
1973
|
+
performScroll(in: app, upward: true)
|
|
1974
|
+
screenActionsTried[titleStr, default: []].insert("scroll:up")
|
|
1975
|
+
actionCount += 1
|
|
1976
|
+
print("OCQA_ACTION:{\"type\":\"scroll\",\"direction\":\"up\",\"reason\":\"screen_exhausted\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("scroll_reveal", screen: titleStr)))\"}")
|
|
1977
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1978
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1979
|
+
continue
|
|
1980
|
+
}
|
|
1981
|
+
if !triedHere.contains("scroll:down") {
|
|
1982
|
+
performScroll(in: app, upward: false)
|
|
1983
|
+
screenActionsTried[titleStr, default: []].insert("scroll:down")
|
|
1984
|
+
actionCount += 1
|
|
1985
|
+
print("OCQA_ACTION:{\"type\":\"scroll\",\"direction\":\"down\",\"reason\":\"screen_exhausted\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("scroll_back", screen: titleStr)))\"}")
|
|
1986
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1987
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1988
|
+
continue
|
|
1989
|
+
}
|
|
1990
|
+
// Try tapping unexplored areas before giving up
|
|
1991
|
+
if !triedHere.contains("tap:center") {
|
|
1992
|
+
let center = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5))
|
|
1993
|
+
center.tap()
|
|
1994
|
+
screenActionsTried[titleStr, default: []].insert("tap:center")
|
|
1995
|
+
actionCount += 1
|
|
1996
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"target\":\"center_unexplored\",\"reason\":\"screen_exhausted\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("center_probe", screen: titleStr)))\"}")
|
|
1997
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
1998
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
1999
|
+
continue
|
|
2000
|
+
}
|
|
2001
|
+
// Try swipe-left (carousel/onboarding advance) before giving up
|
|
2002
|
+
if !triedHere.contains("swipe:left") {
|
|
2003
|
+
let midY = screenBounds.height / 2
|
|
2004
|
+
let swipeStart = app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: screenBounds.width * 0.8, dy: midY))
|
|
2005
|
+
let swipeEnd = app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: screenBounds.width * 0.2, dy: midY))
|
|
2006
|
+
swipeStart.press(forDuration: 0.05, thenDragTo: swipeEnd)
|
|
2007
|
+
screenActionsTried[titleStr, default: []].insert("swipe:left")
|
|
2008
|
+
actionCount += 1
|
|
2009
|
+
print("OCQA_ACTION:{\"type\":\"swipe\",\"direction\":\"left\",\"reason\":\"carousel_probe\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("carousel_probe", screen: titleStr)))\"}")
|
|
2010
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
2011
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2012
|
+
continue
|
|
2013
|
+
}
|
|
2014
|
+
// Try edge-swipe from left (drawer reveal) before giving up
|
|
2015
|
+
if !triedHere.contains("edge_swipe:left") {
|
|
2016
|
+
let desc = performEdgeSwipeLeft(in: app)
|
|
2017
|
+
screenActionsTried[titleStr, default: []].insert("edge_swipe:left")
|
|
2018
|
+
actionCount += 1
|
|
2019
|
+
print("OCQA_ACTION:{\"type\":\"\(desc)\",\"reason\":\"drawer_probe\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("drawer_probe", screen: titleStr)))\"}")
|
|
2020
|
+
Thread.sleep(forTimeInterval: 0.6)
|
|
2021
|
+
let postEdgeElements = readUITree(app)
|
|
2022
|
+
let postEdgeTitle = detectTitle(postEdgeElements) ?? titleStr
|
|
2023
|
+
if postEdgeTitle != titleStr {
|
|
2024
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2025
|
+
continue
|
|
2026
|
+
}
|
|
2027
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2028
|
+
continue
|
|
2029
|
+
}
|
|
2030
|
+
// Scrolling done — try to go back and verify the screen actually changed
|
|
2031
|
+
let preBackTitle = titleStr
|
|
2032
|
+
let backResult = tryGoBack()
|
|
2033
|
+
actionCount += 1
|
|
2034
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
2035
|
+
let postBackElements = readUITree(app)
|
|
2036
|
+
let postBackTitle = detectTitle(postBackElements) ?? "Unknown"
|
|
2037
|
+
if postBackTitle != preBackTitle {
|
|
2038
|
+
print("OCQA_ACTION:{\"type\":\"back\",\"reason\":\"screen_exhausted\",\"screen\":\"\(escapedTitle)\",\"to\":\"\(escapeJSON(postBackTitle))\",\"step\":\(actionCount),\"narrative\":\"\(escapeJSON(recoveryNarrative("back_exhausted", screen: titleStr, to: postBackTitle)))\"}")
|
|
2039
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2040
|
+
continue
|
|
2041
|
+
}
|
|
2042
|
+
if backResult {
|
|
2043
|
+
// Back button worked but screen title didn't change (same-titled parent)
|
|
2044
|
+
print("OCQA_ACTION:{\"type\":\"back\",\"reason\":\"screen_exhausted_same_title\",\"screen\":\"\(escapedTitle)\",\"step\":\(actionCount),\"narrative\":\"\(escapeJSON(recoveryNarrative("back_same_title", screen: titleStr)))\"}")
|
|
2045
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2046
|
+
continue
|
|
2047
|
+
}
|
|
2048
|
+
// Back didn't change screens — fall through to global nav
|
|
2049
|
+
print("OCQA_ACTION:{\"type\":\"back\",\"reason\":\"screen_exhausted_failed\",\"screen\":\"\(escapedTitle)\",\"step\":\(actionCount),\"narrative\":\"\(escapeJSON(recoveryNarrative("back_failed", screen: titleStr)))\"}")
|
|
2050
|
+
// Stuck on this screen — use global navigation (tab bar) to reach unexplored areas
|
|
2051
|
+
let globalNav = interactable
|
|
2052
|
+
.filter { isLikelyGlobalNavigation($0, screenBounds: screenBounds) }
|
|
2053
|
+
.filter { el in
|
|
2054
|
+
let key = actionKey(for: el)
|
|
2055
|
+
// Allow retrying global nav if it leads to undiscovered screens
|
|
2056
|
+
if let dest = knownTransitions["\(titleStr)|\(key)"] {
|
|
2057
|
+
// Only skip if we've explored it many times (9+)
|
|
2058
|
+
return (screenVisitCount[dest] ?? 0) < 9
|
|
2059
|
+
}
|
|
2060
|
+
return true
|
|
2061
|
+
}
|
|
2062
|
+
if !globalNav.isEmpty {
|
|
2063
|
+
activeCandidates = globalNav
|
|
2064
|
+
} else {
|
|
2065
|
+
// No visible global nav — try coordinate-tapping the tab bar area
|
|
2066
|
+
// Tab bars are typically at the very bottom of the screen
|
|
2067
|
+
let tabBarY = screenBounds.height > 0 ? screenBounds.height - 30 : 820.0
|
|
2068
|
+
let screenW = screenBounds.width > 0 ? screenBounds.width : 402.0
|
|
2069
|
+
let tabPositions: [CGFloat] = [0.12, 0.31, 0.5, 0.69, 0.88]
|
|
2070
|
+
let tabTryKey = "tabTry:\(titleStr)"
|
|
2071
|
+
let tabIndex = actionCounts[tabTryKey] ?? 0
|
|
2072
|
+
if tabIndex < tabPositions.count {
|
|
2073
|
+
let xPos = screenW * tabPositions[tabIndex]
|
|
2074
|
+
let coord = app.coordinate(withNormalizedOffset: .zero)
|
|
2075
|
+
.withOffset(CGVector(dx: xPos, dy: tabBarY))
|
|
2076
|
+
coord.tap()
|
|
2077
|
+
actionCounts[tabTryKey, default: 0] += 1
|
|
2078
|
+
actionCount += 1
|
|
2079
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"target\":\"tab_bar_pos_\(tabIndex)\",\"reason\":\"blind_tab_escape\",\"x\":\(Int(xPos)),\"y\":\(Int(tabBarY)),\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("blind_tab_escape", screen: titleStr)))\"}")
|
|
2080
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
2081
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2082
|
+
continue
|
|
2083
|
+
}
|
|
2084
|
+
// All tab positions tried — try swipe-to-dismiss (sheet/modal)
|
|
2085
|
+
let swipeStart = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.3))
|
|
2086
|
+
let swipeEnd = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.9))
|
|
2087
|
+
swipeStart.press(forDuration: 0.1, thenDragTo: swipeEnd)
|
|
2088
|
+
actionCount += 1
|
|
2089
|
+
print("OCQA_ACTION:{\"type\":\"swipe_dismiss\",\"reason\":\"escape_stuck\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("swipe_dismiss", screen: titleStr)))\"}")
|
|
2090
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
2091
|
+
// Check if screen changed
|
|
2092
|
+
let postSwipeElements = readUITree(app)
|
|
2093
|
+
let postSwipeTitle = detectTitle(postSwipeElements) ?? "Unknown"
|
|
2094
|
+
if postSwipeTitle != titleStr {
|
|
2095
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2096
|
+
continue
|
|
2097
|
+
}
|
|
2098
|
+
// Truly stuck — break
|
|
2099
|
+
print("OCQA_STATE:truly_stuck screen=\(escapedTitle) step=\(actionCount)")
|
|
2100
|
+
emitNavigationTrap(titleStr: titleStr, escapedTitle: escapedTitle, step: actionCount, reported: &reportedIssueKeys, issues: &issues)
|
|
2101
|
+
break
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
|
|
2105
|
+
// Sort and select best candidate
|
|
2106
|
+
let persistentThreshold = max(2, totalDistinctStates / 2)
|
|
2107
|
+
var sorted = prioritizeElements(activeCandidates, actionCounts: actionCounts,
|
|
2108
|
+
elementScreenPresence: elementScreenPresence,
|
|
2109
|
+
persistentThreshold: persistentThreshold,
|
|
2110
|
+
screenBounds: screenBounds)
|
|
2111
|
+
|
|
2112
|
+
// ---- Form-completion steering ----
|
|
2113
|
+
// A half-filled form is one tap from progress (submit) or oblivion (Close/Cancel
|
|
2114
|
+
// discards everything typed). While the screen has unfilled text fields, fill the
|
|
2115
|
+
// topmost next (forms read top-to-bottom); once all are filled, prefer the primary
|
|
2116
|
+
// submit; and while the form is in progress, push dismiss-style controls to the very
|
|
2117
|
+
// back of the queue. (First real-app run: the explorer filled 3 of 4 signup fields,
|
|
2118
|
+
// then tapped the sheet's Close button.) Bounded: each field is typed once
|
|
2119
|
+
// (actionCounts guard), so this can never trap the explorer on a screen.
|
|
2120
|
+
// Tree-based (not pool-based): the candidate pool empties as elements get acted on
|
|
2121
|
+
// (tried-here dedup etc.), but the form is still there — steering must see the real
|
|
2122
|
+
// screen or it abandons a form whose remaining field the pool has dropped (observed:
|
|
2123
|
+
// signup's Confirm Password missing from the pool while hittable in the tree).
|
|
2124
|
+
// Termination is per-field via typedFieldKeys, so this can never loop.
|
|
2125
|
+
let treeHasFields = elements.contains(where: { isTextField($0.type) })
|
|
2126
|
+
let unfilledFields = elements.filter {
|
|
2127
|
+
isTextField($0.type) && $0.isEnabled && $0.isHittable
|
|
2128
|
+
&& fieldLooksUnfilled($0, screen: titleStr, typedKeys: typedFieldKeys)
|
|
2129
|
+
}
|
|
2130
|
+
if treeHasFields {
|
|
2131
|
+
// Observability for form steering: which fields the pool actually contains vs the
|
|
2132
|
+
// full tree, so field-skipped bugs are diagnosable from any run log.
|
|
2133
|
+
let treeFields = elements.filter { isTextField($0.type) }
|
|
2134
|
+
let poolDesc = activeCandidates.filter { isTextField($0.type) }
|
|
2135
|
+
.map { "\(actionKey(for: $0))@\(Int($0.frame.midY))" }.joined(separator: ",")
|
|
2136
|
+
let treeDesc = treeFields.map { "\(Int($0.frame.midY)):h\($0.isHittable ? 1 : 0)e\($0.isEnabled ? 1 : 0)" }.joined(separator: ",")
|
|
2137
|
+
print("OCQA_STATE:form_steering step=\(actionCount) pool=[\(poolDesc)] tree=[\(treeDesc)] unfilled=\(unfilledFields.count) kb=\(app.keyboards.firstMatch.exists ? 1 : 0)")
|
|
2138
|
+
}
|
|
2139
|
+
if let nextField = unfilledFields.min(by: { $0.frame.minY < $1.frame.minY }) {
|
|
2140
|
+
let nextKey = actionKey(for: nextField)
|
|
2141
|
+
sorted = [nextField] + sorted.filter { actionKey(for: $0) != nextKey }
|
|
2142
|
+
let (dismiss, rest) = sorted.dropFirst().reduce(into: ([SimpleElement](), [SimpleElement]())) {
|
|
2143
|
+
if isDismissControl($1) { $0.0.append($1) } else { $0.1.append($1) }
|
|
2144
|
+
}
|
|
2145
|
+
sorted = [nextField] + rest + dismiss
|
|
2146
|
+
} else if treeHasFields, !app.keyboards.firstMatch.exists,
|
|
2147
|
+
let submit = sorted.first(where: { isLikelySubmitControl($0) && (actionCounts[actionKey(for: $0)] ?? 0) == 0 }) {
|
|
2148
|
+
let submitKey = actionKey(for: submit)
|
|
2149
|
+
sorted = [submit] + sorted.filter { actionKey(for: $0) != submitKey }
|
|
2150
|
+
} else if treeHasFields, app.keyboards.firstMatch.exists,
|
|
2151
|
+
(actionCounts["kbreveal:\(titleStr)"] ?? 0) < max(2, elements.filter { isTextField($0.type) }.count) {
|
|
2152
|
+
// With the keyboard up, field values are UNRELIABLE (a never-typed SecureField can
|
|
2153
|
+
// read as filled) and the submit may be occluded — settle the keyboard before
|
|
2154
|
+
// choosing a submit or leaving. (Observed live: submit tapped before Confirm
|
|
2155
|
+
// Password was filled → "Passwords do not match.") Bounded per screen by field
|
|
2156
|
+
// count so a stubborn keyboard can't loop this forever.
|
|
2157
|
+
actionCounts["kbreveal:\(titleStr)", default: 0] += 1
|
|
2158
|
+
dismissKeyboardIfNeeded()
|
|
2159
|
+
actionCount += 1
|
|
2160
|
+
repeatedStateCount = 0 // settling the keyboard is progress, not a frozen screen
|
|
2161
|
+
print("OCQA_ACTION:{\"type\":\"keyboard_dismiss\",\"reason\":\"reveal_submit_after_fill\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON("Hiding the keyboard to look for the form's submit button."))\"}")
|
|
2162
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
2163
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2164
|
+
continue
|
|
2165
|
+
} else if (screenRole == "login" || screenRole == "signup"),
|
|
2166
|
+
!app.keyboards.firstMatch.exists,
|
|
2167
|
+
screenTextEntryCount[titleStr, default: 0] >= 2,
|
|
2168
|
+
// Interactable controls only — a static TITLE like "Create Account" matches
|
|
2169
|
+
// the submit tokens but is not a control the user can tap.
|
|
2170
|
+
!elements.contains(where: { isInteractable($0.type) && $0.isEnabled && isLikelySubmitControl($0) }),
|
|
2171
|
+
!reportedIssueKeys.contains("nosubmit:\(titleStr)") {
|
|
2172
|
+
// A credentials form we FILLED (≥2 fields) with the keyboard down and still no
|
|
2173
|
+
// submit control anywhere in the tree — the user cannot complete this flow.
|
|
2174
|
+
// Found on a real app: a signup sheet with four fields and only a Close button.
|
|
2175
|
+
// Scoped to login/signup roles: settings-style forms legitimately auto-save.
|
|
2176
|
+
reportedIssueKeys.insert("nosubmit:\(titleStr)")
|
|
2177
|
+
let issueTitle = "Form has no submit control: \(titleStr)"
|
|
2178
|
+
issues.append((type: "form_no_submit", severity: "medium", title: issueTitle,
|
|
2179
|
+
desc: "Filled the form on '\(titleStr)' but found no button to submit it — the flow cannot be completed."))
|
|
2180
|
+
print("OCQA_ISSUE:{\"type\":\"form_no_submit\",\"severity\":\"medium\",\"title\":\"\(escapeJSON(issueTitle))\",\"screen\":\"\(escapedTitle)\",\"step\":\(actionCount)}")
|
|
2181
|
+
}
|
|
2182
|
+
|
|
2183
|
+
// ---- Numeric-grid (calendar) handling ----
|
|
2184
|
+
// Day cells in a booking calendar are value-selection, not navigation — each tap
|
|
2185
|
+
// "succeeds" without a new state, so a month of cells can eat the whole budget
|
|
2186
|
+
// (observed live: 3,4,5,6,7,8 tapped in sequence until the run died). After two
|
|
2187
|
+
// bare-numeric buttons on this screen have been tried, push the rest to the back.
|
|
2188
|
+
let numericTriedHere = (screenActionsTried[titleStr] ?? []).filter {
|
|
2189
|
+
$0.hasPrefix("label:") && Int($0.dropFirst("label:".count)) != nil
|
|
2190
|
+
}.count
|
|
2191
|
+
if numericTriedHere >= 2 {
|
|
2192
|
+
let (numericCells, others) = sorted.reduce(into: ([SimpleElement](), [SimpleElement]())) {
|
|
2193
|
+
if Int($1.label.trimmingCharacters(in: .whitespaces)) != nil { $0.0.append($1) } else { $0.1.append($1) }
|
|
2194
|
+
}
|
|
2195
|
+
if !numericCells.isEmpty { sorted = others + numericCells }
|
|
2196
|
+
}
|
|
2197
|
+
|
|
2198
|
+
guard let target = sorted.first else { break }
|
|
2199
|
+
|
|
2200
|
+
if !isTextField(target.type) {
|
|
2201
|
+
dismissKeyboardIfNeeded()
|
|
2202
|
+
}
|
|
2203
|
+
|
|
2204
|
+
let preContentSig = contentSignature(elements)
|
|
2205
|
+
let actionDesc = performSmartAction(
|
|
2206
|
+
on: target,
|
|
2207
|
+
in: app,
|
|
2208
|
+
screenTitle: titleStr,
|
|
2209
|
+
inputOverrides: inputOverrides,
|
|
2210
|
+
elementTapCount: actionCounts[actionKey(for: target), default: 0],
|
|
2211
|
+
screenElements: elements,
|
|
2212
|
+
screenRole: screenRole
|
|
2213
|
+
)
|
|
2214
|
+
let key = actionKey(for: target)
|
|
2215
|
+
actionCounts[key, default: 0] += 1
|
|
2216
|
+
let stateKey = "\(stateHash)|\(key)"
|
|
2217
|
+
stateActionCounts[stateKey, default: 0] += 1
|
|
2218
|
+
if isTextField(target.type) {
|
|
2219
|
+
typedFieldKeys.insert("\(titleStr)|\(key)")
|
|
2220
|
+
// Typing changes field VALUES, which the (value-blind) state hash can't see — now
|
|
2221
|
+
// that keyboard elements are excluded from the tree, a multi-field fill is a run of
|
|
2222
|
+
// identical hashes. That's progress, not a frozen screen: keep the repeated-state
|
|
2223
|
+
// counter from false-firing "Unresponsive UI" mid-form.
|
|
2224
|
+
repeatedStateCount = 0
|
|
2225
|
+
}
|
|
2226
|
+
lastActionKey = key
|
|
2227
|
+
lastActionFromStateHash = stateHash
|
|
2228
|
+
screenActionsTried[titleStr, default: []].insert(key)
|
|
2229
|
+
if isTextField(target.type) {
|
|
2230
|
+
screenTextEntryCount[titleStr, default: 0] += 1
|
|
2231
|
+
}
|
|
2232
|
+
pendingTransitionFrom = (title: titleStr, actionKey: key, hash: stateHash)
|
|
2233
|
+
actionCount += 1
|
|
2234
|
+
|
|
2235
|
+
let targetName = target.identifier.isEmpty ? target.label : target.identifier
|
|
2236
|
+
let escapedTarget = escapeJSON(targetName)
|
|
2237
|
+
let actionType = isTextField(target.type) ? "type" : "tap"
|
|
2238
|
+
let actionNarrative = narrate(action: actionType, target: target, screenTitle: titleStr, actionDesc: actionDesc)
|
|
2239
|
+
let escapedNarrative = escapeJSON(actionNarrative)
|
|
2240
|
+
print("OCQA_ACTION:{\"type\":\"\(actionType)\",\"target\":\"\(escapedTarget)\",\"elementType\":\"\(target.type)\",\"step\":\(actionCount),\"x\":\(Int(target.frame.midX)),\"y\":\(Int(target.frame.midY)),\"narrative\":\"\(escapedNarrative)\",\"screen\":\"\(escapedTitle)\"}")
|
|
2241
|
+
|
|
2242
|
+
waitForAnimationsToSettle()
|
|
2243
|
+
|
|
2244
|
+
// ---- Early crash check ----
|
|
2245
|
+
// If the action just killed the app, report it NOW — before any tree/submit query below,
|
|
2246
|
+
// which would throw "Application is not running" and record a spurious test failure while
|
|
2247
|
+
// the crash itself goes unreported. app.state is non-throwing even when the app is dead.
|
|
2248
|
+
// (Found on a real app: a login submit terminated the app; the run limped on but emitted
|
|
2249
|
+
// no crash finding.) Try one relaunch to distinguish a hard crash from a transient exit.
|
|
2250
|
+
if app.state != .runningForeground {
|
|
2251
|
+
print("OCQA_STATE:app_left_foreground step=\(actionCount) state=\(app.state.rawValue)")
|
|
2252
|
+
app.activate()
|
|
2253
|
+
Thread.sleep(forTimeInterval: 3.0)
|
|
2254
|
+
if app.state != .runningForeground {
|
|
2255
|
+
let crashKey = "crash:\(titleStr)|\(key)"
|
|
2256
|
+
if !reportedIssueKeys.contains(crashKey) {
|
|
2257
|
+
reportedIssueKeys.insert(crashKey)
|
|
2258
|
+
issues.append((type: "crash", severity: "critical", title: "App crashed after \(actionType) on \(titleStr)",
|
|
2259
|
+
desc: "The app terminated after \(actionType) '\(targetName)' on '\(titleStr)' and did not recover on relaunch."))
|
|
2260
|
+
print("OCQA_ISSUE:{\"type\":\"crash\",\"severity\":\"critical\",\"title\":\"\(escapeJSON("App crashed after \(actionType) on \(titleStr)"))\",\"screen\":\"\(escapedTitle)\",\"control\":\"\(escapedTarget)\",\"step\":\(actionCount)}")
|
|
2261
|
+
}
|
|
2262
|
+
break
|
|
2263
|
+
}
|
|
2264
|
+
print("OCQA_STATE:app_reactivated step=\(actionCount)")
|
|
2265
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2266
|
+
continue
|
|
2267
|
+
}
|
|
2268
|
+
|
|
2269
|
+
// After a submit/login tap, the result is usually a network round-trip. Wait for the
|
|
2270
|
+
// screen to actually change (success navigation or inline error) before continuing,
|
|
2271
|
+
// so we don't evaluate the next action against a still-loading screen.
|
|
2272
|
+
if !isTextField(target.type) && isLikelySubmitControl(target) {
|
|
2273
|
+
print("OCQA_STATE:awaiting_submit_result screen=\(escapedTitle) step=\(actionCount)")
|
|
2274
|
+
waitForSubmitResult(previousHash: stateHash)
|
|
2275
|
+
// Evaluate the submit result. A submit that navigates away OR changes the visible
|
|
2276
|
+
// content (an inline "Saved"/"Thanks" confirmation, a validation error) DID have an
|
|
2277
|
+
// effect. Only a submit that leaves us on the same screen with NO content change is a
|
|
2278
|
+
// genuine failed submit/sign-in.
|
|
2279
|
+
let postSubmit = readUITree(app)
|
|
2280
|
+
let postSubmitTitle = detectTitle(postSubmit) ?? "Unknown"
|
|
2281
|
+
let postSubmitContentChanged = !postSubmit.isEmpty && contentSignature(postSubmit) != preContentSig
|
|
2282
|
+
let onAuthScreen = detectedInputs.contains { $0.secure }
|
|
2283
|
+
if postSubmitTitle != titleStr {
|
|
2284
|
+
// Moved forward — success. Remember a successful sign-in so a later re-poke of the
|
|
2285
|
+
// login form (after Sign Out) isn't mis-reported as an auth failure.
|
|
2286
|
+
if onAuthScreen { authSucceeded = true }
|
|
2287
|
+
} else if !postSubmitContentChanged {
|
|
2288
|
+
// Still on the same screen with no visible change — record it so we stop re-filling
|
|
2289
|
+
// and re-submitting the same form and instead navigate away.
|
|
2290
|
+
failedSubmits[titleStr, default: 0] += 1
|
|
2291
|
+
print("OCQA_STATE:submit_no_change screen=\(escapedTitle) attempts=\(failedSubmits[titleStr, default: 0])")
|
|
2292
|
+
let submitKey = "submitfail:\(titleStr)"
|
|
2293
|
+
// Only a genuine, novel failure counts: a strong submit/sign-in control, an actual
|
|
2294
|
+
// form present, not an already-selected segment, and NOT a login screen we've
|
|
2295
|
+
// already authenticated past (Sign Out → re-login churn, not a product defect).
|
|
2296
|
+
if !reportedIssueKeys.contains(submitKey)
|
|
2297
|
+
&& isStrongSubmitControl(target)
|
|
2298
|
+
&& !detectedInputs.isEmpty
|
|
2299
|
+
&& !target.isSelected
|
|
2300
|
+
&& !(onAuthScreen && authSucceeded) {
|
|
2301
|
+
reportedIssueKeys.insert(submitKey)
|
|
2302
|
+
let issueType = onAuthScreen ? "auth_failed" : "submit_failed"
|
|
2303
|
+
let issueTitle = onAuthScreen ? "Sign-in did not succeed" : "Form submission had no effect"
|
|
2304
|
+
let issueDesc = "Tapping '\(targetName)' on '\(titleStr)' left the user on the same screen with no visible change — likely a failed \(onAuthScreen ? "sign-in" : "validation or broken action")."
|
|
2305
|
+
issues.append((type: issueType, severity: "high", title: issueTitle, desc: issueDesc))
|
|
2306
|
+
print("OCQA_ISSUE:{\"type\":\"\(issueType)\",\"severity\":\"high\",\"title\":\"\(escapeJSON(issueTitle))\",\"screen\":\"\(escapedTitle)\",\"control\":\"\(escapedTarget)\",\"step\":\(actionCount)}")
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
} else if !isTextField(target.type) {
|
|
2310
|
+
// ---- No-op / dead control detection ----
|
|
2311
|
+
// A labeled BUTTON that produces NO visible change (no navigation, no content or
|
|
2312
|
+
// value change) is likely a dead control — exactly the "clicking a button does
|
|
2313
|
+
// nothing" case. We compare a content signature (labels + values) so controls that
|
|
2314
|
+
// only change a value (counters, toggles) are never falsely flagged, exclude
|
|
2315
|
+
// selection/value controls, and require a second confirming read to rule out a
|
|
2316
|
+
// merely-delayed update. NATIVE buttons only (rawValue:9): web buttons/links live
|
|
2317
|
+
// inside a WKWebView, whose dynamic DOM changes aren't reliably reflected in the
|
|
2318
|
+
// accessibility tree, so we can't measure their responsiveness this way.
|
|
2319
|
+
let isButton = target.type.contains("rawValue: 9")
|
|
2320
|
+
let isToggle = target.type.contains("Switch") || target.type.contains("Toggle") || target.type.contains("rawValue: 40")
|
|
2321
|
+
// Require a human-readable VISIBLE LABEL — not just a dotted developer identifier
|
|
2322
|
+
// like "resident.home.curatedForYou" (those are usually containers exposed as buttons,
|
|
2323
|
+
// not real tappable controls a user would expect to act).
|
|
2324
|
+
let humanLabel = target.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
2325
|
+
let named = !humanLabel.isEmpty && !isSymbolLikeLabel(humanLabel)
|
|
2326
|
+
&& !(humanLabel.contains(".") && !humanLabel.contains(" "))
|
|
2327
|
+
let noOpKey = "noop:\(titleStr)|\(key)"
|
|
2328
|
+
if isButton && !isToggle && named
|
|
2329
|
+
&& !isSelectionOrValueControl(target)
|
|
2330
|
+
&& !isInsideSelectionContainer(target)
|
|
2331
|
+
&& !isSystemHandoffControl(humanLabel)
|
|
2332
|
+
&& app.state == .runningForeground && !reportedIssueKeys.contains(noOpKey) {
|
|
2333
|
+
let post1 = readUITree(app)
|
|
2334
|
+
if !post1.isEmpty, contentSignature(post1) == preContentSig {
|
|
2335
|
+
// Confirm it's genuinely inert, not just a delayed update.
|
|
2336
|
+
Thread.sleep(forTimeInterval: 0.6)
|
|
2337
|
+
let post2 = readUITree(app)
|
|
2338
|
+
if app.state == .runningForeground, !post2.isEmpty, contentSignature(post2) == preContentSig {
|
|
2339
|
+
reportedIssueKeys.insert(noOpKey)
|
|
2340
|
+
// A dead NAVIGATION control (Back/Close/Done/Cancel) is worse than a dead
|
|
2341
|
+
// feature button: it strands the user on the screen (and strands this
|
|
2342
|
+
// explorer — see the reachability-loss regression it causes). HIGH, not low.
|
|
2343
|
+
let navLabels: Set<String> = ["back", "close", "done", "cancel", "dismiss", "exit"]
|
|
2344
|
+
let isNavControl = navLabels.contains(humanLabel.lowercased())
|
|
2345
|
+
let sev = isNavControl ? "high" : "low"
|
|
2346
|
+
let issueTitle = isNavControl
|
|
2347
|
+
? "Navigation control does nothing: '\(humanLabel)' — users may be stuck on this screen"
|
|
2348
|
+
: "Control may be unresponsive: '\(humanLabel)'"
|
|
2349
|
+
let issueDesc = "Tapping '\(humanLabel)' on '\(titleStr)' produced no visible change (no navigation, content, or state change)."
|
|
2350
|
+
issues.append((type: "unresponsive_element", severity: sev, title: issueTitle, desc: issueDesc))
|
|
2351
|
+
print("OCQA_ISSUE:{\"type\":\"unresponsive_element\",\"severity\":\"\(sev)\",\"title\":\"\(escapeJSON(issueTitle))\",\"screen\":\"\(escapedTitle)\",\"control\":\"\(escapeJSON(humanLabel))\",\"step\":\(actionCount)}")
|
|
2352
|
+
}
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
|
|
2357
|
+
// ---- App left foreground / crash detection ----
|
|
2358
|
+
// Check both .exists and .state — external links may cause either to fail
|
|
2359
|
+
let appInForeground = app.state == .runningForeground
|
|
2360
|
+
if !appInForeground || !app.exists {
|
|
2361
|
+
print("OCQA_STATE:app_left_foreground step=\(actionCount) state=\(app.state.rawValue)")
|
|
2362
|
+
app.activate()
|
|
2363
|
+
Thread.sleep(forTimeInterval: 3.0)
|
|
2364
|
+
if app.state != .runningForeground {
|
|
2365
|
+
issues.append((type: "crash", severity: "critical",
|
|
2366
|
+
title: "App not recoverable",
|
|
2367
|
+
desc: "App left foreground after: \(actionDesc)"))
|
|
2368
|
+
print("OCQA_ISSUE:{\"type\":\"crash\",\"severity\":\"critical\",\"title\":\"App not recoverable\",\"action\":\"\(escapedTarget)\",\"step\":\(actionCount)}")
|
|
2369
|
+
break
|
|
2370
|
+
}
|
|
2371
|
+
print("OCQA_STATE:app_reactivated step=\(actionCount)")
|
|
2372
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2373
|
+
continue
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
// ---- Track transition ----
|
|
2377
|
+
stateTransitions.append((from: stateHash, to: "deferred", action: actionDesc))
|
|
2378
|
+
print("OCQA_TRANSITION:{\"from\":\"\(escapedTitle)\",\"fromHash\":\"\(stateHash)\",\"to\":\"pending\",\"action\":\"\(escapedTarget)\"}")
|
|
2379
|
+
|
|
2380
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: visitedStates.count)
|
|
2381
|
+
}
|
|
2382
|
+
|
|
2383
|
+
// The action budget is checked at the top of the loop. Previously, when the last allowed
|
|
2384
|
+
// action navigated, the loop ended before reading the resulting state: screenshots could
|
|
2385
|
+
// contain the destination, but OCQA_STATE/TRANSITION evidence (and therefore the shared UI
|
|
2386
|
+
// Map) stopped at the source. Resolve exactly one outstanding terminal transition and emit
|
|
2387
|
+
// the grounded destination without taking another action.
|
|
2388
|
+
if let pending = pendingTransitionFrom, app.state == .runningForeground {
|
|
2389
|
+
Thread.sleep(forTimeInterval: 0.4)
|
|
2390
|
+
let terminalElements = readUITree(app)
|
|
2391
|
+
if !terminalElements.isEmpty {
|
|
2392
|
+
let terminalHash = computeHash(terminalElements)
|
|
2393
|
+
let terminalTitle = detectTitle(terminalElements) ?? "Unknown"
|
|
2394
|
+
knownTransitions["\(pending.title)|\(pending.actionKey)"] = terminalTitle
|
|
2395
|
+
pendingTransitionFrom = nil
|
|
2396
|
+
if pending.hash != terminalHash {
|
|
2397
|
+
print("OCQA_TRANSITION_RESOLVED:{\"from\":\"\(escapeJSON(pending.title))\",\"fromHash\":\"\(pending.hash)\",\"to\":\"\(escapeJSON(terminalTitle))\",\"toHash\":\"\(terminalHash)\",\"action\":\"\(escapeJSON(pending.actionKey))\"}")
|
|
2398
|
+
}
|
|
2399
|
+
knownScreenTitles.insert(terminalTitle)
|
|
2400
|
+
screenTitles[terminalHash] = terminalTitle
|
|
2401
|
+
visitedStates.insert(terminalHash)
|
|
2402
|
+
|
|
2403
|
+
let terminalInputs = detectInputDescriptors(in: terminalElements)
|
|
2404
|
+
let terminalInputJSON = terminalInputs.map { descriptor in
|
|
2405
|
+
"{\"key\":\"\(escapeJSON(descriptor.key))\",\"label\":\"\(escapeJSON(descriptor.label))\",\"secure\":\(descriptor.secure ? "true" : "false"),\"placeholder\":\"\(escapeJSON(descriptor.placeholder))\"}"
|
|
2406
|
+
}.joined(separator: ",")
|
|
2407
|
+
let terminalInteractable = terminalElements.filter { $0.isEnabled && isInteractable($0.type) }
|
|
2408
|
+
let terminalRole = classifyScreenRole(title: terminalTitle, elements: terminalElements, inputs: terminalInputs, interactable: terminalInteractable)
|
|
2409
|
+
let terminalSummary = describeScreen(title: terminalTitle, role: terminalRole, elements: terminalElements, inputs: terminalInputs, interactable: terminalInteractable)
|
|
2410
|
+
let terminalTextJSON = visionTextInventory(terminalElements).map { "\"\(escapeJSON($0))\"" }.joined(separator: ",")
|
|
2411
|
+
let terminalControlsJSON = mapControlsJSON(terminalElements)
|
|
2412
|
+
print("OCQA_STATE:{\"screen\":\"\(escapeJSON(terminalTitle))\",\"hash\":\"\(terminalHash)\",\"elements\":\(terminalElements.count),\"action\":\(actionCount),\"role\":\"\(escapeJSON(terminalRole))\",\"summary\":\"\(escapeJSON(terminalSummary))\",\"settled\":\(isScreenSettled() ? "true" : "false"),\"atext\":[\(terminalTextJSON)],\"inputs\":[\(terminalInputJSON)],\"controls\":[\(terminalControlsJSON)]}")
|
|
2413
|
+
}
|
|
2414
|
+
}
|
|
2415
|
+
|
|
2416
|
+
let uniqueScreens = screenTitles.values
|
|
2417
|
+
let screenList = Array(Set(uniqueScreens)).sorted().joined(separator: ",")
|
|
2418
|
+
didEmitComplete = true
|
|
2419
|
+
print("OCQA_COMPLETE:{\"actions\":\(actionCount),\"states\":\(visitedStates.count),\"issues\":\(issues.count),\"screens\":\"\(screenList)\"}")
|
|
2420
|
+
|
|
2421
|
+
let finalScreenshot = app.screenshot()
|
|
2422
|
+
let finalAttachment = XCTAttachment(screenshot: finalScreenshot)
|
|
2423
|
+
finalAttachment.name = "final_state"
|
|
2424
|
+
finalAttachment.lifetime = .keepAlways
|
|
2425
|
+
add(finalAttachment)
|
|
2426
|
+
}
|
|
2427
|
+
|
|
2428
|
+
// MARK: - Helpers
|
|
2429
|
+
|
|
2430
|
+
private struct SimpleElement {
|
|
2431
|
+
let type: String
|
|
2432
|
+
let identifier: String
|
|
2433
|
+
let label: String
|
|
2434
|
+
let value: String
|
|
2435
|
+
let isSelected: Bool
|
|
2436
|
+
let frame: CGRect
|
|
2437
|
+
let isEnabled: Bool
|
|
2438
|
+
let isHittable: Bool
|
|
2439
|
+
let xcElement: XCUIElement?
|
|
2440
|
+
}
|
|
2441
|
+
|
|
2442
|
+
/// Compact semantic control inventory for the shared Tapp UI Map. Values are
|
|
2443
|
+
/// deliberately excluded: typed credentials and customer content never enter
|
|
2444
|
+
/// the map marker. Host-side map normalization performs additional PII redaction.
|
|
2445
|
+
private func mapControlsJSON(_ elements: [SimpleElement]) -> String {
|
|
2446
|
+
elements.filter { element in
|
|
2447
|
+
isInteractable(element.type)
|
|
2448
|
+
&& (!element.identifier.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
|
2449
|
+
|| !element.label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)
|
|
2450
|
+
}.prefix(60).map { element in
|
|
2451
|
+
let lower = element.type.lowercased()
|
|
2452
|
+
let secure = lower.contains("secure")
|
|
2453
|
+
let kind: String
|
|
2454
|
+
if lower.contains("textfield") || lower.contains("textview") || lower.contains("rawvalue: 49") || lower.contains("rawvalue: 50") || lower.contains("rawvalue: 52") {
|
|
2455
|
+
kind = secure ? "secureField" : "field"
|
|
2456
|
+
} else if lower.contains("tab") { kind = "tab" }
|
|
2457
|
+
else if lower.contains("switch") || lower.contains("toggle") { kind = "toggle" }
|
|
2458
|
+
else if lower.contains("link") { kind = "link" }
|
|
2459
|
+
else { kind = "button" }
|
|
2460
|
+
let label = element.label.isEmpty ? element.identifier : element.label
|
|
2461
|
+
return "{\"kind\":\"\(kind)\",\"type\":\"\(escapeJSON(element.type))\",\"identifier\":\"\(escapeJSON(element.identifier))\",\"accessibilityId\":\"\(escapeJSON(element.identifier))\",\"label\":\"\(escapeJSON(label))\",\"secure\":\(secure ? "true" : "false"),\"enabled\":\(element.isEnabled ? "true" : "false"),\"hittable\":\(element.isHittable ? "true" : "false")}"
|
|
2462
|
+
}.joined(separator: ",")
|
|
2463
|
+
}
|
|
2464
|
+
|
|
2465
|
+
private func readUITree(_ app: XCUIApplication) -> [SimpleElement] {
|
|
2466
|
+
// Check app is accessible before attempting snapshot
|
|
2467
|
+
guard app.state == .runningForeground else { return [] }
|
|
2468
|
+
// Try snapshot-based read first — single IPC call, ~100x faster
|
|
2469
|
+
var elements: [SimpleElement]
|
|
2470
|
+
if let snapshotElements = readViaSnapshot(app), !snapshotElements.isEmpty {
|
|
2471
|
+
elements = snapshotElements
|
|
2472
|
+
} else {
|
|
2473
|
+
// Fallback to element-by-element read (pre-Xcode 15 or snapshot failure)
|
|
2474
|
+
guard app.state == .runningForeground else { return [] }
|
|
2475
|
+
elements = readElementByElement(app)
|
|
2476
|
+
}
|
|
2477
|
+
// Augment with accessible web content from any WKWebView containers
|
|
2478
|
+
elements += extractWebViewElements(app, existingCount: elements.count)
|
|
2479
|
+
|
|
2480
|
+
// The system keyboard is not part of the app's surface. With it in the tree, every form
|
|
2481
|
+
// screen's hash flips between keyboard-up/keyboard-down variants — read as a false
|
|
2482
|
+
// "navigation loop" (period 2) that aborted exploration mid-form — and its keys inflate
|
|
2483
|
+
// the distinct-state count. Drop keyboard-region elements (keeping text fields, which can
|
|
2484
|
+
// legitimately sit near the keyboard's top edge); the keyboard itself remains observable
|
|
2485
|
+
// via app.keyboards for the settled flag and dismissal logic.
|
|
2486
|
+
let kb = app.keyboards.firstMatch
|
|
2487
|
+
if kb.exists {
|
|
2488
|
+
let kbFrame = kb.frame
|
|
2489
|
+
if kbFrame.height > 0 {
|
|
2490
|
+
elements = elements.filter { isTextField($0.type) || $0.frame.midY < kbFrame.minY }
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
return elements
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
/// Extract accessible links/buttons/text from embedded WKWebView containers.
|
|
2497
|
+
/// WKWebViews expose a limited accessibility subtree; we harvest what's available.
|
|
2498
|
+
private func extractWebViewElements(_ app: XCUIApplication, existingCount: Int) -> [SimpleElement] {
|
|
2499
|
+
guard existingCount < 150 else { return [] } // skip if tree already dense
|
|
2500
|
+
let safeScreen = screenBounds.width > 0 ? screenBounds : CGRect(x: 0, y: 0, width: 390, height: 844)
|
|
2501
|
+
|
|
2502
|
+
// SNAPSHOT-based harvest — one IPC per webview. The previous per-element version resolved
|
|
2503
|
+
// every link/button property with a live query (8+ round-trips per link, each with an idle
|
|
2504
|
+
// wait); on a Wikipedia article that was ~40s of "Find the X Link" per few actions and made
|
|
2505
|
+
// WebView-heavy apps time out their budget. Property reads on a snapshot are free.
|
|
2506
|
+
let firstWV = app.webViews.firstMatch
|
|
2507
|
+
guard firstWV.exists else { return [] }
|
|
2508
|
+
|
|
2509
|
+
// Web content loads asynchronously — when we first land on a web screen the accessibility
|
|
2510
|
+
// tree may be empty. Give it a moment to populate before harvesting.
|
|
2511
|
+
var rootSnap = try? firstWV.snapshot()
|
|
2512
|
+
if (rootSnap?.children.isEmpty ?? true) {
|
|
2513
|
+
_ = firstWV.links.firstMatch.waitForExistence(timeout: 2.5)
|
|
2514
|
+
rootSnap = try? firstWV.snapshot()
|
|
2515
|
+
}
|
|
2516
|
+
guard let root = rootSnap else { return [] }
|
|
2517
|
+
|
|
2518
|
+
var webElements: [SimpleElement] = []
|
|
2519
|
+
var links = 0, buttons = 0, fields = 0, secures = 0, texts = 0
|
|
2520
|
+
|
|
2521
|
+
func harvest(_ snap: XCUIElementSnapshot) {
|
|
2522
|
+
guard webElements.count < 60 else { return }
|
|
2523
|
+
let frame = snap.frame
|
|
2524
|
+
let visible = frame.width > 0 && frame.height > 0
|
|
2525
|
+
&& frame.origin.x.isFinite && frame.origin.y.isFinite
|
|
2526
|
+
&& safeScreen.contains(CGPoint(x: frame.midX, y: frame.midY))
|
|
2527
|
+
|
|
2528
|
+
var typeName: String?
|
|
2529
|
+
switch snap.elementType {
|
|
2530
|
+
case .link: links += 1; if links <= 30 { typeName = "Link" }
|
|
2531
|
+
case .button: buttons += 1; if buttons <= 20 { typeName = "Button" }
|
|
2532
|
+
case .textField: fields += 1; if fields <= 10 { typeName = "TextField" }
|
|
2533
|
+
case .secureTextField: secures += 1; if secures <= 5 { typeName = "SecureTextField" }
|
|
2534
|
+
case .staticText: texts += 1
|
|
2535
|
+
default: break
|
|
2536
|
+
}
|
|
2537
|
+
if visible, let typeName {
|
|
2538
|
+
webElements.append(SimpleElement(
|
|
2539
|
+
type: typeName,
|
|
2540
|
+
identifier: snap.identifier,
|
|
2541
|
+
label: snap.label,
|
|
2542
|
+
value: (snap.value as? String) ?? "",
|
|
2543
|
+
isSelected: snap.isSelected,
|
|
2544
|
+
frame: frame,
|
|
2545
|
+
isEnabled: snap.isEnabled,
|
|
2546
|
+
isHittable: visible && snap.isEnabled,
|
|
2547
|
+
xcElement: nil // taps fall back to coordinates; typing re-resolves live by frame
|
|
2548
|
+
))
|
|
2549
|
+
}
|
|
2550
|
+
for child in snap.children {
|
|
2551
|
+
guard webElements.count < 60 else { return }
|
|
2552
|
+
if let c = child as? XCUIElementSnapshot { harvest(c) }
|
|
2553
|
+
}
|
|
2554
|
+
}
|
|
2555
|
+
harvest(root)
|
|
2556
|
+
|
|
2557
|
+
print("OCQA_STATE:webview_probe links=\(links) buttons=\(buttons) staticTexts=\(texts) textFields=\(fields) harvested=\(webElements.count)")
|
|
2558
|
+
return webElements
|
|
2559
|
+
}
|
|
2560
|
+
|
|
2561
|
+
/// Reads the full UI tree via a single snapshot() call — dramatically faster than
|
|
2562
|
+
/// per-element queries since it's one IPC round-trip for the entire hierarchy.
|
|
2563
|
+
private func readViaSnapshot(_ app: XCUIApplication) -> [SimpleElement]? {
|
|
2564
|
+
guard let snapshot = try? app.snapshot() else { return nil }
|
|
2565
|
+
|
|
2566
|
+
var elements: [SimpleElement] = []
|
|
2567
|
+
let limit = 200
|
|
2568
|
+
let safeScreen = screenBounds.width > 0 ? screenBounds : CGRect(x: 0, y: 0, width: 500, height: 1000)
|
|
2569
|
+
|
|
2570
|
+
func walk(_ snap: XCUIElementSnapshot) {
|
|
2571
|
+
guard elements.count < limit else { return }
|
|
2572
|
+
|
|
2573
|
+
let frame = snap.frame
|
|
2574
|
+
if frame.width > 0, frame.height > 0,
|
|
2575
|
+
frame.origin.x.isFinite, frame.origin.y.isFinite,
|
|
2576
|
+
frame.width.isFinite, frame.height.isFinite {
|
|
2577
|
+
|
|
2578
|
+
let hittable = snap.isEnabled && safeScreen.contains(CGPoint(x: frame.midX, y: frame.midY))
|
|
2579
|
+
elements.append(SimpleElement(
|
|
2580
|
+
type: String(describing: snap.elementType),
|
|
2581
|
+
identifier: snap.identifier,
|
|
2582
|
+
label: snap.label ?? "",
|
|
2583
|
+
value: (snap.value as? String) ?? "",
|
|
2584
|
+
isSelected: snap.isSelected,
|
|
2585
|
+
frame: frame,
|
|
2586
|
+
isEnabled: snap.isEnabled,
|
|
2587
|
+
isHittable: hittable,
|
|
2588
|
+
xcElement: nil
|
|
2589
|
+
))
|
|
2590
|
+
}
|
|
2591
|
+
|
|
2592
|
+
for child in snap.children {
|
|
2593
|
+
guard elements.count < limit else { return }
|
|
2594
|
+
if let childSnap = child as? XCUIElementSnapshot {
|
|
2595
|
+
walk(childSnap)
|
|
2596
|
+
}
|
|
2597
|
+
}
|
|
2598
|
+
}
|
|
2599
|
+
|
|
2600
|
+
walk(snapshot)
|
|
2601
|
+
return elements
|
|
2602
|
+
}
|
|
2603
|
+
|
|
2604
|
+
/// Fallback element-by-element read — slower but works on all Xcode versions.
|
|
2605
|
+
private func readElementByElement(_ app: XCUIApplication) -> [SimpleElement] {
|
|
2606
|
+
var elements: [SimpleElement] = []
|
|
2607
|
+
let query = app.descendants(matching: .any)
|
|
2608
|
+
_ = query.firstMatch.waitForExistence(timeout: 5)
|
|
2609
|
+
let count = query.count
|
|
2610
|
+
|
|
2611
|
+
let safeScreen = screenBounds.width > 0 ? screenBounds : CGRect(x: 0, y: 0, width: 500, height: 1000)
|
|
2612
|
+
|
|
2613
|
+
for i in 0..<min(count, 150) {
|
|
2614
|
+
let el = query.element(boundBy: i)
|
|
2615
|
+
guard el.exists else { continue }
|
|
2616
|
+
|
|
2617
|
+
let frame = el.frame
|
|
2618
|
+
guard frame.width > 0, frame.height > 0,
|
|
2619
|
+
frame.origin.x.isFinite, frame.origin.y.isFinite,
|
|
2620
|
+
frame.width.isFinite, frame.height.isFinite else { continue }
|
|
2621
|
+
|
|
2622
|
+
let hittable = el.isEnabled && safeScreen.contains(CGPoint(x: frame.midX, y: frame.midY))
|
|
2623
|
+
|
|
2624
|
+
elements.append(SimpleElement(
|
|
2625
|
+
type: String(describing: el.elementType),
|
|
2626
|
+
identifier: el.identifier,
|
|
2627
|
+
label: el.label,
|
|
2628
|
+
value: (el.value as? String) ?? "",
|
|
2629
|
+
isSelected: el.isSelected,
|
|
2630
|
+
frame: frame,
|
|
2631
|
+
isEnabled: el.isEnabled,
|
|
2632
|
+
isHittable: hittable,
|
|
2633
|
+
xcElement: el
|
|
2634
|
+
))
|
|
2635
|
+
}
|
|
2636
|
+
return elements
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
private func actionKey(for element: SimpleElement) -> String {
|
|
2640
|
+
let id = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
2641
|
+
if !id.isEmpty { return "id:\(id)" }
|
|
2642
|
+
let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
2643
|
+
if !label.isEmpty { return "label:\(label)" }
|
|
2644
|
+
|
|
2645
|
+
let bucketX = Int(element.frame.midX / 80)
|
|
2646
|
+
let bucketY = Int(element.frame.midY / 72)
|
|
2647
|
+
if element.type.contains("Cell") || element.type.contains("rawValue: 75") {
|
|
2648
|
+
return "anonCell:\(bucketY)"
|
|
2649
|
+
}
|
|
2650
|
+
if element.type.contains("Button") || element.type.contains("rawValue: 9") {
|
|
2651
|
+
return "anonButton:\(bucketX)x\(bucketY)"
|
|
2652
|
+
}
|
|
2653
|
+
return "anon:\(bucketX)x\(bucketY):\(element.type)"
|
|
2654
|
+
}
|
|
2655
|
+
|
|
2656
|
+
/// Structural fingerprint of a screen.
|
|
2657
|
+
/// Identity = (element type, stable identifier if present, interactability, coarse grid position).
|
|
2658
|
+
/// Free-text labels are deliberately excluded so cosmetic changes
|
|
2659
|
+
/// (greetings, counts, timestamps, badges) don't fork a screen into duplicates.
|
|
2660
|
+
/// Acceptance: "Good Evening Luis" and "Good Morning Luis" hash identical.
|
|
2661
|
+
private func computeHash(_ elements: [SimpleElement]) -> String {
|
|
2662
|
+
let structure = elements.map { el -> String in
|
|
2663
|
+
let id = el.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
2664
|
+
let interactable = isInteractable(el.type) ? "1" : "0"
|
|
2665
|
+
// 40px grid: stable against minor reflows but still distinguishes layouts.
|
|
2666
|
+
let gridX = Int(el.frame.midX / 40)
|
|
2667
|
+
let gridY = Int(el.frame.midY / 40)
|
|
2668
|
+
return "\(el.type):\(id):\(interactable):\(gridX):\(gridY)"
|
|
2669
|
+
}.sorted().joined(separator: "|")
|
|
2670
|
+
var hash: UInt64 = 5381
|
|
2671
|
+
for byte in structure.utf8 {
|
|
2672
|
+
hash = ((hash << 5) &+ hash) &+ UInt64(byte)
|
|
2673
|
+
}
|
|
2674
|
+
return String(hash, radix: 16)
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2677
|
+
/// Affordance fingerprint: a hash of the SET of labels of *content* interactable elements — i.e.
|
|
2678
|
+
/// *what you can do* on a screen, EXCLUDING global navigation (tab bar) and the back button
|
|
2679
|
+
/// (which appear on nearly every screen). Unlike computeHash (which keys on element
|
|
2680
|
+
/// type/identifier/position and so gives a different value to every screen whose nav-title
|
|
2681
|
+
/// differs), this collapses structurally-identical sibling screens that offer the same actions
|
|
2682
|
+
/// (e.g. a list of detail screens reachable from one hub) while keeping screens whose action set
|
|
2683
|
+
/// differs distinct — an error screen's "Retry"/"Warning" or an empty state's "Add Report" keep
|
|
2684
|
+
/// those screens separate, so a template-skip built on this never skips a screen that hides a
|
|
2685
|
+
/// finding. Returns the content-affordance count too: a screen with ZERO content affordances
|
|
2686
|
+
/// (just text/a spinner — e.g. a changelog or a loading screen) is deliberately NOT eligible to
|
|
2687
|
+
/// form a template, because its identity lives in its text (which we can't fingerprint) and may
|
|
2688
|
+
/// hide a finding. (Empirically validated against real captured UI trees before adopting.)
|
|
2689
|
+
private func affordanceFingerprint(_ elements: [SimpleElement]) -> (fingerprint: String, contentCount: Int) {
|
|
2690
|
+
var labels = Set<String>()
|
|
2691
|
+
for el in elements where isInteractable(el.type) {
|
|
2692
|
+
if isNavBackButton(el) || isLikelyGlobalNavigation(el, screenBounds: screenBounds) { continue }
|
|
2693
|
+
let label = el.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
2694
|
+
let id = el.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
2695
|
+
let key = (label.isEmpty ? id : label).lowercased()
|
|
2696
|
+
if !key.isEmpty { labels.insert(key) }
|
|
2697
|
+
}
|
|
2698
|
+
let joined = labels.sorted().joined(separator: "|")
|
|
2699
|
+
var hash: UInt64 = 5381
|
|
2700
|
+
for byte in joined.utf8 {
|
|
2701
|
+
hash = ((hash << 5) &+ hash) &+ UInt64(byte)
|
|
2702
|
+
}
|
|
2703
|
+
return (String(hash, radix: 16), labels.count)
|
|
2704
|
+
}
|
|
2705
|
+
|
|
2706
|
+
/// Dismiss any launch-time sheet/modal (e.g. a "Get Started" welcome sheet) then walk back to the
|
|
2707
|
+
/// navigation root and reset to the first tab if the app has a tab bar. Used at the start of
|
|
2708
|
+
/// autonomous exploration so wandering/coverage logic always starts from a stable, canonical
|
|
2709
|
+
/// screen rather than a one-time onboarding modal. NOT used by flow replay — a Flow's own first
|
|
2710
|
+
/// step (e.g. `tap: Continue`) is expected to dismiss a launch sheet if the recording/grounding
|
|
2711
|
+
/// captured one; the caller emits an OCQA_STATE for the true initial screen before calling this,
|
|
2712
|
+
/// so exploration/grounding never swallows a launch-time sheet without a trace.
|
|
2713
|
+
private func navigateToRootScreen(actionCount: inout Int) {
|
|
2714
|
+
// First dismiss any sheets. Every dismissal that actually changes the screen is emitted as
|
|
2715
|
+
// a real OCQA_ACTION so the coverage graph gets a genuine launch-screen → root edge — without
|
|
2716
|
+
// it the launch screen is an island: the Flows visual editor can't script past it, and a
|
|
2717
|
+
// flow authored from the (only-connected) post-dismiss root fails on fresh replay, which
|
|
2718
|
+
// launches to the undismissed sheet.
|
|
2719
|
+
for _ in 0..<3 {
|
|
2720
|
+
let preElements = readUITree(app)
|
|
2721
|
+
let preTitle = detectTitle(preElements)
|
|
2722
|
+
// Conservative dismiss controls first (unambiguous close semantics, safe anywhere).
|
|
2723
|
+
// Primary-CTA labels (Continue/Get Started/…) are only tried when a modal is actually
|
|
2724
|
+
// occluding the app's own chrome — on a plain root screen a "Continue" is a normal
|
|
2725
|
+
// navigation (e.g. a wizard) and must be left to exploration proper, not the preamble.
|
|
2726
|
+
var candidates = ["Close", "Cancel", "Done", "Dismiss"]
|
|
2727
|
+
if isLikelySheetPresented() {
|
|
2728
|
+
candidates += ["Continue", "Get Started", "Not Now", "Skip", "OK", "Maybe Later"]
|
|
2729
|
+
}
|
|
2730
|
+
var tappedLabel: String? = nil
|
|
2731
|
+
for label in candidates {
|
|
2732
|
+
let btn = app.buttons[label]
|
|
2733
|
+
if btn.exists && btn.isHittable {
|
|
2734
|
+
btn.tap()
|
|
2735
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
2736
|
+
tappedLabel = label
|
|
2737
|
+
break
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
if tappedLabel == nil {
|
|
2741
|
+
// Try swiping down to dismiss sheet
|
|
2742
|
+
let start = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.3))
|
|
2743
|
+
let end = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.9))
|
|
2744
|
+
start.press(forDuration: 0.1, thenDragTo: end)
|
|
2745
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
2746
|
+
}
|
|
2747
|
+
let postElements = readUITree(app)
|
|
2748
|
+
let postTitle = detectTitle(postElements)
|
|
2749
|
+
if postTitle == preTitle { break } // didn't change — no more sheets
|
|
2750
|
+
// The screen changed: record the dismissal as a normal action so the transition is a
|
|
2751
|
+
// real, replayable edge (tap) or at least a visible one (swipe) in the coverage graph.
|
|
2752
|
+
actionCount += 1
|
|
2753
|
+
if let label = tappedLabel {
|
|
2754
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"target\":\"\(escapeJSON(label))\",\"reason\":\"launch_sheet_dismiss\",\"step\":\(actionCount),\"screen\":\"\(escapeJSON(preTitle ?? "Unknown"))\",\"narrative\":\"\(escapeJSON("Dismissing the launch screen via '\(label)'."))\"}")
|
|
2755
|
+
print("OCQA_TRANSITION_RESOLVED:{\"from\":\"\(escapeJSON(preTitle ?? "Unknown"))\",\"to\":\"\(escapeJSON(postTitle ?? "Unknown"))\",\"action\":\"\(escapeJSON(label))\",\"type\":\"tap\"}")
|
|
2756
|
+
} else {
|
|
2757
|
+
print("OCQA_ACTION:{\"type\":\"swipe\",\"direction\":\"down\",\"reason\":\"launch_sheet_dismiss\",\"step\":\(actionCount),\"screen\":\"\(escapeJSON(preTitle ?? "Unknown"))\",\"narrative\":\"\(escapeJSON("Swiping down to dismiss the launch sheet."))\"}")
|
|
2758
|
+
print("OCQA_TRANSITION_RESOLVED:{\"from\":\"\(escapeJSON(preTitle ?? "Unknown"))\",\"to\":\"\(escapeJSON(postTitle ?? "Unknown"))\",\"action\":\"swipe down\",\"type\":\"swipe\"}")
|
|
2759
|
+
}
|
|
2760
|
+
}
|
|
2761
|
+
// Then go back through navigation stack
|
|
2762
|
+
for _ in 0..<10 {
|
|
2763
|
+
let navBack = app.navigationBars.buttons.firstMatch
|
|
2764
|
+
if navBack.exists && navBack.isHittable {
|
|
2765
|
+
navBack.tap()
|
|
2766
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
2767
|
+
} else {
|
|
2768
|
+
break
|
|
2769
|
+
}
|
|
2770
|
+
}
|
|
2771
|
+
// If a visible tab bar exists, reset to the first tab. Avoid blind taps for non-tab apps.
|
|
2772
|
+
let rootElements = readUITree(app)
|
|
2773
|
+
let rootInteractable = rootElements.filter { $0.isEnabled && $0.isHittable && isInteractable($0.type) }
|
|
2774
|
+
if hasVisibleGlobalNavigation(rootInteractable, screenBounds: screenBounds) {
|
|
2775
|
+
tapTab(atRotationIndex: 0)
|
|
2776
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
2777
|
+
}
|
|
2778
|
+
print("OCQA_STATE:navigated_to_root")
|
|
2779
|
+
}
|
|
2780
|
+
|
|
2781
|
+
/// True when a modal overlay is likely covering the app's own chrome: a tab bar or navigation
|
|
2782
|
+
/// bar that EXISTS but is not hittable is occluded by a presented sheet/fullscreen cover. On a
|
|
2783
|
+
/// plain root screen (wizard step, form) all chrome is hittable, so this stays false and the
|
|
2784
|
+
/// preamble won't tap navigation-y CTAs like "Continue" that aren't dismissals there.
|
|
2785
|
+
private func isLikelySheetPresented() -> Bool {
|
|
2786
|
+
let tabBar = app.tabBars.firstMatch
|
|
2787
|
+
if tabBar.exists && !tabBar.isHittable { return true }
|
|
2788
|
+
for bar in app.navigationBars.allElementsBoundByIndex where bar.exists && !bar.isHittable {
|
|
2789
|
+
return true
|
|
2790
|
+
}
|
|
2791
|
+
return false
|
|
2792
|
+
}
|
|
2793
|
+
|
|
2794
|
+
/// The frontmost navigation bar's title, or nil. When sheets stack multiple bars, prefer the
|
|
2795
|
+
/// last (frontmost) one. This is the canonical XCUITest way to identify a screen — far more
|
|
2796
|
+
/// reliable than scraping static text, which picks up section headers and list rows.
|
|
2797
|
+
private func navigationBarTitle() -> String? {
|
|
2798
|
+
let bars = app.navigationBars.allElementsBoundByIndex.filter { $0.exists }
|
|
2799
|
+
guard !bars.isEmpty else { return nil }
|
|
2800
|
+
// Frontmost first: a nav bar behind a presented sheet is not hittable, so a hittable bar
|
|
2801
|
+
// (the sheet's own) wins. This is what makes a composer/detail sheet read as its real
|
|
2802
|
+
// title (e.g. "New Task") instead of the screen underneath it ("Todo List").
|
|
2803
|
+
let ordered = bars.sorted { ($0.isHittable ? 1 : 0) > ($1.isHittable ? 1 : 0) }
|
|
2804
|
+
for bar in ordered {
|
|
2805
|
+
if let title = titleFromBar(bar) { return title }
|
|
2806
|
+
}
|
|
2807
|
+
return nil
|
|
2808
|
+
}
|
|
2809
|
+
|
|
2810
|
+
private func titleFromBar(_ bar: XCUIElement) -> String? {
|
|
2811
|
+
guard bar.exists else { return nil }
|
|
2812
|
+
let id = bar.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
2813
|
+
if !id.isEmpty, isLikelyTitleText(id) { return id }
|
|
2814
|
+
// SwiftUI sometimes leaves the bar's identifier empty and renders the title as a
|
|
2815
|
+
// static-text child instead.
|
|
2816
|
+
let child = bar.staticTexts.allElementsBoundByIndex
|
|
2817
|
+
.first { $0.exists && !$0.label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }
|
|
2818
|
+
if let child {
|
|
2819
|
+
let lbl = normalizeVisibleText(child.label)
|
|
2820
|
+
if !lbl.isEmpty, isLikelyTitleText(lbl) { return lbl }
|
|
2821
|
+
}
|
|
2822
|
+
return nil
|
|
2823
|
+
}
|
|
2824
|
+
|
|
2825
|
+
/// Content fingerprint including labels + values — unlike computeHash (which deliberately
|
|
2826
|
+
/// ignores free text), this changes when a control updates a value (counter, toggle, inline
|
|
2827
|
+
/// text). Used to tell a genuinely dead button ("nothing happened") from one that only changed
|
|
2828
|
+
/// a value.
|
|
2829
|
+
private func contentSignature(_ elements: [SimpleElement]) -> String {
|
|
2830
|
+
elements.map { "\($0.type)|\($0.identifier)|\($0.label)|\($0.value)" }.sorted().joined(separator: "~")
|
|
2831
|
+
}
|
|
2832
|
+
|
|
2833
|
+
/// Machine identifiers sometimes leak into a11y titles — mangled runtime type names
|
|
2834
|
+
/// ("_TtGC7SwiftUI32NavigationStackHosting", "_UIContextMenuActionsOnlyView") and raw hex
|
|
2835
|
+
/// asset/state ids ("5aadc347c4d963a0") — never a real screen title, and they read as broken
|
|
2836
|
+
/// output on every finding/coverage row they touch.
|
|
2837
|
+
private func isMangledTypeName(_ s: String) -> Bool {
|
|
2838
|
+
if s.hasPrefix("_") { return true }
|
|
2839
|
+
if !s.contains(" ") && (s.contains("SwiftUI") || s.contains("Hosting") || s.contains("ViewController")) { return true }
|
|
2840
|
+
// Long all-hex token (an image/state id) with no spaces.
|
|
2841
|
+
if s.count >= 12, !s.contains(" "), s.allSatisfy({ $0.isHexDigit }) { return true }
|
|
2842
|
+
// Module.Class identifiers ("Wikipedia.SinglePageWebView") — dotted CamelCase with no
|
|
2843
|
+
// spaces. Titles like "Node.js" survive (component after the dot starts lowercase).
|
|
2844
|
+
if !s.contains(" "),
|
|
2845
|
+
s.range(of: "^[A-Za-z][A-Za-z0-9]*\\.[A-Z][A-Za-z0-9.]*$", options: .regularExpression) != nil {
|
|
2846
|
+
return true
|
|
2847
|
+
}
|
|
2848
|
+
return false
|
|
2849
|
+
}
|
|
2850
|
+
|
|
2851
|
+
private func detectTitle(_ elements: [SimpleElement]) -> String? {
|
|
2852
|
+
// Most reliable signal: the SwiftUI navigation-bar title (queried live).
|
|
2853
|
+
if let navTitle = navigationBarTitle(), !isMangledTypeName(navTitle) { return navTitle }
|
|
2854
|
+
|
|
2855
|
+
// NavigationBar = rawValue: 74 (snapshot fallback)
|
|
2856
|
+
if let navTitle = elements.first(where: { $0.type.contains("rawValue: 74") || $0.type.contains("NavigationBar") }) {
|
|
2857
|
+
if !navTitle.identifier.isEmpty, !isMangledTypeName(navTitle.identifier) { return navTitle.identifier }
|
|
2858
|
+
if !navTitle.label.isEmpty, !isMangledTypeName(navTitle.label) { return navTitle.label }
|
|
2859
|
+
}
|
|
2860
|
+
|
|
2861
|
+
let screenHeight = screenBounds.height > 0 ? screenBounds.height : 900
|
|
2862
|
+
let screenWidth = screenBounds.width > 0 ? screenBounds.width : 390
|
|
2863
|
+
let centerX = screenWidth / 2
|
|
2864
|
+
|
|
2865
|
+
let staticTexts = elements
|
|
2866
|
+
.filter { isStaticTextType($0.type) }
|
|
2867
|
+
.map { ($0, normalizeVisibleText($0.label)) }
|
|
2868
|
+
.filter { !$0.1.isEmpty && isLikelyTitleText($0.1) }
|
|
2869
|
+
|
|
2870
|
+
// Prefer centered, prominent text in upper ~65% of screen.
|
|
2871
|
+
let centeredCandidates = staticTexts
|
|
2872
|
+
.filter { item in
|
|
2873
|
+
let el = item.0
|
|
2874
|
+
return el.frame.minY < screenHeight * 0.65
|
|
2875
|
+
&& abs(el.frame.midX - centerX) <= screenWidth * 0.22
|
|
2876
|
+
&& el.frame.width >= screenWidth * 0.22
|
|
2877
|
+
&& el.frame.height <= 64
|
|
2878
|
+
}
|
|
2879
|
+
.sorted { lhs, rhs in
|
|
2880
|
+
let l = lhs.0
|
|
2881
|
+
let r = rhs.0
|
|
2882
|
+
if abs(l.frame.midX - centerX) != abs(r.frame.midX - centerX) {
|
|
2883
|
+
return abs(l.frame.midX - centerX) < abs(r.frame.midX - centerX)
|
|
2884
|
+
}
|
|
2885
|
+
if l.frame.minY != r.frame.minY {
|
|
2886
|
+
return l.frame.minY < r.frame.minY
|
|
2887
|
+
}
|
|
2888
|
+
return (l.frame.width * l.frame.height) > (r.frame.width * r.frame.height)
|
|
2889
|
+
}
|
|
2890
|
+
|
|
2891
|
+
if let best = centeredCandidates.first?.1 {
|
|
2892
|
+
return best
|
|
2893
|
+
}
|
|
2894
|
+
|
|
2895
|
+
// Fallback to upper text region with broader threshold.
|
|
2896
|
+
let topCandidates = staticTexts
|
|
2897
|
+
.filter { $0.0.frame.minY < screenHeight * 0.45 }
|
|
2898
|
+
.sorted { lhs, rhs in
|
|
2899
|
+
let l = lhs.0
|
|
2900
|
+
let r = rhs.0
|
|
2901
|
+
let lArea = l.frame.width * l.frame.height
|
|
2902
|
+
let rArea = r.frame.width * r.frame.height
|
|
2903
|
+
if lArea != rArea { return lArea > rArea }
|
|
2904
|
+
return abs(l.frame.midX - centerX) < abs(r.frame.midX - centerX)
|
|
2905
|
+
}
|
|
2906
|
+
|
|
2907
|
+
if let top = topCandidates.first?.1 {
|
|
2908
|
+
return top
|
|
2909
|
+
}
|
|
2910
|
+
|
|
2911
|
+
// Final fallback for tabbed root screens where no good header text is visible.
|
|
2912
|
+
if let tabTitle = selectedTabTitle(), !tabTitle.isEmpty {
|
|
2913
|
+
return tabTitle
|
|
2914
|
+
}
|
|
2915
|
+
|
|
2916
|
+
// Last resort: the topmost meaningful on-screen text. For custom SwiftUI screens that
|
|
2917
|
+
// expose no nav bar or centered header, this beats labelling everything "Unknown".
|
|
2918
|
+
if let firstText = collectVisibleTexts(elements, limit: 1).first {
|
|
2919
|
+
return firstText
|
|
2920
|
+
}
|
|
2921
|
+
|
|
2922
|
+
return nil
|
|
2923
|
+
}
|
|
2924
|
+
|
|
2925
|
+
private func isInteractable(_ type: String) -> Bool {
|
|
2926
|
+
let interactableRawValues = [9, 39, 40, 42, 43, 49, 50, 53, 54, 56, 75]
|
|
2927
|
+
for rv in interactableRawValues {
|
|
2928
|
+
if type.contains("rawValue: \(rv)") { return true }
|
|
2929
|
+
}
|
|
2930
|
+
let types = ["Button", "TextField", "SecureTextField", "Link", "Cell",
|
|
2931
|
+
"Switch", "Slider", "Tab", "MenuItem", "SegmentedControl",
|
|
2932
|
+
"Picker", "Toggle", "Stepper", "DatePicker"]
|
|
2933
|
+
return types.contains(where: { type.contains($0) })
|
|
2934
|
+
}
|
|
2935
|
+
|
|
2936
|
+
private func isTextField(_ type: String) -> Bool {
|
|
2937
|
+
// 52 = textView — chat composers/notes boxes; typeable exactly like fields.
|
|
2938
|
+
return type.contains("TextField") || type.contains("SecureTextField") || type.contains("TextView") ||
|
|
2939
|
+
type.contains("rawValue: 49") || type.contains("rawValue: 50") || type.contains("rawValue: 52")
|
|
2940
|
+
}
|
|
2941
|
+
|
|
2942
|
+
/// Secure (password) field. The runtime reports the type as "rawValue: 50", NOT the friendly
|
|
2943
|
+
/// "SecureTextField" string — checking only the latter missed password fields, which broke
|
|
2944
|
+
/// password masking, auth-vs-form classification, and login screen-role detection.
|
|
2945
|
+
private func isSecureTextField(_ type: String) -> Bool {
|
|
2946
|
+
return type.contains("SecureTextField") || type.contains("rawValue: 50")
|
|
2947
|
+
}
|
|
2948
|
+
|
|
2949
|
+
/// Checks if an element is likely an external link that will leave the app
|
|
2950
|
+
private func isExternalLink(_ element: SimpleElement) -> Bool {
|
|
2951
|
+
let text = (element.identifier + " " + element.label).lowercased()
|
|
2952
|
+
let externalPrefixes = ["open in ", "download on ", "get it on ", "available on ", "order on ", "buy on "]
|
|
2953
|
+
if externalPrefixes.contains(where: { text.contains($0) }) { return true }
|
|
2954
|
+
// Legal / help / "learn more" links open web content (in-app browser or Safari) that is
|
|
2955
|
+
// out of QA scope AND a hang trap: the external page may never reach idle, so the NEXT
|
|
2956
|
+
// XCUITest query blocks to its 60s timeout and ABORTS the whole run (observed on Wikipedia
|
|
2957
|
+
// onboarding: tapping "Privacy policy" ended the run). Match whole words so app content
|
|
2958
|
+
// like a "Terms" tab isn't wrongly skipped.
|
|
2959
|
+
let externalWordSets: [[String]] = [
|
|
2960
|
+
["privacy", "policy"], ["terms", "of", "service"], ["terms", "of", "use"],
|
|
2961
|
+
["terms", "and", "conditions"], ["cookie", "policy"], ["learn", "more", "about"],
|
|
2962
|
+
]
|
|
2963
|
+
let words = Set(text.split(whereSeparator: { !$0.isLetter }).map(String.init))
|
|
2964
|
+
return externalWordSets.contains { $0.allSatisfy(words.contains) }
|
|
2965
|
+
}
|
|
2966
|
+
|
|
2967
|
+
/// The leading navigation-bar back button (top-left). It shouldn't be a primary exploration
|
|
2968
|
+
/// candidate — otherwise the explorer leaves a screen before exploring/scrolling it. Going back
|
|
2969
|
+
/// is still handled explicitly as a recovery action (tryGoBack) once the screen is exhausted.
|
|
2970
|
+
private func isNavBackButton(_ element: SimpleElement) -> Bool {
|
|
2971
|
+
guard element.type.contains("Button") || element.type.contains("rawValue: 9") else { return false }
|
|
2972
|
+
let h = screenBounds.height > 0 ? screenBounds.height : 900
|
|
2973
|
+
let w = screenBounds.width > 0 ? screenBounds.width : 390
|
|
2974
|
+
return element.frame.midY < h * 0.12 && element.frame.midX < w * 0.22
|
|
2975
|
+
}
|
|
2976
|
+
|
|
2977
|
+
private func isLikelyGlobalNavigation(_ element: SimpleElement, screenBounds: CGRect) -> Bool {
|
|
2978
|
+
let screenHeight = screenBounds.height > 0 ? screenBounds.height : 1000
|
|
2979
|
+
let bottomZoneThreshold = screenHeight * 0.88
|
|
2980
|
+
let inBottomZone = element.frame.midY >= bottomZoneThreshold
|
|
2981
|
+
|
|
2982
|
+
// Tab bar elements (rawValue: 53 = TabBar, 54 = Tab)
|
|
2983
|
+
if element.type.contains("rawValue: 53") || element.type.contains("rawValue: 54") ||
|
|
2984
|
+
element.type.contains("TabBar") || element.type.contains("Tab") {
|
|
2985
|
+
return true
|
|
2986
|
+
}
|
|
2987
|
+
|
|
2988
|
+
// Only treat bottom-zone elements as global nav if they also match nav tokens
|
|
2989
|
+
if inBottomZone {
|
|
2990
|
+
let navTokens = ["home", "profile", "account", "settings", "menu", "more", "dashboard", "tasks", "inbox", "search", "activity", "explore"]
|
|
2991
|
+
let text = (element.identifier + " " + element.label).lowercased()
|
|
2992
|
+
return navTokens.contains { text.contains($0) }
|
|
2993
|
+
}
|
|
2994
|
+
|
|
2995
|
+
return false
|
|
2996
|
+
}
|
|
2997
|
+
|
|
2998
|
+
private func selectedTabTitle() -> String? {
|
|
2999
|
+
let selectedPred = NSPredicate(format: "isSelected == 1")
|
|
3000
|
+
let selected = app.tabBars.buttons.matching(selectedPred).allElementsBoundByIndex
|
|
3001
|
+
.first { $0.exists && !$0.label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }
|
|
3002
|
+
if let selected {
|
|
3003
|
+
return selected.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3004
|
+
}
|
|
3005
|
+
|
|
3006
|
+
let allTabs = app.tabBars.buttons.allElementsBoundByIndex
|
|
3007
|
+
.filter { $0.exists && $0.isHittable && !$0.label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }
|
|
3008
|
+
if allTabs.count >= 2 {
|
|
3009
|
+
return allTabs.first?.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3010
|
+
}
|
|
3011
|
+
return nil
|
|
3012
|
+
}
|
|
3013
|
+
|
|
3014
|
+
private func tapTab(atRotationIndex tabIdx: Int) {
|
|
3015
|
+
let allTabs = app.tabBars.buttons.allElementsBoundByIndex
|
|
3016
|
+
.filter { $0.exists && $0.isHittable }
|
|
3017
|
+
.sorted { $0.frame.midX < $1.frame.midX }
|
|
3018
|
+
|
|
3019
|
+
if !allTabs.isEmpty {
|
|
3020
|
+
let safeIdx = max(0, min(tabIdx, allTabs.count - 1))
|
|
3021
|
+
allTabs[safeIdx].tap()
|
|
3022
|
+
return
|
|
3023
|
+
}
|
|
3024
|
+
|
|
3025
|
+
// Fallback for tab bars that don't expose buttons reliably in this runtime.
|
|
3026
|
+
let tabBarY = screenBounds.height > 0 ? screenBounds.height - 30 : 820.0
|
|
3027
|
+
let screenW = screenBounds.width > 0 ? screenBounds.width : 402.0
|
|
3028
|
+
let tabPositions: [CGFloat] = [0.12, 0.31, 0.5, 0.69, 0.88]
|
|
3029
|
+
let safeIdx = max(0, min(tabIdx, tabPositions.count - 1))
|
|
3030
|
+
let xPos = screenW * tabPositions[safeIdx]
|
|
3031
|
+
app.coordinate(withNormalizedOffset: .zero)
|
|
3032
|
+
.withOffset(CGVector(dx: xPos, dy: tabBarY)).tap()
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
// MARK: - Directed (targeted) exploration
|
|
3036
|
+
|
|
3037
|
+
/// Beeline from the current (root) screen to `target` as fast as possible by tapping each
|
|
3038
|
+
/// control label in `route` (learned from prior runs). Emits the same OCQA markers as normal
|
|
3039
|
+
/// exploration so the path is recorded and narrated ("Heading to X: tapping Y").
|
|
3040
|
+
@discardableResult
|
|
3041
|
+
private func beelineToTarget(target: String, route: [String], actionCount: inout Int, maxActions: Int) -> Bool {
|
|
3042
|
+
print("OCQA_STATE:directed_start target=\(escapeJSON(target)) route_len=\(route.count)")
|
|
3043
|
+
if currentTitleMatches(target) {
|
|
3044
|
+
print("OCQA_STATE:directed_reached target=\(escapeJSON(target)) step=\(actionCount)")
|
|
3045
|
+
emitPrExplorationTarget(status: "observed", screen: target)
|
|
3046
|
+
return true
|
|
3047
|
+
}
|
|
3048
|
+
for (routeIndex, label) in route.enumerated() {
|
|
3049
|
+
if actionCount >= maxActions { break }
|
|
3050
|
+
let elements = readUITree(app)
|
|
3051
|
+
let titleStr = detectTitle(elements) ?? "Unknown"
|
|
3052
|
+
let escapedTitle = escapeJSON(titleStr)
|
|
3053
|
+
let inputs = detectInputDescriptors(in: elements)
|
|
3054
|
+
let interactable = elements.filter { $0.isEnabled && isInteractable($0.type) }
|
|
3055
|
+
let role = classifyScreenRole(title: titleStr, elements: elements, inputs: inputs, interactable: interactable)
|
|
3056
|
+
let summary = describeScreen(title: titleStr, role: role, elements: elements, inputs: inputs, interactable: interactable)
|
|
3057
|
+
print("OCQA_STATE:{\"screen\":\"\(escapedTitle)\",\"hash\":\"\(computeHash(elements))\",\"elements\":\(elements.count),\"action\":\(actionCount),\"role\":\"\(escapeJSON(role))\",\"summary\":\"\(escapeJSON(summary))\",\"inputs\":[]}")
|
|
3058
|
+
|
|
3059
|
+
let directedTimeout = routeIndex < routeTimeouts.count ? routeTimeouts[routeIndex] : 0
|
|
3060
|
+
let tapped = tapControlByLabel(label, timeoutSeconds: directedTimeout)
|
|
3061
|
+
actionCount += 1
|
|
3062
|
+
let narrative = "Heading to \(target): tapping \(label)."
|
|
3063
|
+
print("OCQA_ACTION:{\"type\":\"tap\",\"target\":\"\(escapeJSON(label))\",\"reason\":\"directed_route\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(narrative))\"}")
|
|
3064
|
+
if !tapped {
|
|
3065
|
+
print("OCQA_STATE:directed_step_missed label=\(escapeJSON(label)) step=\(actionCount)")
|
|
3066
|
+
}
|
|
3067
|
+
waitForAnimationsToSettle()
|
|
3068
|
+
let afterElements = readUITree(app)
|
|
3069
|
+
let afterTitle = detectTitle(afterElements) ?? "Unknown"
|
|
3070
|
+
let afterInputs = detectInputDescriptors(in: afterElements)
|
|
3071
|
+
let afterInteractable = afterElements.filter { $0.isEnabled && isInteractable($0.type) }
|
|
3072
|
+
let afterRole = classifyScreenRole(title: afterTitle, elements: afterElements, inputs: afterInputs, interactable: afterInteractable)
|
|
3073
|
+
let afterSummary = describeScreen(title: afterTitle, role: afterRole, elements: afterElements, inputs: afterInputs, interactable: afterInteractable)
|
|
3074
|
+
let afterControls = mapControlsJSON(afterElements)
|
|
3075
|
+
print("OCQA_STATE:{\"screen\":\"\(escapeJSON(afterTitle))\",\"hash\":\"\(computeHash(afterElements))\",\"elements\":\(afterElements.count),\"action\":\(actionCount),\"role\":\"\(escapeJSON(afterRole))\",\"summary\":\"\(escapeJSON(afterSummary))\",\"inputs\":[],\"controls\":[\(afterControls)]}")
|
|
3076
|
+
if titleStr.caseInsensitiveCompare(afterTitle) != .orderedSame {
|
|
3077
|
+
print("OCQA_TRANSITION_RESOLVED:{\"from\":\"\(escapedTitle)\",\"to\":\"\(escapeJSON(afterTitle))\",\"action\":\"\(escapeJSON(label))\"}")
|
|
3078
|
+
}
|
|
3079
|
+
emitProgress(action: actionCount, maxActions: maxActions, states: 0)
|
|
3080
|
+
if afterTitle.caseInsensitiveCompare(target) == .orderedSame {
|
|
3081
|
+
print("OCQA_STATE:directed_reached target=\(escapeJSON(target)) step=\(actionCount)")
|
|
3082
|
+
emitPrExplorationTarget(status: "observed", screen: afterTitle)
|
|
3083
|
+
return true
|
|
3084
|
+
}
|
|
3085
|
+
}
|
|
3086
|
+
let reached = currentTitleMatches(target)
|
|
3087
|
+
print("OCQA_STATE:directed_\(reached ? "reached" : "not_reached") target=\(escapeJSON(target)) step=\(actionCount)")
|
|
3088
|
+
emitPrExplorationTarget(status: reached ? "observed" : "failed", screen: reached ? target : "", error: reached ? "" : "The observed UI Map path did not reach its target screen")
|
|
3089
|
+
return reached
|
|
3090
|
+
}
|
|
3091
|
+
|
|
3092
|
+
private func emitPrExplorationTarget(status: String, screen: String, error: String = "") {
|
|
3093
|
+
guard !prExplorationTargetId.isEmpty else { return }
|
|
3094
|
+
let errorField = error.isEmpty ? "" : ",\"error\":\"\(escapeJSON(error))\""
|
|
3095
|
+
print("OCQA_PR_TARGET:{\"targetId\":\"\(escapeJSON(prExplorationTargetId))\",\"status\":\"\(escapeJSON(status))\",\"screen\":\"\(escapeJSON(screen))\"\(errorField)}")
|
|
3096
|
+
}
|
|
3097
|
+
|
|
3098
|
+
private func currentTitleMatches(_ target: String) -> Bool {
|
|
3099
|
+
guard !target.isEmpty else { return false }
|
|
3100
|
+
let current = detectTitle(readUITree(app)) ?? ""
|
|
3101
|
+
return current.caseInsensitiveCompare(target) == .orderedSame
|
|
3102
|
+
}
|
|
3103
|
+
|
|
3104
|
+
/// Tap a control identified by its visible label/title. Tries fast element queries first, then a
|
|
3105
|
+
/// case-insensitive contains match, then a coordinate tap on the matching snapshot element.
|
|
3106
|
+
@discardableResult
|
|
3107
|
+
private func tapControlByLabel(_ label: String, timeoutSeconds: TimeInterval = 0) -> Bool {
|
|
3108
|
+
let trimmed = label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3109
|
+
guard !trimmed.isEmpty else { return false }
|
|
3110
|
+
let deadline = Date().addingTimeInterval(max(0, timeoutSeconds))
|
|
3111
|
+
repeat {
|
|
3112
|
+
for query in [app.buttons, app.cells, app.links, app.staticTexts] {
|
|
3113
|
+
let el = query[trimmed]
|
|
3114
|
+
if el.exists && el.isHittable { el.tap(); return true }
|
|
3115
|
+
}
|
|
3116
|
+
let exact = app.descendants(matching: .any).matching(NSPredicate(format: "label == %@", trimmed)).allElementsBoundByIndex
|
|
3117
|
+
if let match = exact.first(where: { $0.exists && $0.isHittable }) { match.tap(); return true }
|
|
3118
|
+
let contains = app.descendants(matching: .any).matching(NSPredicate(format: "label CONTAINS[c] %@", trimmed)).allElementsBoundByIndex
|
|
3119
|
+
if let match = contains.first(where: { $0.exists && $0.isHittable }) { match.tap(); return true }
|
|
3120
|
+
// Fall back to the flat snapshot (coordinate tap on the closest label match).
|
|
3121
|
+
if let hit = readUITree(app).first(where: {
|
|
3122
|
+
$0.isHittable && isInteractable($0.type)
|
|
3123
|
+
&& ($0.label.caseInsensitiveCompare(trimmed) == .orderedSame
|
|
3124
|
+
|| $0.label.localizedCaseInsensitiveContains(trimmed))
|
|
3125
|
+
}) {
|
|
3126
|
+
app.coordinate(withNormalizedOffset: .zero)
|
|
3127
|
+
.withOffset(CGVector(dx: hit.frame.midX, dy: hit.frame.midY)).tap()
|
|
3128
|
+
return true
|
|
3129
|
+
}
|
|
3130
|
+
if (exact + contains).contains(where: { $0.exists && !$0.isHittable }) { app.swipeUp() }
|
|
3131
|
+
if Date() < deadline { Thread.sleep(forTimeInterval: 0.2) }
|
|
3132
|
+
} while Date() < deadline
|
|
3133
|
+
return false
|
|
3134
|
+
}
|
|
3135
|
+
|
|
3136
|
+
private func hasVisibleGlobalNavigation(_ elements: [SimpleElement], screenBounds: CGRect) -> Bool {
|
|
3137
|
+
elements.filter { isLikelyGlobalNavigation($0, screenBounds: screenBounds) }.count >= 2
|
|
3138
|
+
}
|
|
3139
|
+
|
|
3140
|
+
private func isAnonymousControl(_ element: SimpleElement) -> Bool {
|
|
3141
|
+
element.identifier.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
|
3142
|
+
&& element.label.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
|
|
3143
|
+
}
|
|
3144
|
+
|
|
3145
|
+
private func isLikelyExploratoryNavigation(_ element: SimpleElement) -> Bool {
|
|
3146
|
+
if isTextField(element.type) { return false }
|
|
3147
|
+
if isLikelySubmitControl(element) { return true }
|
|
3148
|
+
if isAnonymousControl(element) { return false }
|
|
3149
|
+
|
|
3150
|
+
let text = (element.identifier + " " + element.label).lowercased()
|
|
3151
|
+
let mutationTokens = [
|
|
3152
|
+
"delete", "remove", "reset", "increment", "decrement", "mark complete",
|
|
3153
|
+
"mark incomplete", "toggle", "enable", "disable", "archive"
|
|
3154
|
+
]
|
|
3155
|
+
if mutationTokens.contains(where: { text.contains($0) }) {
|
|
3156
|
+
return false
|
|
3157
|
+
}
|
|
3158
|
+
|
|
3159
|
+
if element.type.contains("Cell") || element.type.contains("rawValue: 75") {
|
|
3160
|
+
return true
|
|
3161
|
+
}
|
|
3162
|
+
if element.type.contains("Link") || element.type.contains("rawValue: 39") {
|
|
3163
|
+
return true
|
|
3164
|
+
}
|
|
3165
|
+
if element.type.contains("Button") || element.type.contains("rawValue: 9") {
|
|
3166
|
+
return true
|
|
3167
|
+
}
|
|
3168
|
+
return false
|
|
3169
|
+
}
|
|
3170
|
+
|
|
3171
|
+
private func prioritizeElements(
|
|
3172
|
+
_ elements: [SimpleElement],
|
|
3173
|
+
actionCounts: [String: Int],
|
|
3174
|
+
elementScreenPresence: [String: Set<String>],
|
|
3175
|
+
persistentThreshold: Int,
|
|
3176
|
+
screenBounds: CGRect
|
|
3177
|
+
) -> [SimpleElement] {
|
|
3178
|
+
let bottomBarY = screenBounds.height > 0 ? screenBounds.height * 0.88 : 850.0
|
|
3179
|
+
|
|
3180
|
+
return elements.sorted { a, b in
|
|
3181
|
+
let keyA = actionKey(for: a)
|
|
3182
|
+
let keyB = actionKey(for: b)
|
|
3183
|
+
|
|
3184
|
+
let persistA = (elementScreenPresence[keyA]?.count ?? 0) >= persistentThreshold
|
|
3185
|
+
let persistB = (elementScreenPresence[keyB]?.count ?? 0) >= persistentThreshold
|
|
3186
|
+
if persistA != persistB { return !persistA }
|
|
3187
|
+
|
|
3188
|
+
let navA = isLikelyExploratoryNavigation(a)
|
|
3189
|
+
let navB = isLikelyExploratoryNavigation(b)
|
|
3190
|
+
if navA != navB { return navA }
|
|
3191
|
+
|
|
3192
|
+
let countA = actionCounts[keyA] ?? 0
|
|
3193
|
+
let countB = actionCounts[keyB] ?? 0
|
|
3194
|
+
if countA != countB { return countA < countB }
|
|
3195
|
+
|
|
3196
|
+
let anonymousA = isAnonymousControl(a)
|
|
3197
|
+
let anonymousB = isAnonymousControl(b)
|
|
3198
|
+
if anonymousA != anonymousB { return !anonymousA }
|
|
3199
|
+
|
|
3200
|
+
let inBarA = a.frame.midY > bottomBarY
|
|
3201
|
+
let inBarB = b.frame.midY > bottomBarY
|
|
3202
|
+
if inBarA != inBarB { return !inBarA }
|
|
3203
|
+
|
|
3204
|
+
let submitA = isLikelyPrimarySubmitControl(a)
|
|
3205
|
+
let submitB = isLikelyPrimarySubmitControl(b)
|
|
3206
|
+
if submitA != submitB { return submitA }
|
|
3207
|
+
|
|
3208
|
+
let pa = baseTypePriority(a.type)
|
|
3209
|
+
let pb = baseTypePriority(b.type)
|
|
3210
|
+
return pa > pb
|
|
3211
|
+
}
|
|
3212
|
+
}
|
|
3213
|
+
|
|
3214
|
+
private func baseTypePriority(_ type: String) -> Int {
|
|
3215
|
+
if type.contains("Cell") || type.contains("rawValue: 75") { return 5 }
|
|
3216
|
+
if type.contains("Link") || type.contains("rawValue: 39") { return 4 }
|
|
3217
|
+
if type.contains("Button") || type.contains("rawValue: 9") { return 4 }
|
|
3218
|
+
if type.contains("SegmentedControl") || type.contains("Picker") { return 3 }
|
|
3219
|
+
if type.contains("TextField") || type.contains("rawValue: 49") || type.contains("rawValue: 50") { return 2 }
|
|
3220
|
+
if type.contains("Switch") || type.contains("Toggle") || type.contains("rawValue: 40") { return 1 }
|
|
3221
|
+
return 0
|
|
3222
|
+
}
|
|
3223
|
+
|
|
3224
|
+
private func isLikelySubmitControl(_ element: SimpleElement) -> Bool {
|
|
3225
|
+
let text = (element.label + " " + element.identifier).lowercased()
|
|
3226
|
+
let submitTokens = [
|
|
3227
|
+
"continue", "next", "submit", "sign up", "create account",
|
|
3228
|
+
"register", "finish", "done", "save", "log in", "login", "sign in"
|
|
3229
|
+
]
|
|
3230
|
+
let negativeTokens = [
|
|
3231
|
+
"with apple", "with google", "with facebook", "forgot", "help",
|
|
3232
|
+
"terms", "privacy", "learn more", "cancel", "back"
|
|
3233
|
+
]
|
|
3234
|
+
guard submitTokens.contains(where: { text.contains($0) }) else { return false }
|
|
3235
|
+
return !negativeTokens.contains(where: { text.contains($0) })
|
|
3236
|
+
}
|
|
3237
|
+
|
|
3238
|
+
/// Stricter than isLikelySubmitControl — used only to decide whether a "stayed on the same
|
|
3239
|
+
/// screen" outcome is a real failed submit/sign-in. Excludes ambiguous tokens like "done"/"next"
|
|
3240
|
+
/// that are commonly filter segments or toolbar buttons (which caused false positives).
|
|
3241
|
+
private func isStrongSubmitControl(_ element: SimpleElement) -> Bool {
|
|
3242
|
+
let text = (element.label + " " + element.identifier).lowercased()
|
|
3243
|
+
let strong = ["sign in", "sign-in", "log in", "login", "signin",
|
|
3244
|
+
"sign up", "create account", "register", "submit", "continue", "save"]
|
|
3245
|
+
let negative = ["with apple", "with google", "with facebook", "forgot", "cancel", "back", "skip"]
|
|
3246
|
+
guard strong.contains(where: { text.contains($0) }) else { return false }
|
|
3247
|
+
return !negative.contains(where: { text.contains($0) })
|
|
3248
|
+
}
|
|
3249
|
+
|
|
3250
|
+
/// A text field we haven't put a value into yet. The typed-keys guard is the loop-safety:
|
|
3251
|
+
/// once typed into on this screen (even if the value never shows in the a11y tree — custom
|
|
3252
|
+
/// fields), a field counts as filled, so form-completion steering always terminates. The set is
|
|
3253
|
+
/// screen-scoped because anonymous action keys (position buckets) collide across screens.
|
|
3254
|
+
private func fieldLooksUnfilled(_ element: SimpleElement, screen: String, typedKeys: Set<String>) -> Bool {
|
|
3255
|
+
guard !typedKeys.contains("\(screen)|\(actionKey(for: element))") else { return false }
|
|
3256
|
+
let value = element.value.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3257
|
+
if value.isEmpty { return true }
|
|
3258
|
+
// A filled SecureField renders its value as bullets; any other non-empty value on a
|
|
3259
|
+
// never-typed secure field is its placeholder showing through (common under the keyboard,
|
|
3260
|
+
// where placeholderValue isn't resolvable) — treat as unfilled. (Live consequence: the
|
|
3261
|
+
// signup submit was tapped before Confirm Password → "Passwords do not match.")
|
|
3262
|
+
if isSecureTextField(element.type) && !value.contains("•") { return true }
|
|
3263
|
+
// An empty field often reports its placeholder as the value.
|
|
3264
|
+
let placeholder = element.xcElement?.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
|
3265
|
+
return !placeholder.isEmpty && value == placeholder
|
|
3266
|
+
}
|
|
3267
|
+
|
|
3268
|
+
/// Dismiss-style controls (Close/Cancel/X) that discard a form in progress. Deprioritized —
|
|
3269
|
+
/// never excluded — while unfilled fields remain, so they stay available as an escape hatch.
|
|
3270
|
+
private func isDismissControl(_ element: SimpleElement) -> Bool {
|
|
3271
|
+
let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
3272
|
+
let id = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
3273
|
+
let tokens: Set<String> = ["close", "cancel", "dismiss", "x", "xmark"]
|
|
3274
|
+
return tokens.contains(label) || tokens.contains(id) || id.hasPrefix("xmark.")
|
|
3275
|
+
}
|
|
3276
|
+
|
|
3277
|
+
/// Controls whose effect happens in SYSTEM UI presented by another process — OAuth sign-in
|
|
3278
|
+
/// (ASAuthorizationController / ASWebAuthenticationSession) and Apple Pay sheets. Those never
|
|
3279
|
+
/// register in this app's accessibility tree, so "no visible change after tapping" is the
|
|
3280
|
+
/// EXPECTED outcome of a working control, not evidence of a dead one. Excluded from
|
|
3281
|
+
/// unresponsive-element detection (first real-app run flagged "Continue with Apple/Google").
|
|
3282
|
+
private func isSystemHandoffControl(_ label: String) -> Bool {
|
|
3283
|
+
let l = label.lowercased()
|
|
3284
|
+
if l.contains("apple pay") { return true }
|
|
3285
|
+
let verbs = ["continue with", "sign in with", "sign up with", "log in with", "login with"]
|
|
3286
|
+
let providers = ["apple", "google", "facebook", "twitter", "github", "microsoft", "amazon", "linkedin", "x"]
|
|
3287
|
+
for v in verbs where l.hasPrefix(v) {
|
|
3288
|
+
let rest = l.dropFirst(v.count).trimmingCharacters(in: .whitespaces)
|
|
3289
|
+
// Whole-word provider match ("apple", "apple id") — not a prefix, so e.g.
|
|
3290
|
+
// "continue with xfinity" is NOT treated as the "x" provider.
|
|
3291
|
+
let firstWord = rest.split(separator: " ").first.map(String.init) ?? rest
|
|
3292
|
+
if providers.contains(firstWord) { return true }
|
|
3293
|
+
}
|
|
3294
|
+
return false
|
|
3295
|
+
}
|
|
3296
|
+
|
|
3297
|
+
/// Selection/value controls (segmented-control segments, stepper +/- buttons) legitimately
|
|
3298
|
+
/// produce no change when already-selected or at a boundary, so they must be excluded from
|
|
3299
|
+
/// "dead control" detection to avoid false positives.
|
|
3300
|
+
private func isSelectionOrValueControl(_ element: SimpleElement) -> Bool {
|
|
3301
|
+
if element.isSelected { return true }
|
|
3302
|
+
let name = (element.label + " " + element.identifier).lowercased()
|
|
3303
|
+
if name.contains("increment") || name.contains("decrement") { return true }
|
|
3304
|
+
// Reversible state-toggle buttons: their effect is a value/state flip that SwiftUI Forms
|
|
3305
|
+
// don't always reflect promptly in the accessibility tree, causing no-op false positives.
|
|
3306
|
+
let toggleVerbs = ["mark complete", "mark incomplete", "mark done", "mark as", "reopen",
|
|
3307
|
+
"toggle", "favorite", "unfavorite", "follow", "unfollow", "like",
|
|
3308
|
+
"show more", "show less", "read more", "expand", "collapse"]
|
|
3309
|
+
return toggleVerbs.contains { name.contains($0) }
|
|
3310
|
+
}
|
|
3311
|
+
|
|
3312
|
+
/// True if the control sits inside a segmented control or picker — selecting a segment whose
|
|
3313
|
+
/// filtered result happens to look identical is not a "dead control". Uses live queries (only
|
|
3314
|
+
/// called when we're about to flag, so the cost is negligible).
|
|
3315
|
+
private func isInsideSelectionContainer(_ element: SimpleElement) -> Bool {
|
|
3316
|
+
let center = CGPoint(x: element.frame.midX, y: element.frame.midY)
|
|
3317
|
+
for container in app.segmentedControls.allElementsBoundByIndex where container.exists {
|
|
3318
|
+
if container.frame.contains(center) { return true }
|
|
3319
|
+
}
|
|
3320
|
+
for container in app.pickers.allElementsBoundByIndex where container.exists {
|
|
3321
|
+
if container.frame.contains(center) { return true }
|
|
3322
|
+
}
|
|
3323
|
+
// Calendar / date-picker cells ("Monday, June 1", "Month") select a value rather than
|
|
3324
|
+
// navigate — a tap legitimately changes nothing the value-blind hash can see, so they
|
|
3325
|
+
// must not be flagged as dead controls (observed: 4 FPs on one booking screen).
|
|
3326
|
+
for container in app.datePickers.allElementsBoundByIndex where container.exists {
|
|
3327
|
+
if container.frame.contains(center) { return true }
|
|
3328
|
+
}
|
|
3329
|
+
return false
|
|
3330
|
+
}
|
|
3331
|
+
|
|
3332
|
+
private func isLikelyDismissControl(_ element: SimpleElement) -> Bool {
|
|
3333
|
+
let text = (element.label + " " + element.identifier).lowercased()
|
|
3334
|
+
let dismissTokens = ["done", "close", "cancel", "dismiss", "back", "skip", "not now"]
|
|
3335
|
+
let negative = ["delete", "remove", "reset", "logout"]
|
|
3336
|
+
guard dismissTokens.contains(where: { text.contains($0) }) else { return false }
|
|
3337
|
+
guard !negative.contains(where: { text.contains($0) }) else { return false }
|
|
3338
|
+
return element.type.contains("Button") || element.type.contains("rawValue: 9")
|
|
3339
|
+
}
|
|
3340
|
+
|
|
3341
|
+
private func isLikelyPrimarySubmitControl(_ element: SimpleElement) -> Bool {
|
|
3342
|
+
guard isLikelySubmitControl(element) else { return false }
|
|
3343
|
+
guard element.type.contains("Button") || element.type.contains("rawValue: 9") else { return false }
|
|
3344
|
+
|
|
3345
|
+
let screenHeight = screenBounds.height > 0 ? screenBounds.height : 1000
|
|
3346
|
+
// Primary CTA buttons are often lower half and wide.
|
|
3347
|
+
let lowerHalf = element.frame.midY > screenHeight * 0.45
|
|
3348
|
+
let wideEnough = element.frame.width > 120
|
|
3349
|
+
return lowerHalf || wideEnough
|
|
3350
|
+
}
|
|
3351
|
+
|
|
3352
|
+
private func shouldDismissKeyboardForSubmit(elements: [SimpleElement]) -> Bool {
|
|
3353
|
+
let keyboard = app.keyboards.firstMatch
|
|
3354
|
+
guard keyboard.exists, keyboard.frame.height > 0 else { return false }
|
|
3355
|
+
let keyboardTop = keyboard.frame.minY
|
|
3356
|
+
|
|
3357
|
+
return elements.contains { el in
|
|
3358
|
+
guard isLikelySubmitControl(el) else { return false }
|
|
3359
|
+
// If submit control is overlapped by keyboard, dismiss keyboard first.
|
|
3360
|
+
return el.frame.maxY >= keyboardTop - 8
|
|
3361
|
+
}
|
|
3362
|
+
}
|
|
3363
|
+
|
|
3364
|
+
private func resolveTextElement(for element: SimpleElement, in app: XCUIApplication) -> XCUIElement? {
|
|
3365
|
+
if !element.identifier.isEmpty {
|
|
3366
|
+
let tf = app.textFields[element.identifier]
|
|
3367
|
+
if tf.exists { return tf }
|
|
3368
|
+
let stf = app.secureTextFields[element.identifier]
|
|
3369
|
+
if stf.exists { return stf }
|
|
3370
|
+
}
|
|
3371
|
+
|
|
3372
|
+
let allTextFields = app.textFields.allElementsBoundByIndex + app.secureTextFields.allElementsBoundByIndex
|
|
3373
|
+
let tapped = CGPoint(x: element.frame.midX, y: element.frame.midY)
|
|
3374
|
+
return allTextFields
|
|
3375
|
+
.filter { $0.exists && $0.frame.width > 0 }
|
|
3376
|
+
.min(by: {
|
|
3377
|
+
let d1 = abs($0.frame.midX - tapped.x) + abs($0.frame.midY - tapped.y)
|
|
3378
|
+
let d2 = abs($1.frame.midX - tapped.x) + abs($1.frame.midY - tapped.y)
|
|
3379
|
+
return d1 < d2
|
|
3380
|
+
})
|
|
3381
|
+
}
|
|
3382
|
+
|
|
3383
|
+
private func elementNeedsInput(_ element: SimpleElement, in app: XCUIApplication) -> Bool {
|
|
3384
|
+
guard isTextField(element.type) else { return false }
|
|
3385
|
+
guard let resolved = resolveTextElement(for: element, in: app), resolved.exists else { return true }
|
|
3386
|
+
|
|
3387
|
+
guard let current = resolved.value as? String else { return true }
|
|
3388
|
+
let value = current.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3389
|
+
if value.isEmpty { return true }
|
|
3390
|
+
|
|
3391
|
+
let placeholder = (resolved.placeholderValue ?? "")
|
|
3392
|
+
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3393
|
+
.lowercased()
|
|
3394
|
+
let lower = value.lowercased()
|
|
3395
|
+
if !placeholder.isEmpty && lower == placeholder { return true }
|
|
3396
|
+
if lower == "optional" { return true }
|
|
3397
|
+
if lower.contains("enter ") && lower.contains("password") { return true }
|
|
3398
|
+
|
|
3399
|
+
return false
|
|
3400
|
+
}
|
|
3401
|
+
|
|
3402
|
+
private func performSmartAction(
|
|
3403
|
+
on element: SimpleElement,
|
|
3404
|
+
in app: XCUIApplication,
|
|
3405
|
+
screenTitle: String,
|
|
3406
|
+
inputOverrides: [String: String],
|
|
3407
|
+
elementTapCount: Int = 0,
|
|
3408
|
+
screenElements: [SimpleElement] = [],
|
|
3409
|
+
screenRole: String = ""
|
|
3410
|
+
) -> String {
|
|
3411
|
+
let frame = element.frame
|
|
3412
|
+
guard frame.width > 0, frame.height > 0 else {
|
|
3413
|
+
return "skip_invalid_frame"
|
|
3414
|
+
}
|
|
3415
|
+
|
|
3416
|
+
let coord = app.coordinate(withNormalizedOffset: .zero)
|
|
3417
|
+
.withOffset(CGVector(dx: frame.midX, dy: frame.midY))
|
|
3418
|
+
|
|
3419
|
+
if isTextField(element.type) {
|
|
3420
|
+
// Resolve the real XCUIElement so XCUITest can scroll it into view before tapping.
|
|
3421
|
+
// Coordinate taps cannot scroll and get fooled when the keyboard occludes lower
|
|
3422
|
+
// fields — that's why multi-field forms (e.g. Create Account) only filled the top
|
|
3423
|
+
// two fields. Tapping the resolved element auto-scrolls it above the keyboard.
|
|
3424
|
+
let resolvedField: XCUIElement? = {
|
|
3425
|
+
if let xc = element.xcElement, xc.exists { return xc }
|
|
3426
|
+
return resolveTextElement(for: element, in: app)
|
|
3427
|
+
}()
|
|
3428
|
+
|
|
3429
|
+
if let field = resolvedField, field.exists {
|
|
3430
|
+
// If a keyboard from a previous field is occluding this one, drop it first so
|
|
3431
|
+
// the tap lands on the field rather than a key, then XCUITest scrolls it up.
|
|
3432
|
+
if app.keyboards.firstMatch.exists && !field.isHittable {
|
|
3433
|
+
dismissKeyboardIfNeeded()
|
|
3434
|
+
Thread.sleep(forTimeInterval: 0.25)
|
|
3435
|
+
}
|
|
3436
|
+
field.tap()
|
|
3437
|
+
} else {
|
|
3438
|
+
coord.tap()
|
|
3439
|
+
}
|
|
3440
|
+
Thread.sleep(forTimeInterval: 0.4)
|
|
3441
|
+
|
|
3442
|
+
// Check if keyboard appeared — if not, the field didn't gain focus
|
|
3443
|
+
let keyboard = app.keyboards.firstMatch
|
|
3444
|
+
guard keyboard.waitForExistence(timeout: 1.5) else {
|
|
3445
|
+
let name = element.identifier.isEmpty ? element.label : element.identifier
|
|
3446
|
+
return "tap(\(name))_no_keyboard"
|
|
3447
|
+
}
|
|
3448
|
+
|
|
3449
|
+
// Bare SwiftUI TextFields often expose NO label/identifier of their own — their meaning
|
|
3450
|
+
// lives in a nearby static text ("Email", "Full Name"). Use the same inference the
|
|
3451
|
+
// input-descriptor path uses, so the default value matches the field's purpose
|
|
3452
|
+
// (email → a real address) instead of a generic "test" that any validation rejects.
|
|
3453
|
+
let ownHint = (element.identifier + element.label).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3454
|
+
let inferredLabel = ownHint.isEmpty ? (inferFieldLabel(for: element, in: screenElements) ?? "") : ""
|
|
3455
|
+
let effectiveLabel = element.label.isEmpty ? inferredLabel : element.label
|
|
3456
|
+
let hint = (element.identifier + " " + element.label + " " + inferredLabel).lowercased()
|
|
3457
|
+
let isSecure = isSecureTextField(element.type) || hint.contains("password") || hint.contains("passcode")
|
|
3458
|
+
let testText: String
|
|
3459
|
+
var isOverride = false
|
|
3460
|
+
if let overrideValue = resolveInputOverride(screenTitle: screenTitle, element: element, overrides: inputOverrides), !overrideValue.isEmpty {
|
|
3461
|
+
testText = overrideValue
|
|
3462
|
+
isOverride = true
|
|
3463
|
+
} else {
|
|
3464
|
+
testText = defaultInputValue(label: effectiveLabel, identifier: element.identifier,
|
|
3465
|
+
secure: isSecure, screenRole: screenRole)
|
|
3466
|
+
}
|
|
3467
|
+
lastTypedWasOverride = isOverride
|
|
3468
|
+
lastTypedSecure = isSecure
|
|
3469
|
+
|
|
3470
|
+
// Explicit, parseable marker of exactly what value was entered (masked if secure).
|
|
3471
|
+
let fieldName = element.identifier.isEmpty ? effectiveLabel : element.identifier
|
|
3472
|
+
let displayValue = isSecure ? String(repeating: "•", count: min(8, max(4, testText.count))) : testText
|
|
3473
|
+
print("OCQA_INPUT:{\"field\":\"\(escapeJSON(fieldName))\",\"value\":\"\(escapeJSON(displayValue))\",\"source\":\"\(isOverride ? "user_override" : "auto")\",\"secure\":\(isSecure),\"screen\":\"\(escapeJSON(screenTitle))\"}")
|
|
3474
|
+
if let field = resolvedField, field.exists {
|
|
3475
|
+
replaceText(on: field, with: testText)
|
|
3476
|
+
} else if let xcEl = element.xcElement, xcEl.exists {
|
|
3477
|
+
replaceText(on: xcEl, with: testText)
|
|
3478
|
+
} else if let resolved = resolveTextElement(for: element, in: app) {
|
|
3479
|
+
resolved.tap()
|
|
3480
|
+
Thread.sleep(forTimeInterval: 0.3)
|
|
3481
|
+
if keyboard.exists {
|
|
3482
|
+
replaceText(on: resolved, with: testText)
|
|
3483
|
+
}
|
|
3484
|
+
}
|
|
3485
|
+
|
|
3486
|
+
// The typed text is VISIBLE on screen — but does the field expose it to accessibility?
|
|
3487
|
+
// If its a11y value still reads empty (or just the placeholder), VoiceOver and any
|
|
3488
|
+
// accessibility-driven tooling see nothing: a real a11y defect, found live on a real
|
|
3489
|
+
// app's login form (custom-styled fields). Secure fields are exempt — masking is
|
|
3490
|
+
// intentional. One report per screen.
|
|
3491
|
+
if !isSecure, !testText.isEmpty, !valueHiddenReported.contains(screenTitle) {
|
|
3492
|
+
Thread.sleep(forTimeInterval: 0.2)
|
|
3493
|
+
let checkEl = (resolvedField?.exists == true) ? resolvedField : resolveTextElement(for: element, in: app)
|
|
3494
|
+
if let checkEl, checkEl.exists {
|
|
3495
|
+
let v = ((checkEl.value as? String) ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3496
|
+
let ph = (checkEl.placeholderValue ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3497
|
+
// Persistence probe memory: only values PROVEN visible in the a11y value are
|
|
3498
|
+
// remembered — a11y-hidden fields must not later read as "didn't persist".
|
|
3499
|
+
// And only PERSISTENT-CLASS fields (profile/settings data users expect to
|
|
3500
|
+
// stick): transient composers (feedback, message, search, comment) clear by
|
|
3501
|
+
// design and must never fire this probe (measured FP on a clean fixture).
|
|
3502
|
+
let persistentWords = ["name", "email", "phone", "username", "address", "city", "zip", "company", "title", "nickname", "bio"]
|
|
3503
|
+
let fieldHint = (fieldName + " " + effectiveLabel).lowercased()
|
|
3504
|
+
if v.contains(testText), persistentWords.contains(where: { fieldHint.contains($0) }) {
|
|
3505
|
+
typedFieldMemory["\(screenTitle)|\(fieldName)"] = testText
|
|
3506
|
+
}
|
|
3507
|
+
if v.isEmpty || v == ph {
|
|
3508
|
+
valueHiddenReported.insert(screenTitle)
|
|
3509
|
+
let issueTitle = "Field content invisible to accessibility: \(screenTitle)"
|
|
3510
|
+
print("OCQA_ISSUE:{\"type\":\"a11y_value_hidden\",\"severity\":\"low\",\"title\":\"\(escapeJSON(issueTitle))\",\"screen\":\"\(escapeJSON(screenTitle))\",\"desc\":\"Text typed into a field on '\(escapeJSON(screenTitle))' is visible on screen but the field's accessibility value reads empty — invisible to VoiceOver and accessibility tooling.\"}")
|
|
3511
|
+
}
|
|
3512
|
+
}
|
|
3513
|
+
}
|
|
3514
|
+
|
|
3515
|
+
// Try to advance focus to the next field or dismiss keyboard where supported.
|
|
3516
|
+
if keyboard.exists {
|
|
3517
|
+
for key in ["Next", "Done", "Go", "Return", "Continue"] {
|
|
3518
|
+
let keyButton = keyboard.buttons[key]
|
|
3519
|
+
if keyButton.exists && keyButton.isHittable {
|
|
3520
|
+
keyButton.tap()
|
|
3521
|
+
Thread.sleep(forTimeInterval: 0.2)
|
|
3522
|
+
break
|
|
3523
|
+
}
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
let name = element.identifier.isEmpty ? element.label : element.identifier
|
|
3527
|
+
return "type(\(name), \"\(testText)\")"
|
|
3528
|
+
}
|
|
3529
|
+
|
|
3530
|
+
// Long-press for cells and images (context menus, quick actions)
|
|
3531
|
+
// Only long-press after we've tapped this element at least twice without nav change
|
|
3532
|
+
let longPressTypes = ["Cell", "Image"]
|
|
3533
|
+
if longPressTypes.contains(element.type) && elementTapCount >= 2 {
|
|
3534
|
+
coord.press(forDuration: 1.2)
|
|
3535
|
+
let name = element.identifier.isEmpty ? element.label : element.identifier
|
|
3536
|
+
return "long_press(\(name))"
|
|
3537
|
+
}
|
|
3538
|
+
|
|
3539
|
+
|
|
3540
|
+
|
|
3541
|
+
coord.tap()
|
|
3542
|
+
return "tap(\(element.identifier.isEmpty ? element.label : element.identifier))"
|
|
3543
|
+
}
|
|
3544
|
+
|
|
3545
|
+
/// Attempt a swipe-left or swipe-right gesture on the given element (carousel/onboarding).
|
|
3546
|
+
private func performSwipeLateral(on element: SimpleElement, in app: XCUIApplication, direction: String) -> String {
|
|
3547
|
+
let frame = element.frame
|
|
3548
|
+
let startX: CGFloat = direction == "left" ? frame.maxX - 10 : frame.minX + 10
|
|
3549
|
+
let endX: CGFloat = direction == "left" ? frame.minX + 10 : frame.maxX - 10
|
|
3550
|
+
let midY = frame.midY
|
|
3551
|
+
let start = app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: startX, dy: midY))
|
|
3552
|
+
let end = app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: endX, dy: midY))
|
|
3553
|
+
start.press(forDuration: 0.05, thenDragTo: end)
|
|
3554
|
+
let name = element.identifier.isEmpty ? element.label : element.identifier
|
|
3555
|
+
return "swipe_\(direction)(\(name))"
|
|
3556
|
+
}
|
|
3557
|
+
|
|
3558
|
+
/// Full-width leftward swipe across the screen middle — advances a paged carousel.
|
|
3559
|
+
private func swipeScreenLeft() {
|
|
3560
|
+
let midY = screenBounds.height > 0 ? screenBounds.height / 2 : 400
|
|
3561
|
+
let w = screenBounds.width > 0 ? screenBounds.width : 390
|
|
3562
|
+
let start = app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: w * 0.85, dy: midY))
|
|
3563
|
+
let end = app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: w * 0.15, dy: midY))
|
|
3564
|
+
start.press(forDuration: 0.05, thenDragTo: end)
|
|
3565
|
+
}
|
|
3566
|
+
|
|
3567
|
+
/// Attempt an edge-swipe from the left edge (drawer/hamburger reveal).
|
|
3568
|
+
private func performEdgeSwipeLeft(in app: XCUIApplication) -> String {
|
|
3569
|
+
let midY = screenBounds.height > 0 ? screenBounds.height / 2 : 400
|
|
3570
|
+
let start = app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: 5, dy: midY))
|
|
3571
|
+
let end = app.coordinate(withNormalizedOffset: .zero).withOffset(CGVector(dx: screenBounds.width * 0.6, dy: midY))
|
|
3572
|
+
start.press(forDuration: 0.05, thenDragTo: end)
|
|
3573
|
+
return "edge_swipe_right"
|
|
3574
|
+
}
|
|
3575
|
+
|
|
3576
|
+
private func tryGoBack() -> Bool {
|
|
3577
|
+
// Try navigation bar back button
|
|
3578
|
+
let backButtons = app.navigationBars.buttons
|
|
3579
|
+
if backButtons.count > 0 {
|
|
3580
|
+
let first = backButtons.firstMatch
|
|
3581
|
+
if first.exists && first.isHittable {
|
|
3582
|
+
first.tap()
|
|
3583
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
3584
|
+
return true
|
|
3585
|
+
}
|
|
3586
|
+
}
|
|
3587
|
+
// Try common dismiss buttons (expanded set)
|
|
3588
|
+
for label in ["Close", "Cancel", "Done", "Dismiss", "Back", "X", "close"] {
|
|
3589
|
+
let btn = app.buttons[label]
|
|
3590
|
+
if btn.exists && btn.isHittable {
|
|
3591
|
+
btn.tap()
|
|
3592
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
3593
|
+
return true
|
|
3594
|
+
}
|
|
3595
|
+
}
|
|
3596
|
+
// Try swipe-down to dismiss sheets/modals
|
|
3597
|
+
let swipeStart = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.3))
|
|
3598
|
+
let swipeEnd = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.9))
|
|
3599
|
+
swipeStart.press(forDuration: 0.1, thenDragTo: swipeEnd)
|
|
3600
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
3601
|
+
return false // can't know if swipe worked — caller should verify
|
|
3602
|
+
}
|
|
3603
|
+
|
|
3604
|
+
private func performScroll(in app: XCUIApplication, upward: Bool) -> String {
|
|
3605
|
+
let startY: CGFloat = upward ? 0.78 : 0.28
|
|
3606
|
+
let endY: CGFloat = upward ? 0.30 : 0.78
|
|
3607
|
+
let start = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: startY))
|
|
3608
|
+
let end = app.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: endY))
|
|
3609
|
+
start.press(forDuration: 0.05, thenDragTo: end)
|
|
3610
|
+
return upward ? "scroll_up" : "scroll_down"
|
|
3611
|
+
}
|
|
3612
|
+
|
|
3613
|
+
private func performCoordinateTap(in app: XCUIApplication, at point: CGPoint, label: String) -> String {
|
|
3614
|
+
let coord = app.coordinate(withNormalizedOffset: .zero)
|
|
3615
|
+
.withOffset(CGVector(dx: point.x, dy: point.y))
|
|
3616
|
+
coord.tap()
|
|
3617
|
+
return "tap(\(label))"
|
|
3618
|
+
}
|
|
3619
|
+
|
|
3620
|
+
/// Wait for UI element count to stabilize (animations settled, content loaded).
|
|
3621
|
+
/// Polls every 200ms; exits early when count is stable for 2 consecutive reads or timeout reached.
|
|
3622
|
+
@discardableResult
|
|
3623
|
+
private func waitForUIStability(timeout: TimeInterval = 2.0) -> Int {
|
|
3624
|
+
let deadline = Date().addingTimeInterval(timeout)
|
|
3625
|
+
var previousCount = -1
|
|
3626
|
+
var stableCount = 0
|
|
3627
|
+
while Date() < deadline {
|
|
3628
|
+
// If the app died (e.g. it crashed on the action we just performed), stop polling
|
|
3629
|
+
// immediately. Querying a non-running app throws an XCTest failure that would abort the
|
|
3630
|
+
// whole run — instead we return and let the main loop's crash detection report it.
|
|
3631
|
+
guard app.state == .runningForeground else { return previousCount }
|
|
3632
|
+
let count = app.descendants(matching: .any).count
|
|
3633
|
+
if count == previousCount {
|
|
3634
|
+
stableCount += 1
|
|
3635
|
+
if stableCount >= 2 { return count }
|
|
3636
|
+
} else {
|
|
3637
|
+
stableCount = 0
|
|
3638
|
+
}
|
|
3639
|
+
previousCount = count
|
|
3640
|
+
Thread.sleep(forTimeInterval: 0.2)
|
|
3641
|
+
}
|
|
3642
|
+
return previousCount
|
|
3643
|
+
}
|
|
3644
|
+
|
|
3645
|
+
private func waitForAnimationsToSettle() {
|
|
3646
|
+
waitForUIStability(timeout: 1.2)
|
|
3647
|
+
}
|
|
3648
|
+
|
|
3649
|
+
/// Wait for the result of a form/login submission: poll until the screen changes (success
|
|
3650
|
+
/// navigation or inline error) and no activity indicator/"loading" spinner remains, up to
|
|
3651
|
+
/// `timeout`. Avoids racing ahead while a network request is still in flight.
|
|
3652
|
+
private func waitForSubmitResult(previousHash: String, timeout: TimeInterval = 8.0) {
|
|
3653
|
+
let deadline = Date().addingTimeInterval(timeout)
|
|
3654
|
+
var settledAfterChange = false
|
|
3655
|
+
while Date() < deadline {
|
|
3656
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
3657
|
+
// Treat a visible spinner or progress indicator as "still working".
|
|
3658
|
+
let busy = app.activityIndicators.firstMatch.exists
|
|
3659
|
+
|| app.progressIndicators.firstMatch.exists
|
|
3660
|
+
let elements = readUITree(app)
|
|
3661
|
+
if elements.isEmpty { continue }
|
|
3662
|
+
let newHash = computeHash(elements)
|
|
3663
|
+
if newHash != previousHash {
|
|
3664
|
+
// Screen changed — let it settle one more beat, then stop waiting.
|
|
3665
|
+
if !busy {
|
|
3666
|
+
if settledAfterChange { return }
|
|
3667
|
+
settledAfterChange = true
|
|
3668
|
+
}
|
|
3669
|
+
} else if busy {
|
|
3670
|
+
settledAfterChange = false
|
|
3671
|
+
}
|
|
3672
|
+
}
|
|
3673
|
+
}
|
|
3674
|
+
|
|
3675
|
+
/// Frame of the on-screen keyboard, or .zero if none. Used to keep keyboard keys out of the
|
|
3676
|
+
/// tappable-candidate pool.
|
|
3677
|
+
private func keyboardFrame() -> CGRect {
|
|
3678
|
+
let kb = app.keyboards.firstMatch
|
|
3679
|
+
return kb.exists ? kb.frame : .zero
|
|
3680
|
+
}
|
|
3681
|
+
|
|
3682
|
+
/// True when the screen is in a "settled" resting state — no on-screen keyboard and no open
|
|
3683
|
+
/// transient overlay (menu / dropdown / popover / sheet / picker wheel). A screenshot taken while
|
|
3684
|
+
/// one of these is up is inherently ambiguous to a visual reviewer (the keyboard "covers" the
|
|
3685
|
+
/// form, an open menu "overlaps" content) and is the dominant source of vision false positives, so
|
|
3686
|
+
/// the post-run vision pass prefers settled captures. Emitted as `settled` on OCQA_STATE.
|
|
3687
|
+
private func isScreenSettled() -> Bool {
|
|
3688
|
+
if app.keyboards.firstMatch.exists { return false }
|
|
3689
|
+
if app.sheets.firstMatch.exists { return false }
|
|
3690
|
+
if app.popovers.firstMatch.exists { return false }
|
|
3691
|
+
if app.menus.firstMatch.exists { return false }
|
|
3692
|
+
if app.pickerWheels.firstMatch.exists { return false }
|
|
3693
|
+
// A SwiftUI Menu (bridged UIMenu) presents as NEITHER app.menus nor app.popovers in
|
|
3694
|
+
// current runtimes, so the checks above miss it (verified live: a menu-open Dashboard
|
|
3695
|
+
// frame carried settled:true and became the vision pass's representative screenshot —
|
|
3696
|
+
// the exact mid-interaction FP class settled-selection exists to prevent). Its scrim
|
|
3697
|
+
// blocks hit-testing of everything beneath, so chrome that exists but isn't hittable
|
|
3698
|
+
// is a type-taxonomy-free "something is covering this screen" signal. A false negative
|
|
3699
|
+
// here only deprioritizes a frame (selection falls back when no settled capture exists).
|
|
3700
|
+
let tabBar = app.tabBars.firstMatch
|
|
3701
|
+
if tabBar.exists && !tabBar.isHittable { return false }
|
|
3702
|
+
let navBar = app.navigationBars.firstMatch
|
|
3703
|
+
if navBar.exists && !navBar.isHittable { return false }
|
|
3704
|
+
return true
|
|
3705
|
+
}
|
|
3706
|
+
|
|
3707
|
+
/// True if the current screen is scrollable (ScrollView / List-Form table / collection), so
|
|
3708
|
+
/// below-the-fold content can be revealed by scrolling.
|
|
3709
|
+
private func isScrollableScreen() -> Bool {
|
|
3710
|
+
app.scrollViews.firstMatch.exists || app.tables.firstMatch.exists || app.collectionViews.firstMatch.exists
|
|
3711
|
+
}
|
|
3712
|
+
|
|
3713
|
+
private func dismissKeyboardIfNeeded() {
|
|
3714
|
+
let keyboard = app.keyboards.firstMatch
|
|
3715
|
+
guard keyboard.exists && keyboard.frame.height > 0 else { return }
|
|
3716
|
+
// The return key's label varies by iOS version and submitLabel (Done/Return/Go/Next,
|
|
3717
|
+
// upper- or lowercase) — match case-insensitively via the label list.
|
|
3718
|
+
for key in ["Done", "Return", "Go", "Next", "done", "return", "go", "next"] {
|
|
3719
|
+
let keyButton = keyboard.buttons[key]
|
|
3720
|
+
if keyButton.exists && keyButton.isHittable {
|
|
3721
|
+
keyButton.tap()
|
|
3722
|
+
Thread.sleep(forTimeInterval: 0.25)
|
|
3723
|
+
if !app.keyboards.firstMatch.exists { return }
|
|
3724
|
+
break // key existed but the keyboard stayed (focus advanced) — fall through
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
3727
|
+
// Send a literal return to the focused field. Unlike tapping "just above the keyboard"
|
|
3728
|
+
// (which can land on ANOTHER text field and keep the keyboard up — observed on a real
|
|
3729
|
+
// signup sheet), this reaches the first responder; with submitLabel done/go it also ends
|
|
3730
|
+
// editing. MUST verify something actually has focus first: typeText without a first
|
|
3731
|
+
// responder throws an event-synthesis exception that ABORTS THE ENTIRE TEST even with
|
|
3732
|
+
// continueAfterFailure=true (observed: a system Markup sheet reported a keyboard with no
|
|
3733
|
+
// focused field, ending a real-app run at 17/70 actions).
|
|
3734
|
+
if app.keyboards.firstMatch.exists {
|
|
3735
|
+
let focused = app.descendants(matching: .any)
|
|
3736
|
+
.matching(NSPredicate(format: "hasKeyboardFocus == true")).firstMatch
|
|
3737
|
+
if focused.exists {
|
|
3738
|
+
focused.typeText("\n")
|
|
3739
|
+
Thread.sleep(forTimeInterval: 0.25)
|
|
3740
|
+
}
|
|
3741
|
+
}
|
|
3742
|
+
}
|
|
3743
|
+
|
|
3744
|
+
private func escapeJSON(_ str: String) -> String {
|
|
3745
|
+
return str
|
|
3746
|
+
.replacingOccurrences(of: "\\", with: "\\\\")
|
|
3747
|
+
.replacingOccurrences(of: "\"", with: "'")
|
|
3748
|
+
.replacingOccurrences(of: "\n", with: " ")
|
|
3749
|
+
.replacingOccurrences(of: "\r", with: " ")
|
|
3750
|
+
.replacingOccurrences(of: "\t", with: " ")
|
|
3751
|
+
}
|
|
3752
|
+
|
|
3753
|
+
private func normalizeKey(_ value: String) -> String {
|
|
3754
|
+
value
|
|
3755
|
+
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
3756
|
+
.lowercased()
|
|
3757
|
+
}
|
|
3758
|
+
|
|
3759
|
+
private func inputFieldKey(for element: SimpleElement) -> String {
|
|
3760
|
+
let id = normalizeKey(element.identifier)
|
|
3761
|
+
if !id.isEmpty {
|
|
3762
|
+
return "id:\(id)"
|
|
3763
|
+
}
|
|
3764
|
+
|
|
3765
|
+
let label = normalizeKey(element.label)
|
|
3766
|
+
if !label.isEmpty {
|
|
3767
|
+
return "label:\(label)"
|
|
3768
|
+
}
|
|
3769
|
+
|
|
3770
|
+
return "coord:\(Int(element.frame.midX))x\(Int(element.frame.midY))"
|
|
3771
|
+
}
|
|
3772
|
+
|
|
3773
|
+
private func resolveInputOverride(
|
|
3774
|
+
screenTitle: String,
|
|
3775
|
+
element: SimpleElement,
|
|
3776
|
+
overrides: [String: String]
|
|
3777
|
+
) -> String? {
|
|
3778
|
+
let screen = normalizeKey(screenTitle)
|
|
3779
|
+
let fieldKey = inputFieldKey(for: element)
|
|
3780
|
+
|
|
3781
|
+
let scopedKey = "screen:\(screen)|\(fieldKey)"
|
|
3782
|
+
if let value = overrides[scopedKey], !value.isEmpty {
|
|
3783
|
+
return value
|
|
3784
|
+
}
|
|
3785
|
+
if let value = overrides[fieldKey], !value.isEmpty {
|
|
3786
|
+
return value
|
|
3787
|
+
}
|
|
3788
|
+
return nil
|
|
3789
|
+
}
|
|
3790
|
+
|
|
3791
|
+
/// The value the harness auto-types into a field when there's no user override. Extracted so
|
|
3792
|
+
/// the interactive prompt can display the same default the harness would actually use.
|
|
3793
|
+
private func defaultInputValue(label: String, identifier: String, secure: Bool, screenRole: String = "") -> String {
|
|
3794
|
+
let hint = (identifier + " " + label).lowercased()
|
|
3795
|
+
if hint.contains("email") || hint.contains("e-mail") {
|
|
3796
|
+
return resolve("OCQA_TEST_EMAIL", fallback: "test@example.com")
|
|
3797
|
+
} else if secure {
|
|
3798
|
+
// Any secure field gets the configured password — even when it exposes no
|
|
3799
|
+
// "password" hint (common: a bare SecureTextField with no identifier/placeholder).
|
|
3800
|
+
return resolve("OCQA_TEST_PASSWORD", fallback: "TestPass123!")
|
|
3801
|
+
} else if hint.contains("phone") || hint.contains("mobile") {
|
|
3802
|
+
return "5551234567"
|
|
3803
|
+
} else if hint.contains("name") || hint.contains("first") || hint.contains("last") {
|
|
3804
|
+
return "Test User"
|
|
3805
|
+
} else if hint.contains("zip") || hint.contains("postal") {
|
|
3806
|
+
return "90210"
|
|
3807
|
+
} else if hint.contains("weight") {
|
|
3808
|
+
return "170"
|
|
3809
|
+
} else if hint.contains("height") {
|
|
3810
|
+
return "68"
|
|
3811
|
+
} else if hint.contains("age") || hint.contains("years") {
|
|
3812
|
+
return "30"
|
|
3813
|
+
} else if hint.contains("(lbs") || hint.contains("(kg") || hint.contains("(cm") || hint.contains("(in)") {
|
|
3814
|
+
// Unit-suffixed numeric fields ("Current Weight (lbs)") reject text outright.
|
|
3815
|
+
return "50"
|
|
3816
|
+
} else if hint.contains("search") {
|
|
3817
|
+
return "test"
|
|
3818
|
+
} else if screenRole == "login" || screenRole == "signup" {
|
|
3819
|
+
// An un-hinted plain field on a credentials screen is almost always the email/username
|
|
3820
|
+
// — "test" fails server-side validation everywhere, an address passes both. (Real-app
|
|
3821
|
+
// run: a signup email field whose nearest static text was the screen TITLE got "test",
|
|
3822
|
+
// so the account could never be created.)
|
|
3823
|
+
return resolve("OCQA_TEST_EMAIL", fallback: "test@example.com")
|
|
3824
|
+
} else {
|
|
3825
|
+
return "test"
|
|
3826
|
+
}
|
|
3827
|
+
}
|
|
3828
|
+
|
|
3829
|
+
/// True when neither a screen-scoped nor a global override exists for this field key.
|
|
3830
|
+
private func hasNoOverride(key: String, screen: String, in overrides: [String: String]) -> Bool {
|
|
3831
|
+
let scoped = "screen:\(normalizeKey(screen))|\(key)"
|
|
3832
|
+
return (overrides[scoped]?.isEmpty ?? true) && (overrides[key]?.isEmpty ?? true)
|
|
3833
|
+
}
|
|
3834
|
+
|
|
3835
|
+
/// Pauses exploration and blocks until the host writes field values to `responsePath` (or the
|
|
3836
|
+
/// fallback timeout elapses). Submitted values are merged into `overrides` (screen-scoped) so
|
|
3837
|
+
/// the normal typing path picks them up via `resolveInputOverride`. Never hangs: on timeout it
|
|
3838
|
+
/// falls back to auto-defaults and disables further prompting for the rest of the run.
|
|
3839
|
+
/// Returns the resolved action ("submit" / "defaults" / "skip" / "dont_ask" / "timeout") so
|
|
3840
|
+
/// callers (e.g. the login preamble) can branch on it.
|
|
3841
|
+
@discardableResult
|
|
3842
|
+
private func awaitInteractiveInput(
|
|
3843
|
+
requestId: String,
|
|
3844
|
+
screenTitle: String,
|
|
3845
|
+
displayTitle: String? = nil,
|
|
3846
|
+
descriptors: [InputDescriptor],
|
|
3847
|
+
responsePath: String,
|
|
3848
|
+
waitTimeout: Double,
|
|
3849
|
+
overrides: inout [String: String],
|
|
3850
|
+
totalWaitSeconds: inout Double,
|
|
3851
|
+
dontAskAgain: inout Bool,
|
|
3852
|
+
interactiveEnabled: inout Bool
|
|
3853
|
+
) -> String {
|
|
3854
|
+
// Clear any stale response left over from a previous request.
|
|
3855
|
+
try? FileManager.default.removeItem(atPath: responsePath)
|
|
3856
|
+
|
|
3857
|
+
let fieldsJson = descriptors.map { d -> String in
|
|
3858
|
+
// Secure fields never carry a default over stdout (it would log the auto-password).
|
|
3859
|
+
let def = d.secure ? "" : defaultInputValue(label: d.label, identifier: d.key, secure: d.secure)
|
|
3860
|
+
return "{\"key\":\"\(escapeJSON(d.key))\",\"label\":\"\(escapeJSON(d.label))\",\"secure\":\(d.secure ? "true" : "false"),\"placeholder\":\"\(escapeJSON(d.placeholder))\",\"default\":\"\(escapeJSON(def))\"}"
|
|
3861
|
+
}.joined(separator: ",")
|
|
3862
|
+
// displayTitle is what the human sees; screenTitle keys the override merge below —
|
|
3863
|
+
// they must stay separate or submitted values would be scoped to the display name.
|
|
3864
|
+
print("OCQA_AWAIT_INPUT:{\"requestId\":\"\(requestId)\",\"screen\":\"\(escapeJSON(displayTitle ?? screenTitle))\",\"fields\":[\(fieldsJson)]}")
|
|
3865
|
+
|
|
3866
|
+
let start = Date()
|
|
3867
|
+
var resolvedAction = "timeout"
|
|
3868
|
+
while Date().timeIntervalSince(start) < waitTimeout {
|
|
3869
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
3870
|
+
guard let data = try? Data(contentsOf: URL(fileURLWithPath: responsePath)),
|
|
3871
|
+
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else {
|
|
3872
|
+
continue
|
|
3873
|
+
}
|
|
3874
|
+
// Only honor a response that targets the current request; drop stale ones.
|
|
3875
|
+
guard (obj["requestId"] as? String) == requestId else {
|
|
3876
|
+
try? FileManager.default.removeItem(atPath: responsePath)
|
|
3877
|
+
continue
|
|
3878
|
+
}
|
|
3879
|
+
let action = (obj["action"] as? String) ?? "defaults"
|
|
3880
|
+
if action == "submit", let values = obj["values"] as? [String: String] {
|
|
3881
|
+
let screen = normalizeKey(screenTitle)
|
|
3882
|
+
for (key, value) in values where !value.isEmpty {
|
|
3883
|
+
overrides["screen:\(screen)|\(key)"] = value
|
|
3884
|
+
}
|
|
3885
|
+
} else if action == "dont_ask" {
|
|
3886
|
+
dontAskAgain = true
|
|
3887
|
+
}
|
|
3888
|
+
resolvedAction = action
|
|
3889
|
+
try? FileManager.default.removeItem(atPath: responsePath)
|
|
3890
|
+
break
|
|
3891
|
+
}
|
|
3892
|
+
if resolvedAction == "timeout" {
|
|
3893
|
+
// Host never answered — assume the UI is gone and stop prompting for the rest of the run.
|
|
3894
|
+
interactiveEnabled = false
|
|
3895
|
+
}
|
|
3896
|
+
totalWaitSeconds += Date().timeIntervalSince(start)
|
|
3897
|
+
print("OCQA_INPUT_RESOLVED:{\"requestId\":\"\(requestId)\",\"action\":\"\(resolvedAction)\"}")
|
|
3898
|
+
return resolvedAction
|
|
3899
|
+
}
|
|
3900
|
+
|
|
3901
|
+
/// In-loop vision escalation. Screenshots the current screen, asks the host (via
|
|
3902
|
+
/// OCQA_VISION_QUERY) for the single best next action, and executes the reply. The model call
|
|
3903
|
+
/// happens HOST-side; the harness only requests and acts. Returns true if it performed an action
|
|
3904
|
+
/// (caller should `continue`), false on "none"/timeout (caller falls through to its bail path).
|
|
3905
|
+
private func visionEscalate(
|
|
3906
|
+
app: XCUIApplication,
|
|
3907
|
+
screenTitle: String,
|
|
3908
|
+
reason: String,
|
|
3909
|
+
actionCount: inout Int,
|
|
3910
|
+
usedCount: inout Int,
|
|
3911
|
+
responsePath: String,
|
|
3912
|
+
imageDir: String,
|
|
3913
|
+
waitTimeout: Double
|
|
3914
|
+
) -> Bool {
|
|
3915
|
+
usedCount += 1
|
|
3916
|
+
let requestId = UUID().uuidString
|
|
3917
|
+
|
|
3918
|
+
// Capture the screen to a host-readable PNG (the sim shares the host filesystem).
|
|
3919
|
+
let dir = imageDir.isEmpty ? NSTemporaryDirectory() : imageDir
|
|
3920
|
+
try? FileManager.default.createDirectory(atPath: dir, withIntermediateDirectories: true)
|
|
3921
|
+
let imagePath = (dir as NSString).appendingPathComponent("vision-\(requestId).png")
|
|
3922
|
+
let pngWritten = (try? app.screenshot().pngRepresentation.write(to: URL(fileURLWithPath: imagePath))) != nil
|
|
3923
|
+
guard pngWritten else { return false }
|
|
3924
|
+
|
|
3925
|
+
print("OCQA_VISION_QUERY:{\"requestId\":\"\(requestId)\",\"screen\":\"\(escapeJSON(screenTitle))\",\"image\":\"\(escapeJSON(imagePath))\",\"reason\":\"\(reason)\",\"step\":\(actionCount)}")
|
|
3926
|
+
|
|
3927
|
+
let decision = awaitVisionDecision(requestId: requestId, responsePath: responsePath, waitTimeout: waitTimeout)
|
|
3928
|
+
let esc = escapeJSON(screenTitle)
|
|
3929
|
+
|
|
3930
|
+
func logAction(_ type: String, _ narrative: String) {
|
|
3931
|
+
actionCount += 1
|
|
3932
|
+
print("OCQA_ACTION:{\"type\":\"\(type)\",\"reason\":\"vision_escalation\",\"step\":\(actionCount),\"screen\":\"\(esc)\",\"narrative\":\"\(escapeJSON(narrative))\"}")
|
|
3933
|
+
}
|
|
3934
|
+
|
|
3935
|
+
switch decision.action {
|
|
3936
|
+
case "tap":
|
|
3937
|
+
let x = min(max(decision.x, 0.0), 1.0)
|
|
3938
|
+
let y = min(max(decision.y, 0.0), 1.0)
|
|
3939
|
+
app.coordinate(withNormalizedOffset: CGVector(dx: x, dy: y)).tap()
|
|
3940
|
+
logAction("tap", "AI vision suggested tapping here to get unstuck.")
|
|
3941
|
+
Thread.sleep(forTimeInterval: 0.7)
|
|
3942
|
+
return true
|
|
3943
|
+
case "swipe_up":
|
|
3944
|
+
app.swipeUp()
|
|
3945
|
+
logAction("swipe", "AI vision suggested swiping up to reveal more content.")
|
|
3946
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
3947
|
+
return true
|
|
3948
|
+
case "swipe_down":
|
|
3949
|
+
app.swipeDown()
|
|
3950
|
+
logAction("swipe", "AI vision suggested swiping down.")
|
|
3951
|
+
Thread.sleep(forTimeInterval: 0.5)
|
|
3952
|
+
return true
|
|
3953
|
+
case "back":
|
|
3954
|
+
_ = tryGoBack()
|
|
3955
|
+
logAction("navigate", "AI vision suggested leaving this dead-end screen.")
|
|
3956
|
+
return true
|
|
3957
|
+
default:
|
|
3958
|
+
return false // "none" / timeout — nothing actionable; caller falls through.
|
|
3959
|
+
}
|
|
3960
|
+
}
|
|
3961
|
+
|
|
3962
|
+
private struct VisionDecision { let action: String; let x: CGFloat; let y: CGFloat }
|
|
3963
|
+
|
|
3964
|
+
/// Polls `responsePath` for the host's vision decision (same file-channel shape as interactive
|
|
3965
|
+
/// input). On timeout returns a "none" decision so the caller falls back to its bail path.
|
|
3966
|
+
private func awaitVisionDecision(requestId: String, responsePath: String, waitTimeout: Double) -> VisionDecision {
|
|
3967
|
+
try? FileManager.default.removeItem(atPath: responsePath)
|
|
3968
|
+
let start = Date()
|
|
3969
|
+
while Date().timeIntervalSince(start) < waitTimeout {
|
|
3970
|
+
Thread.sleep(forTimeInterval: 0.4)
|
|
3971
|
+
guard let data = try? Data(contentsOf: URL(fileURLWithPath: responsePath)),
|
|
3972
|
+
let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
|
3973
|
+
(obj["requestId"] as? String) == requestId else { continue }
|
|
3974
|
+
let action = (obj["action"] as? String) ?? "none"
|
|
3975
|
+
let x = CGFloat((obj["x"] as? Double) ?? 0.5)
|
|
3976
|
+
let y = CGFloat((obj["y"] as? Double) ?? 0.5)
|
|
3977
|
+
try? FileManager.default.removeItem(atPath: responsePath)
|
|
3978
|
+
print("OCQA_VISION_RESOLVED:{\"requestId\":\"\(requestId)\",\"action\":\"\(action)\"}")
|
|
3979
|
+
return VisionDecision(action: action, x: x, y: y)
|
|
3980
|
+
}
|
|
3981
|
+
print("OCQA_VISION_RESOLVED:{\"requestId\":\"\(requestId)\",\"action\":\"timeout\"}")
|
|
3982
|
+
return VisionDecision(action: "none", x: 0.5, y: 0.5)
|
|
3983
|
+
}
|
|
3984
|
+
|
|
3985
|
+
private func detectInputDescriptors(in elements: [SimpleElement]) -> [InputDescriptor] {
|
|
3986
|
+
var seenKeys = Set<String>()
|
|
3987
|
+
var descriptors: [InputDescriptor] = []
|
|
3988
|
+
// Credential-form context: a completely unlabeled text field next to a secure field is
|
|
3989
|
+
// the email/username (the login preamble uses the same inference to decide where to type).
|
|
3990
|
+
let hasSecureSibling = elements.contains { isTextField($0.type) && isSecureTextField($0.type) }
|
|
3991
|
+
|
|
3992
|
+
for element in elements where isTextField(element.type) {
|
|
3993
|
+
let key = inputFieldKey(for: element)
|
|
3994
|
+
if seenKeys.contains(key) { continue }
|
|
3995
|
+
seenKeys.insert(key)
|
|
3996
|
+
|
|
3997
|
+
let inferredLabel = inferFieldLabel(for: element, in: elements)
|
|
3998
|
+
let rawLabel = normalizeVisibleText(element.label)
|
|
3999
|
+
let rawId = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
4000
|
+
var placeholder = element.xcElement?.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
|
|
4001
|
+
let secure = isSecureTextField(element.type) || rawId.lowercased().contains("password")
|
|
4002
|
+
// XCUITest quirk that mislabeled a real production login form: custom-styled fields often
|
|
4003
|
+
// return NO placeholderValue — but an EMPTY field's a11y VALUE is its placeholder
|
|
4004
|
+
// text ("Email address"). Recover it from value when it reads like a label (never
|
|
4005
|
+
// for secure fields, whose value is the ••• mask).
|
|
4006
|
+
if placeholder.isEmpty, !secure {
|
|
4007
|
+
let valueText = normalizeVisibleText(element.value)
|
|
4008
|
+
// Data-shaped values (an email address, a number) mean the field has been TYPED
|
|
4009
|
+
// into — that's content, not a placeholder (measured: the login preamble's typed
|
|
4010
|
+
// email became the field's "label" on a post-typing read).
|
|
4011
|
+
let looksLikeData = valueText.contains("@") || valueText.allSatisfy { $0.isNumber || $0.isPunctuation }
|
|
4012
|
+
if !valueText.isEmpty, !looksLikeData, isLikelyFieldLabel(valueText) { placeholder = valueText }
|
|
4013
|
+
}
|
|
4014
|
+
let fallback = secure ? "Password" : (hasSecureSibling ? "Email or username" : "Field \(descriptors.count + 1)")
|
|
4015
|
+
// A field's OWN accessibility label/placeholder is authoritative; only fall back to a
|
|
4016
|
+
// nearby static text (inferred) when the field exposes neither. Avoids mislabeling e.g.
|
|
4017
|
+
// a web search box as the link sitting above it.
|
|
4018
|
+
let ownLabel = !rawLabel.isEmpty ? rawLabel : placeholder
|
|
4019
|
+
let displayLabel = !ownLabel.isEmpty ? ownLabel : (inferredLabel ?? (!rawId.isEmpty ? rawId : fallback))
|
|
4020
|
+
|
|
4021
|
+
descriptors.append(InputDescriptor(
|
|
4022
|
+
key: key,
|
|
4023
|
+
label: displayLabel,
|
|
4024
|
+
secure: secure,
|
|
4025
|
+
placeholder: placeholder
|
|
4026
|
+
))
|
|
4027
|
+
}
|
|
4028
|
+
|
|
4029
|
+
return descriptors
|
|
4030
|
+
}
|
|
4031
|
+
|
|
4032
|
+
private func inferFieldLabel(for field: SimpleElement, in elements: [SimpleElement]) -> String? {
|
|
4033
|
+
let screenWidth = screenBounds.width > 0 ? screenBounds.width : 390
|
|
4034
|
+
let candidates = elements
|
|
4035
|
+
.filter { isStaticTextType($0.type) }
|
|
4036
|
+
.map { ($0, normalizeVisibleText($0.label)) }
|
|
4037
|
+
.filter { !$0.1.isEmpty && isLikelyFieldLabel($0.1) }
|
|
4038
|
+
|
|
4039
|
+
var best: (label: String, score: CGFloat)?
|
|
4040
|
+
|
|
4041
|
+
for (candidate, text) in candidates {
|
|
4042
|
+
let verticalGap = field.frame.minY - candidate.frame.maxY
|
|
4043
|
+
if verticalGap < -4 || verticalGap > 120 {
|
|
4044
|
+
continue
|
|
4045
|
+
}
|
|
4046
|
+
|
|
4047
|
+
let overlap = max(0, min(field.frame.maxX, candidate.frame.maxX) - max(field.frame.minX, candidate.frame.minX))
|
|
4048
|
+
let minWidth = min(field.frame.width, candidate.frame.width)
|
|
4049
|
+
let overlapRatio = minWidth > 0 ? overlap / minWidth : 0
|
|
4050
|
+
let centerDelta = abs(field.frame.midX - candidate.frame.midX)
|
|
4051
|
+
|
|
4052
|
+
if overlapRatio < 0.2 && centerDelta > field.frame.width * 0.8 {
|
|
4053
|
+
continue
|
|
4054
|
+
}
|
|
4055
|
+
|
|
4056
|
+
var score = verticalGap + centerDelta * 0.08
|
|
4057
|
+
if candidate.frame.width > screenWidth * 0.55 {
|
|
4058
|
+
score += 40
|
|
4059
|
+
}
|
|
4060
|
+
|
|
4061
|
+
if best == nil || score < best!.score {
|
|
4062
|
+
best = (text, score)
|
|
4063
|
+
}
|
|
4064
|
+
}
|
|
4065
|
+
|
|
4066
|
+
return best?.label
|
|
4067
|
+
}
|
|
4068
|
+
|
|
4069
|
+
private func isStaticTextType(_ type: String) -> Bool {
|
|
4070
|
+
type.contains("rawValue: 48") || type.contains("StaticText")
|
|
4071
|
+
}
|
|
4072
|
+
|
|
4073
|
+
private func normalizeVisibleText(_ raw: String) -> String {
|
|
4074
|
+
raw
|
|
4075
|
+
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
4076
|
+
.replacingOccurrences(of: "\n", with: " ")
|
|
4077
|
+
.replacingOccurrences(of: " ", with: " ")
|
|
4078
|
+
}
|
|
4079
|
+
|
|
4080
|
+
private func isLikelyTitleText(_ text: String) -> Bool {
|
|
4081
|
+
let lower = text.lowercased()
|
|
4082
|
+
if text.count < 2 || text.count > 60 { return false }
|
|
4083
|
+
if lower.contains("powered by") { return false }
|
|
4084
|
+
if lower == "optional" { return false }
|
|
4085
|
+
if lower.contains("@") { return false }
|
|
4086
|
+
// React Native / Expo DEBUG overlays (RCTDevLoadingView, dev menu) are not app screens.
|
|
4087
|
+
if lower.contains("connect to metro") || lower.contains("metro to develop")
|
|
4088
|
+
|| lower.contains("reload") && lower.contains("javascript") { return false }
|
|
4089
|
+
// Feed content leaking as a "title": timestamps/metadata start with punctuation or a
|
|
4090
|
+
// symbol ("⸱ 3h"), and machine ids are never titles (observed on a content-feed app).
|
|
4091
|
+
if let first = text.unicodeScalars.first, !CharacterSet.alphanumerics.contains(first) { return false }
|
|
4092
|
+
if isMangledTypeName(text) { return false }
|
|
4093
|
+
return true
|
|
4094
|
+
}
|
|
4095
|
+
|
|
4096
|
+
private func isLikelyFieldLabel(_ text: String) -> Bool {
|
|
4097
|
+
let lower = text.lowercased()
|
|
4098
|
+
if text.count < 2 || text.count > 40 { return false }
|
|
4099
|
+
if lower.contains("welcome") || lower.contains("create account") || lower.contains("sign in") { return false }
|
|
4100
|
+
if lower.contains("powered by") { return false }
|
|
4101
|
+
// Sentence-like text is marketing/prose, not a field label ("Real results." was
|
|
4102
|
+
// inferred as an email field's label on a real login screen). Labels don't end in
|
|
4103
|
+
// sentence punctuation and rarely exceed four words.
|
|
4104
|
+
if text.hasSuffix(".") || text.hasSuffix("!") || text.hasSuffix("?") { return false }
|
|
4105
|
+
if text.split(separator: " ").count > 4 { return false }
|
|
4106
|
+
return true
|
|
4107
|
+
}
|
|
4108
|
+
|
|
4109
|
+
// MARK: - Narration
|
|
4110
|
+
|
|
4111
|
+
/// True for accessibility labels that are really SF Symbol identifiers (e.g. "hand.wave.fill")
|
|
4112
|
+
/// rather than human words — these leak when an icon control has no title, and read as noise.
|
|
4113
|
+
private func isSymbolLikeLabel(_ text: String) -> Bool {
|
|
4114
|
+
let t = text.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
4115
|
+
guard !t.isEmpty else { return false }
|
|
4116
|
+
// Private-use-area glyphs (SF Symbols rendered as text, e.g. "") — an icon, not a
|
|
4117
|
+
// human label (observed flagged as a "dead control" on a content-feed app).
|
|
4118
|
+
let puaScalars = t.unicodeScalars.filter {
|
|
4119
|
+
(0xE000...0xF8FF).contains($0.value) || (0xF0000...0xFFFFD).contains($0.value) || (0x100000...0x10FFFD).contains($0.value)
|
|
4120
|
+
}.count
|
|
4121
|
+
if puaScalars * 2 >= t.unicodeScalars.count { return true }
|
|
4122
|
+
guard !t.contains(" "), t.contains(".") else { return false }
|
|
4123
|
+
return t.range(of: "^[a-z0-9]+(\\.[a-z0-9]+)+$", options: .regularExpression) != nil
|
|
4124
|
+
}
|
|
4125
|
+
|
|
4126
|
+
/// Ordered, de-duplicated, human-meaningful on-screen text, top-to-bottom. This is the
|
|
4127
|
+
/// actual wording a user reads — the single richest deterministic signal about a screen.
|
|
4128
|
+
/// A fuller text inventory for the vision reviewer's accessibility context — the FULL a11y label
|
|
4129
|
+
/// of each static-text element (up to 120 chars, so long subtitles that VISUALLY truncate are
|
|
4130
|
+
/// still captured in full), sorted top-to-bottom, bounded. Distinct from collectVisibleTexts
|
|
4131
|
+
/// (capped at 60 for narration): the point here is to let the vision model confirm that text it
|
|
4132
|
+
/// sees cut off actually exists in full (i.e. is only scrolled off-screen, not clipped).
|
|
4133
|
+
private func visionTextInventory(_ elements: [SimpleElement], limit: Int = 30) -> [String] {
|
|
4134
|
+
var seen = Set<String>()
|
|
4135
|
+
var result: [(text: String, y: CGFloat)] = []
|
|
4136
|
+
for el in elements where isStaticTextType(el.type) {
|
|
4137
|
+
let text = normalizeVisibleText(el.label)
|
|
4138
|
+
guard text.count >= 2, text.count <= 120 else { continue }
|
|
4139
|
+
let lower = text.lowercased()
|
|
4140
|
+
if lower == "optional" { continue }
|
|
4141
|
+
if !text.contains(where: { $0.isLetter || $0.isNumber }) { continue }
|
|
4142
|
+
if seen.contains(lower) { continue }
|
|
4143
|
+
seen.insert(lower)
|
|
4144
|
+
result.append((text, el.frame.minY))
|
|
4145
|
+
}
|
|
4146
|
+
return Array(result.sorted { $0.y < $1.y }.map { $0.text }.prefix(limit))
|
|
4147
|
+
}
|
|
4148
|
+
|
|
4149
|
+
private func collectVisibleTexts(_ elements: [SimpleElement], limit: Int = 10) -> [String] {
|
|
4150
|
+
var seen = Set<String>()
|
|
4151
|
+
var result: [(text: String, y: CGFloat)] = []
|
|
4152
|
+
for el in elements where isStaticTextType(el.type) {
|
|
4153
|
+
let text = normalizeVisibleText(el.label)
|
|
4154
|
+
guard text.count >= 2, text.count <= 60 else { continue }
|
|
4155
|
+
let lower = text.lowercased()
|
|
4156
|
+
if lower == "optional" { continue }
|
|
4157
|
+
// Skip pure punctuation / separators with no letters or digits.
|
|
4158
|
+
if !text.contains(where: { $0.isLetter || $0.isNumber }) { continue }
|
|
4159
|
+
if seen.contains(lower) { continue }
|
|
4160
|
+
seen.insert(lower)
|
|
4161
|
+
result.append((text, el.frame.minY))
|
|
4162
|
+
}
|
|
4163
|
+
return Array(result.sorted { $0.y < $1.y }.map { $0.text }.prefix(limit))
|
|
4164
|
+
}
|
|
4165
|
+
|
|
4166
|
+
/// Returns the first visible error/failure message on screen, or nil. Curated, high-precision
|
|
4167
|
+
/// phrases only — empty states ("No posts yet") are deliberately NOT errors.
|
|
4168
|
+
private func detectErrorSurface(_ elements: [SimpleElement]) -> String? {
|
|
4169
|
+
let errorPhrases = [
|
|
4170
|
+
"something went wrong", "an error occurred", "an error has occurred", "an unexpected error",
|
|
4171
|
+
"failed to load", "couldn't load", "could not load", "unable to load", "unable to connect",
|
|
4172
|
+
"no internet", "no connection", "not connected", "you're offline", "you are offline",
|
|
4173
|
+
"connection lost", "connection error", "network error",
|
|
4174
|
+
"request failed", "request timed out", "server error", "internal server error",
|
|
4175
|
+
"please try again", "try again later",
|
|
4176
|
+
// Validation / auth failures surfaced to the user (found live: a raw Firebase
|
|
4177
|
+
// "credential is malformed or has expired" and a "Passwords do not match." both
|
|
4178
|
+
// slipped past the network-centric list above).
|
|
4179
|
+
"do not match", "does not match", "malformed",
|
|
4180
|
+
"invalid email", "invalid password", "incorrect password", "incorrect email",
|
|
4181
|
+
"already in use", "authentication failed", "auth failed"
|
|
4182
|
+
]
|
|
4183
|
+
for el in elements where isStaticTextType(el.type) {
|
|
4184
|
+
let text = normalizeVisibleText(el.label)
|
|
4185
|
+
let lower = text.lowercased()
|
|
4186
|
+
guard lower.count >= 4, lower.count <= 140 else { continue }
|
|
4187
|
+
if errorPhrases.contains(where: { lower.contains($0) }) {
|
|
4188
|
+
return text
|
|
4189
|
+
}
|
|
4190
|
+
}
|
|
4191
|
+
return nil
|
|
4192
|
+
}
|
|
4193
|
+
|
|
4194
|
+
/// Human description of an unlabeled control by kind + position, e.g. "an unlabeled row
|
|
4195
|
+
/// near the top" — so the chat still says something useful instead of "an element".
|
|
4196
|
+
private func describeAnonymous(_ element: SimpleElement) -> String {
|
|
4197
|
+
let kind: String
|
|
4198
|
+
if element.type.contains("Cell") || element.type.contains("rawValue: 75") { kind = "row" }
|
|
4199
|
+
else if element.type.contains("Switch") || element.type.contains("Toggle") || element.type.contains("rawValue: 40") { kind = "toggle" }
|
|
4200
|
+
else if element.type.contains("Link") || element.type.contains("rawValue: 39") { kind = "link" }
|
|
4201
|
+
else if element.type.contains("Button") || element.type.contains("rawValue: 9") { kind = "button" }
|
|
4202
|
+
else if element.type.contains("Image") { kind = "image" }
|
|
4203
|
+
else if isTextField(element.type) { kind = "field" }
|
|
4204
|
+
else { kind = "control" }
|
|
4205
|
+
|
|
4206
|
+
let h = screenBounds.height > 0 ? screenBounds.height : 900
|
|
4207
|
+
let w = screenBounds.width > 0 ? screenBounds.width : 390
|
|
4208
|
+
let vy = element.frame.midY / h
|
|
4209
|
+
let vx = element.frame.midX / w
|
|
4210
|
+
let vBand = vy < 0.33 ? "top" : (vy < 0.66 ? "middle" : "bottom")
|
|
4211
|
+
let hBand = vx < 0.33 ? "-left" : (vx < 0.66 ? "" : "-right")
|
|
4212
|
+
return "an unlabeled \(kind) at the \(vBand)\(hBand) of the screen"
|
|
4213
|
+
}
|
|
4214
|
+
|
|
4215
|
+
/// Human caption for the harness's recovery / navigation actions — the ones that aren't a
|
|
4216
|
+
/// direct tap/type on a labeled control. Without these the chat falls back to raw targets
|
|
4217
|
+
/// like "tab_bar_pos_0", which tell the user nothing.
|
|
4218
|
+
private func recoveryNarrative(_ kind: String, screen: String, to dest: String? = nil) -> String {
|
|
4219
|
+
let here = screen.isEmpty || screen == "Unknown" ? "this screen" : "the \(screen) screen"
|
|
4220
|
+
let there: String = {
|
|
4221
|
+
if let d = dest, !d.isEmpty, d != "Unknown" { return "the \(d) screen" }
|
|
4222
|
+
return "the previous screen"
|
|
4223
|
+
}()
|
|
4224
|
+
switch kind {
|
|
4225
|
+
case "tab_rotation", "tab_rotation_warmup":
|
|
4226
|
+
return "Switching to another tab to explore a different section of the app."
|
|
4227
|
+
case "keyboard_dismiss":
|
|
4228
|
+
return "Dismissing the keyboard to reach the button underneath it."
|
|
4229
|
+
case "dead_end_tab_escape", "blind_tab_escape":
|
|
4230
|
+
return "Nothing left to interact with on \(here) — tapping the tab bar to find new screens."
|
|
4231
|
+
case "back_dead_end", "back_exhausted":
|
|
4232
|
+
return "Finished with \(here) — going back to \(there)."
|
|
4233
|
+
case "back_same_title":
|
|
4234
|
+
return "Going back from \(here) to look for unexplored areas."
|
|
4235
|
+
case "back_failed":
|
|
4236
|
+
return "Tried to go back from \(here) but stayed put — looking for another way out."
|
|
4237
|
+
case "swipe_back":
|
|
4238
|
+
return "Swiping back from \(here) to \(there)."
|
|
4239
|
+
case "scroll_reveal":
|
|
4240
|
+
return "Scrolling to reveal more content on \(here)."
|
|
4241
|
+
case "scroll_back":
|
|
4242
|
+
return "Scrolling back up on \(here)."
|
|
4243
|
+
case "center_probe":
|
|
4244
|
+
return "Probing an unlabeled area of \(here) for hidden controls."
|
|
4245
|
+
case "carousel_probe":
|
|
4246
|
+
return "Swiping sideways on \(here) in case it's a carousel."
|
|
4247
|
+
case "drawer_probe":
|
|
4248
|
+
return "Edge-swiping on \(here) to check for a side menu."
|
|
4249
|
+
case "swipe_dismiss":
|
|
4250
|
+
return "Swiping down to dismiss a sheet on \(here)."
|
|
4251
|
+
default:
|
|
4252
|
+
return "Exploring \(here)."
|
|
4253
|
+
}
|
|
4254
|
+
}
|
|
4255
|
+
|
|
4256
|
+
/// Emit a "navigation trap" finding when exploration genuinely can't leave a screen by any
|
|
4257
|
+
/// means — a real "user could get stuck here" problem. De-duped per screen.
|
|
4258
|
+
private func emitNavigationTrap(titleStr: String, escapedTitle: String, step: Int,
|
|
4259
|
+
reported: inout Set<String>,
|
|
4260
|
+
issues: inout [(type: String, severity: String, title: String, desc: String)]) {
|
|
4261
|
+
let key = "trap:\(titleStr)"
|
|
4262
|
+
guard !reported.contains(key) else { return }
|
|
4263
|
+
reported.insert(key)
|
|
4264
|
+
let title = "Stuck on '\(titleStr)' — no way forward or back"
|
|
4265
|
+
issues.append((type: "navigation_trap", severity: "medium", title: title,
|
|
4266
|
+
desc: "Exploration could not navigate away from '\(titleStr)' by any means (no controls, can't go back, can't dismiss) — users may get trapped here."))
|
|
4267
|
+
print("OCQA_ISSUE:{\"type\":\"navigation_trap\",\"severity\":\"medium\",\"title\":\"\(escapeJSON(title))\",\"screen\":\"\(escapedTitle)\",\"step\":\(step)}")
|
|
4268
|
+
}
|
|
4269
|
+
|
|
4270
|
+
private func classifyScreenRole(
|
|
4271
|
+
title: String,
|
|
4272
|
+
elements: [SimpleElement],
|
|
4273
|
+
inputs: [InputDescriptor],
|
|
4274
|
+
interactable: [SimpleElement]
|
|
4275
|
+
) -> String {
|
|
4276
|
+
let titleLower = title.lowercased()
|
|
4277
|
+
let buttonLabels = interactable
|
|
4278
|
+
.filter { $0.type.contains("Button") || $0.type.contains("rawValue: 9") }
|
|
4279
|
+
.map { ($0.label + " " + $0.identifier).lowercased() }
|
|
4280
|
+
let allText = (buttonLabels + [titleLower]).joined(separator: " ")
|
|
4281
|
+
let hasSecure = inputs.contains(where: { $0.secure })
|
|
4282
|
+
let hasEmail = inputs.contains { $0.label.lowercased().contains("email") || $0.placeholder.lowercased().contains("email") || $0.key.contains("email") }
|
|
4283
|
+
|
|
4284
|
+
if hasSecure && (hasEmail || allText.contains("sign in") || allText.contains("log in") || allText.contains("login")) {
|
|
4285
|
+
return "login"
|
|
4286
|
+
}
|
|
4287
|
+
if hasSecure && (allText.contains("sign up") || allText.contains("create account") || allText.contains("register")) {
|
|
4288
|
+
return "signup"
|
|
4289
|
+
}
|
|
4290
|
+
if titleLower.contains("settings") || allText.contains("preferences") {
|
|
4291
|
+
return "settings"
|
|
4292
|
+
}
|
|
4293
|
+
if titleLower.contains("profile") || allText.contains("edit profile") {
|
|
4294
|
+
return "profile"
|
|
4295
|
+
}
|
|
4296
|
+
if titleLower.contains("welcome") || allText.contains("get started") || allText.contains("continue") && inputs.isEmpty && interactable.count <= 4 {
|
|
4297
|
+
return "onboarding"
|
|
4298
|
+
}
|
|
4299
|
+
let cellCount = elements.filter { $0.type.contains("Cell") || $0.type.contains("rawValue: 75") }.count
|
|
4300
|
+
if cellCount >= 4 {
|
|
4301
|
+
return "list"
|
|
4302
|
+
}
|
|
4303
|
+
if !inputs.isEmpty {
|
|
4304
|
+
return "form"
|
|
4305
|
+
}
|
|
4306
|
+
if cellCount >= 1 {
|
|
4307
|
+
return "detail"
|
|
4308
|
+
}
|
|
4309
|
+
return "screen"
|
|
4310
|
+
}
|
|
4311
|
+
|
|
4312
|
+
private func describeScreen(
|
|
4313
|
+
title: String,
|
|
4314
|
+
role: String,
|
|
4315
|
+
elements: [SimpleElement],
|
|
4316
|
+
inputs: [InputDescriptor],
|
|
4317
|
+
interactable: [SimpleElement]
|
|
4318
|
+
) -> String {
|
|
4319
|
+
let safeTitle = title.isEmpty || title == "Unknown" ? "this screen" : "the \(title) screen"
|
|
4320
|
+
let buttons = interactable
|
|
4321
|
+
.filter { $0.type.contains("Button") || $0.type.contains("rawValue: 9") }
|
|
4322
|
+
.compactMap { btn -> String? in
|
|
4323
|
+
let raw = btn.label.isEmpty ? btn.identifier : btn.label
|
|
4324
|
+
let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
4325
|
+
guard !trimmed.isEmpty, trimmed.count <= 30, !isSymbolLikeLabel(trimmed) else { return nil }
|
|
4326
|
+
return trimmed
|
|
4327
|
+
}
|
|
4328
|
+
let dedupedButtons = Array(NSOrderedSet(array: buttons)) as? [String] ?? []
|
|
4329
|
+
let primaryButtons = Array(dedupedButtons.prefix(4))
|
|
4330
|
+
let cellCount = elements.filter { $0.type.contains("Cell") || $0.type.contains("rawValue: 75") }.count
|
|
4331
|
+
let fieldNames = inputs.prefix(4).map { $0.label }
|
|
4332
|
+
|
|
4333
|
+
var pieces: [String] = []
|
|
4334
|
+
switch role {
|
|
4335
|
+
case "login":
|
|
4336
|
+
pieces.append("Looking at \(safeTitle), which asks for credentials.")
|
|
4337
|
+
case "signup":
|
|
4338
|
+
pieces.append("Looking at \(safeTitle), a sign-up form.")
|
|
4339
|
+
case "settings":
|
|
4340
|
+
pieces.append("Looking at \(safeTitle), exposing app preferences.")
|
|
4341
|
+
case "profile":
|
|
4342
|
+
pieces.append("Looking at \(safeTitle), showing the user's profile.")
|
|
4343
|
+
case "onboarding":
|
|
4344
|
+
pieces.append("Looking at \(safeTitle), an onboarding step.")
|
|
4345
|
+
case "list":
|
|
4346
|
+
pieces.append("Looking at \(safeTitle), a list of \(cellCount) items.")
|
|
4347
|
+
case "form":
|
|
4348
|
+
pieces.append("Looking at \(safeTitle), a form to fill out.")
|
|
4349
|
+
case "detail":
|
|
4350
|
+
pieces.append("Looking at \(safeTitle), a detail view.")
|
|
4351
|
+
default:
|
|
4352
|
+
pieces.append("Looking at \(safeTitle).")
|
|
4353
|
+
}
|
|
4354
|
+
|
|
4355
|
+
// The actual words on screen — exclude text already echoed as the title, a field
|
|
4356
|
+
// label, or a button label so we don't repeat ourselves.
|
|
4357
|
+
var exclude = Set<String>([title.lowercased()])
|
|
4358
|
+
fieldNames.forEach { exclude.insert($0.lowercased()) }
|
|
4359
|
+
dedupedButtons.forEach { exclude.insert($0.lowercased()) }
|
|
4360
|
+
let content = collectVisibleTexts(elements).filter { !exclude.contains($0.lowercased()) }
|
|
4361
|
+
if !content.isEmpty {
|
|
4362
|
+
let shown = content.prefix(5).map { "“\($0)”" }.joined(separator: ", ")
|
|
4363
|
+
pieces.append("On screen: \(shown).")
|
|
4364
|
+
}
|
|
4365
|
+
|
|
4366
|
+
if !fieldNames.isEmpty {
|
|
4367
|
+
pieces.append("Fields: \(fieldNames.joined(separator: ", ")).")
|
|
4368
|
+
}
|
|
4369
|
+
if !primaryButtons.isEmpty {
|
|
4370
|
+
pieces.append("Actions: \(primaryButtons.joined(separator: ", ")).")
|
|
4371
|
+
}
|
|
4372
|
+
|
|
4373
|
+
// Switch / toggle states (on/off) — deterministic via the element's value, and
|
|
4374
|
+
// genuinely useful QA signal.
|
|
4375
|
+
let toggles = interactable
|
|
4376
|
+
.filter { $0.type.contains("Switch") || $0.type.contains("Toggle") || $0.type.contains("rawValue: 40") }
|
|
4377
|
+
.compactMap { sw -> String? in
|
|
4378
|
+
let name = (sw.label.isEmpty ? sw.identifier : sw.label).trimmingCharacters(in: .whitespacesAndNewlines)
|
|
4379
|
+
guard !name.isEmpty, name.count <= 30 else { return nil }
|
|
4380
|
+
switch sw.value {
|
|
4381
|
+
case "1": return "\(name) (on)"
|
|
4382
|
+
case "0": return "\(name) (off)"
|
|
4383
|
+
default: return name
|
|
4384
|
+
}
|
|
4385
|
+
}
|
|
4386
|
+
if !toggles.isEmpty {
|
|
4387
|
+
pieces.append("Toggles: \(toggles.prefix(3).joined(separator: ", ")).")
|
|
4388
|
+
}
|
|
4389
|
+
|
|
4390
|
+
if cellCount > 0 && role != "list" {
|
|
4391
|
+
pieces.append("\(cellCount) tappable rows.")
|
|
4392
|
+
}
|
|
4393
|
+
|
|
4394
|
+
return pieces.joined(separator: " ")
|
|
4395
|
+
}
|
|
4396
|
+
|
|
4397
|
+
private func narrate(
|
|
4398
|
+
action: String,
|
|
4399
|
+
target: SimpleElement,
|
|
4400
|
+
screenTitle: String,
|
|
4401
|
+
actionDesc: String
|
|
4402
|
+
) -> String {
|
|
4403
|
+
// Treat SF Symbol identifiers as "no name" so we describe the control by kind/position
|
|
4404
|
+
// instead of saying e.g. "Tapping the hand.wave.fill button" or "the chevron.forward control".
|
|
4405
|
+
let rawLabel = target.label.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
4406
|
+
let rawIdentifier = target.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
4407
|
+
let label = isSymbolLikeLabel(rawLabel) ? "" : rawLabel
|
|
4408
|
+
let identifier = isSymbolLikeLabel(rawIdentifier) ? "" : rawIdentifier
|
|
4409
|
+
let hasName = (!label.isEmpty && label.count <= 40) || (!identifier.isEmpty && identifier.count <= 40)
|
|
4410
|
+
let displayName = (!label.isEmpty && label.count <= 40) ? label : identifier
|
|
4411
|
+
let screen = screenTitle.isEmpty || screenTitle == "Unknown" ? "the current screen" : "the \(screenTitle) screen"
|
|
4412
|
+
|
|
4413
|
+
let elementKind: String
|
|
4414
|
+
if target.type.contains("Switch") || target.type.contains("Toggle") || target.type.contains("rawValue: 40") {
|
|
4415
|
+
elementKind = "toggle"
|
|
4416
|
+
} else if target.type.contains("Button") || target.type.contains("rawValue: 9") {
|
|
4417
|
+
elementKind = "button"
|
|
4418
|
+
} else if target.type.contains("Link") || target.type.contains("rawValue: 39") {
|
|
4419
|
+
elementKind = "link"
|
|
4420
|
+
} else if target.type.contains("Cell") || target.type.contains("rawValue: 75") {
|
|
4421
|
+
elementKind = "row"
|
|
4422
|
+
} else if target.type.contains("Tab") || target.type.contains("rawValue: 54") {
|
|
4423
|
+
elementKind = "tab"
|
|
4424
|
+
} else if isTextField(target.type) {
|
|
4425
|
+
elementKind = "field"
|
|
4426
|
+
} else if target.type.contains("Image") || target.type.contains("rawValue: 43") {
|
|
4427
|
+
elementKind = "image"
|
|
4428
|
+
} else {
|
|
4429
|
+
elementKind = "control"
|
|
4430
|
+
}
|
|
4431
|
+
|
|
4432
|
+
if action == "type" {
|
|
4433
|
+
// Extract typed value if present in actionDesc like type(name, "value")
|
|
4434
|
+
let typed: String = {
|
|
4435
|
+
if let range = actionDesc.range(of: "\""),
|
|
4436
|
+
let endRange = actionDesc.range(of: "\"", options: .backwards),
|
|
4437
|
+
range.lowerBound < endRange.lowerBound {
|
|
4438
|
+
return String(actionDesc[actionDesc.index(after: range.lowerBound)..<endRange.lowerBound])
|
|
4439
|
+
}
|
|
4440
|
+
return ""
|
|
4441
|
+
}()
|
|
4442
|
+
let suffix = lastTypedWasOverride ? " (your saved value)" : ""
|
|
4443
|
+
let into = hasName ? "the \(displayName) field" : describeAnonymous(target)
|
|
4444
|
+
if typed.isEmpty {
|
|
4445
|
+
return "Typing into \(into) on \(screen)."
|
|
4446
|
+
}
|
|
4447
|
+
// Mask secure values so passwords never appear in the chat transcript
|
|
4448
|
+
let shown = lastTypedSecure ? String(repeating: "•", count: min(8, max(4, typed.count))) : typed
|
|
4449
|
+
return "Typing “\(shown)”\(suffix) into \(into) on \(screen)."
|
|
4450
|
+
}
|
|
4451
|
+
|
|
4452
|
+
// For toggles, surface the state we're flipping from when we have it.
|
|
4453
|
+
if elementKind == "toggle", hasName {
|
|
4454
|
+
switch target.value {
|
|
4455
|
+
case "1": return "Turning off the \(displayName) toggle on \(screen)."
|
|
4456
|
+
case "0": return "Turning on the \(displayName) toggle on \(screen)."
|
|
4457
|
+
default: break
|
|
4458
|
+
}
|
|
4459
|
+
}
|
|
4460
|
+
|
|
4461
|
+
if hasName {
|
|
4462
|
+
return "Tapping the \(displayName) \(elementKind) on \(screen)."
|
|
4463
|
+
}
|
|
4464
|
+
return "Tapping \(describeAnonymous(target)) on \(screen)."
|
|
4465
|
+
}
|
|
4466
|
+
|
|
4467
|
+
/// Friendly role for the raw XCUIElementType so a client can tell a password box (secureField)
|
|
4468
|
+
/// from an email box (textField) without decoding rawValue numbers.
|
|
4469
|
+
private func elementRole(_ type: String) -> String {
|
|
4470
|
+
if type.contains("rawValue: 9)") { return "button" }
|
|
4471
|
+
if type.contains("rawValue: 49)") { return "textField" }
|
|
4472
|
+
if type.contains("rawValue: 50)") { return "secureField" }
|
|
4473
|
+
if type.contains("rawValue: 52)") { return "textView" }
|
|
4474
|
+
if type.contains("rawValue: 48)") { return "text" }
|
|
4475
|
+
if type.contains("rawValue: 75)") { return "cell" }
|
|
4476
|
+
if type.contains("rawValue: 12)") { return "image" }
|
|
4477
|
+
if type.contains("rawValue: 10)") { return "link" }
|
|
4478
|
+
if type.contains("rawValue: 41)") { return "switch" }
|
|
4479
|
+
return "other"
|
|
4480
|
+
}
|
|
4481
|
+
|
|
4482
|
+
private func emitUITree(_ state: (title: String?, elements: [SimpleElement])) {
|
|
4483
|
+
var json = "{\"screenTitle\":\"\(escapeJSON(state.title ?? "Unknown"))\",\"elements\":["
|
|
4484
|
+
let arr = state.elements.prefix(100).map { el -> String in
|
|
4485
|
+
// Every string field must be escaped — apps with multi-line labels ("Active\nClients")
|
|
4486
|
+
// otherwise inject raw newlines/quotes and make the whole tree invalid JSON → 0 elements.
|
|
4487
|
+
let role = elementRole(el.type)
|
|
4488
|
+
// Values + placeholders let the client SEE what a field contains (a mis-typed value is
|
|
4489
|
+
// visible immediately) and name placeholder-only fields. Secure values stay masked.
|
|
4490
|
+
let ph = String((el.xcElement?.placeholderValue ?? "").prefix(40))
|
|
4491
|
+
// Secure values: length-preserving dots (content-free, capped) — a double-typed
|
|
4492
|
+
// password is then VISIBLE as 16 dots where 8 were expected.
|
|
4493
|
+
let val = role == "secureField"
|
|
4494
|
+
? (el.value.isEmpty ? "" : String(repeating: "•", count: min(el.value.count, 24)))
|
|
4495
|
+
: String(el.value.prefix(60))
|
|
4496
|
+
return "{\"type\":\"\(el.type)\",\"role\":\"\(role)\",\"id\":\"\(escapeJSON(el.identifier))\",\"label\":\"\(escapeJSON(el.label))\",\"value\":\"\(escapeJSON(val))\",\"placeholder\":\"\(escapeJSON(ph))\",\"enabled\":\(el.isEnabled),\"hittable\":\(el.isHittable),\"x\":\(Int(el.frame.midX)),\"y\":\(Int(el.frame.midY)),\"w\":\(Int(el.frame.width)),\"h\":\(Int(el.frame.height))}"
|
|
4497
|
+
}
|
|
4498
|
+
json += arr.joined(separator: ",")
|
|
4499
|
+
json += "]}"
|
|
4500
|
+
print("OCQA_UITREE_START")
|
|
4501
|
+
print(json)
|
|
4502
|
+
print("OCQA_UITREE_END")
|
|
4503
|
+
}
|
|
4504
|
+
|
|
4505
|
+
private func buildAppState(elements: [SimpleElement]) -> (title: String?, elements: [SimpleElement]) {
|
|
4506
|
+
return (detectTitle(elements), elements)
|
|
4507
|
+
}
|
|
4508
|
+
|
|
4509
|
+
private func emitProgress(action: Int, maxActions: Int, states: Int) {
|
|
4510
|
+
print("OCQA_PROGRESS:{\"action\":\(action),\"max\":\(maxActions),\"states\":\(states)}")
|
|
4511
|
+
}
|
|
4512
|
+
|
|
4513
|
+
// Replace existing field contents to avoid repeatedly appending test text.
|
|
4514
|
+
private func replaceText(on element: XCUIElement, with text: String) {
|
|
4515
|
+
element.tap()
|
|
4516
|
+
|
|
4517
|
+
if let existing = element.value as? String,
|
|
4518
|
+
!existing.isEmpty,
|
|
4519
|
+
existing.lowercased() != "optional" {
|
|
4520
|
+
let deleteString = String(repeating: XCUIKeyboardKey.delete.rawValue, count: existing.count)
|
|
4521
|
+
element.typeText(deleteString)
|
|
4522
|
+
}
|
|
4523
|
+
|
|
4524
|
+
element.typeText(text)
|
|
4525
|
+
}
|
|
4526
|
+
}
|