@aarwitz/tapp 0.17.0-rc.1 → 0.17.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Harness/OCQAHarnessUITests/ExplorerTests.swift +62 -22
- package/bin/tapp.js +68 -10
- package/mcp-server/src/android-driver.js +15 -2
- package/mcp-server/src/android-explorer.js +54 -12
- package/mcp-server/src/ci-report.js +3 -3
- package/mcp-server/src/html-report.js +16 -2
- package/mcp-server/src/index.js +43 -22
- package/mcp-server/src/pr-selection.js +4 -3
- package/mcp-server/src/report.js +64 -15
- package/mcp-server/src/web-explorer.js +1 -1
- package/package.json +2 -2
- package/scripts/ci-gate.sh +6 -5
- package/scripts/platform-gate.js +11 -5
- package/scripts/quick-capture.sh +71 -37
|
@@ -946,6 +946,7 @@ class ExplorerTests: XCTestCase {
|
|
|
946
946
|
var elementScreenPresence: [String: Set<String>] = [:]
|
|
947
947
|
var totalDistinctStates = 0
|
|
948
948
|
var recentStateHashes: [String] = []
|
|
949
|
+
var recentScreenTitles: [String] = []
|
|
949
950
|
var actionsSinceNewState = 0
|
|
950
951
|
var sameScreenStreak = 0
|
|
951
952
|
var screenTextEntryCount: [String: Int] = [:]
|
|
@@ -1238,6 +1239,15 @@ class ExplorerTests: XCTestCase {
|
|
|
1238
1239
|
let initialControlsJson = mapControlsJSON(initialElements)
|
|
1239
1240
|
print("OCQA_STATE:{\"screen\":\"\(escapeJSON(initialTitle))\",\"hash\":\"\(computeHash(initialElements))\",\"elements\":\(initialElements.count),\"action\":0,\"role\":\"\(escapeJSON(initialRole))\",\"summary\":\"\(escapeJSON(initialSummary))\",\"settled\":\(isScreenSettled() ? "true" : "false"),\"atext\":[\(initialAtext)],\"inputs\":[\(initialInputJson)],\"controls\":[\(initialControlsJson)]}")
|
|
1240
1241
|
|
|
1242
|
+
// The launch surface is real evidence even when root normalization immediately dismisses
|
|
1243
|
+
// it. Attach it before the first action so the HTML report's "every screen explored" claim
|
|
1244
|
+
// includes onboarding/login sheets rather than beginning at the post-dismiss destination.
|
|
1245
|
+
let initialScreenshot = app.screenshot()
|
|
1246
|
+
let initialAttachment = XCTAttachment(screenshot: initialScreenshot)
|
|
1247
|
+
initialAttachment.name = "state_0_\(initialTitle.replacingOccurrences(of: " ", with: "_"))"
|
|
1248
|
+
initialAttachment.lifetime = .keepAlways
|
|
1249
|
+
add(initialAttachment)
|
|
1250
|
+
|
|
1241
1251
|
navigateToRootScreen(actionCount: &actionCount)
|
|
1242
1252
|
|
|
1243
1253
|
// The first state is the true customer launch surface. Directed replay begins after
|
|
@@ -1386,8 +1396,10 @@ class ExplorerTests: XCTestCase {
|
|
|
1386
1396
|
}
|
|
1387
1397
|
|
|
1388
1398
|
recentStateHashes.append(stateHash)
|
|
1399
|
+
recentScreenTitles.append(titleStr)
|
|
1389
1400
|
if recentStateHashes.count > 12 {
|
|
1390
1401
|
recentStateHashes.removeFirst(recentStateHashes.count - 12)
|
|
1402
|
+
recentScreenTitles.removeFirst(recentScreenTitles.count - 12)
|
|
1391
1403
|
}
|
|
1392
1404
|
|
|
1393
1405
|
if previousStateHash == stateHash {
|
|
@@ -1474,10 +1486,19 @@ class ExplorerTests: XCTestCase {
|
|
|
1474
1486
|
// is the fuller a11y text inventory that grounds the vision reviewer (see visionTextInventory).
|
|
1475
1487
|
let escapedTitle = escapeJSON(titleStr)
|
|
1476
1488
|
let settled = isScreenSettled()
|
|
1477
|
-
let
|
|
1489
|
+
let visibleTextInventory = visionTextInventory(elements)
|
|
1490
|
+
let atextJson = visibleTextInventory.map { "\"\(escapeJSON($0))\"" }.joined(separator: ",")
|
|
1478
1491
|
let controlsJson = mapControlsJSON(elements)
|
|
1479
1492
|
print("OCQA_STATE:{\"screen\":\"\(escapedTitle)\",\"hash\":\"\(stateHash)\",\"elements\":\(elements.count),\"action\":\(actionCount),\"role\":\"\(escapeJSON(screenRole))\",\"summary\":\"\(escapeJSON(screenSummary))\",\"settled\":\(settled ? "true" : "false"),\"atext\":[\(atextJson)],\"inputs\":[\(inputJsonArray)],\"controls\":[\(controlsJson)]}")
|
|
1480
1493
|
|
|
1494
|
+
// Signing out after a successful login is a completed auth cycle, not a navigation
|
|
1495
|
+
// trap and not lost-form-state. Stop cleanly instead of probing the root login screen
|
|
1496
|
+
// for an impossible back path or expecting credentials to persist after logout.
|
|
1497
|
+
if authSucceeded && detectedInputs.contains(where: { $0.secure }) {
|
|
1498
|
+
print("OCQA_STATE:auth_cycle_complete screen=\(escapedTitle) step=\(actionCount)")
|
|
1499
|
+
break
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1481
1502
|
// ---- Persistence probe: on a fresh RE-ARRIVAL at a screen, fields we previously
|
|
1482
1503
|
// typed into (and verified visible in the a11y value) should still hold their value.
|
|
1483
1504
|
// An empty field here means the entered state was silently lost on navigation —
|
|
@@ -1591,14 +1612,14 @@ class ExplorerTests: XCTestCase {
|
|
|
1591
1612
|
// content-feed app: post detail + replies-loading spinner flagged app_hang HIGH).
|
|
1592
1613
|
let visibleTextCount = elements.filter { isStaticTextType($0.type) && normalizeVisibleText($0.label).count >= 3 }.count
|
|
1593
1614
|
if screenVisitCount[titleStr] ?? 0 <= 1, visibleTextCount <= 4,
|
|
1594
|
-
|
|
1615
|
+
hasIndeterminateLoadingIndicator() {
|
|
1595
1616
|
let loadingKey = "loading:\(titleStr)"
|
|
1596
1617
|
if !reportedIssueKeys.contains(loadingKey) {
|
|
1597
1618
|
var resolved = false
|
|
1598
1619
|
let deadline = Date().addingTimeInterval(8.0)
|
|
1599
1620
|
while Date() < deadline {
|
|
1600
1621
|
Thread.sleep(forTimeInterval: 1.0)
|
|
1601
|
-
if !(
|
|
1622
|
+
if !hasIndeterminateLoadingIndicator() {
|
|
1602
1623
|
resolved = true
|
|
1603
1624
|
break
|
|
1604
1625
|
}
|
|
@@ -1658,7 +1679,7 @@ class ExplorerTests: XCTestCase {
|
|
|
1658
1679
|
|
|
1659
1680
|
// ---- Blank-screen detection ----
|
|
1660
1681
|
// Distinguish between "no a11y labels / custom UI" vs genuinely empty.
|
|
1661
|
-
if
|
|
1682
|
+
if visibleTextInventory.isEmpty && interactable.count == 0 {
|
|
1662
1683
|
let blankKey = "blank:\(titleStr)"
|
|
1663
1684
|
let blankCount = (actionCounts[blankKey] ?? 0) + 1
|
|
1664
1685
|
actionCounts[blankKey] = blankCount
|
|
@@ -1690,16 +1711,26 @@ class ExplorerTests: XCTestCase {
|
|
|
1690
1711
|
}
|
|
1691
1712
|
|
|
1692
1713
|
// ---- Navigation-loop detection ----
|
|
1693
|
-
//
|
|
1714
|
+
// A cycle must actually move across distinct states. Four identical reads satisfy the
|
|
1715
|
+
// arithmetic shape A,A,A,A of the old period-2 check, which mislabeled ordinary
|
|
1716
|
+
// scroll/probe recovery on a stable screen as a navigation loop.
|
|
1694
1717
|
if recentStateHashes.count >= 6 {
|
|
1695
1718
|
let recent = recentStateHashes
|
|
1719
|
+
// Distinct structural hashes are not enough: a list and its detail rows can share
|
|
1720
|
+
// one navigation title and alternate A/B while the explorer intentionally samples
|
|
1721
|
+
// different rows. Calling that a navigation loop is a false positive. Require the
|
|
1722
|
+
// cycle to cross distinct user-visible screen titles as well.
|
|
1696
1723
|
let hasLoop2 = recent.count >= 4 &&
|
|
1697
1724
|
recent[recent.count - 1] == recent[recent.count - 3] &&
|
|
1698
|
-
recent[recent.count - 2] == recent[recent.count - 4]
|
|
1725
|
+
recent[recent.count - 2] == recent[recent.count - 4] &&
|
|
1726
|
+
Set(recent.suffix(2)).count == 2 &&
|
|
1727
|
+
Set(recentScreenTitles.suffix(2)).count == 2
|
|
1699
1728
|
let hasLoop3 = recent.count >= 6 &&
|
|
1700
1729
|
recent[recent.count - 1] == recent[recent.count - 4] &&
|
|
1701
1730
|
recent[recent.count - 2] == recent[recent.count - 5] &&
|
|
1702
|
-
recent[recent.count - 3] == recent[recent.count - 6]
|
|
1731
|
+
recent[recent.count - 3] == recent[recent.count - 6] &&
|
|
1732
|
+
Set(recent.suffix(3)).count == 3 &&
|
|
1733
|
+
Set(recentScreenTitles.suffix(3)).count == 3
|
|
1703
1734
|
if (hasLoop2 || hasLoop3) && !(authSucceeded && detectedInputs.contains { $0.secure }) {
|
|
1704
1735
|
let loopKey = "nav_loop:\(titleStr)"
|
|
1705
1736
|
if actionCounts[loopKey] == nil {
|
|
@@ -1711,17 +1742,10 @@ class ExplorerTests: XCTestCase {
|
|
|
1711
1742
|
}
|
|
1712
1743
|
}
|
|
1713
1744
|
|
|
1714
|
-
//
|
|
1715
|
-
//
|
|
1716
|
-
//
|
|
1717
|
-
|
|
1718
|
-
let unrespKey = "unresponsive:\(titleStr)"
|
|
1719
|
-
if actionCounts[unrespKey] == nil {
|
|
1720
|
-
issues.append((type: "unresponsive_element", severity: "medium", title: "Unresponsive UI on \(titleStr)", desc: "Actions are not changing app state — possible frozen or broken screen"))
|
|
1721
|
-
print("OCQA_ISSUE:{\"type\":\"unresponsive_element\",\"severity\":\"medium\",\"title\":\"Unresponsive UI\",\"screen\":\"\(escapedTitle)\",\"repeated_state_count\":\(repeatedStateCount),\"step\":\(actionCount)}")
|
|
1722
|
-
actionCounts[unrespKey] = 1
|
|
1723
|
-
}
|
|
1724
|
-
}
|
|
1745
|
+
// Do not infer an unresponsive app merely from an unchanged state streak: recovery
|
|
1746
|
+
// gestures (scroll, carousel probe, center probe) are expected to be no-ops on many
|
|
1747
|
+
// healthy screens. Labeled controls have a stronger detector below: a direct tap plus
|
|
1748
|
+
// two delayed, content-signature reads. Hangs have their own time-based detector.
|
|
1725
1749
|
|
|
1726
1750
|
if interactable.count < 3 {
|
|
1727
1751
|
print("OCQA_STATE:low_interactable screen=\(escapedTitle) total=\(elements.count) interactable=\(interactable.count) global=\(globalNavElements.count) nonGlobal=\(nonGlobalCandidates.count)")
|
|
@@ -1757,10 +1781,10 @@ class ExplorerTests: XCTestCase {
|
|
|
1757
1781
|
break
|
|
1758
1782
|
}
|
|
1759
1783
|
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1784
|
+
// Exhausting Tapp's untried candidate pool is not itself a user-visible dead end:
|
|
1785
|
+
// leaf screens commonly have only a working Back control that was already mapped.
|
|
1786
|
+
// Recover first; only the stronger navigation-trap path below emits a finding when
|
|
1787
|
+
// every real back/dismiss route fails.
|
|
1764
1788
|
// tryGoBack does swipe-down as its last resort (sheet dismiss)
|
|
1765
1789
|
let preBackTitle = titleStr
|
|
1766
1790
|
let backWorked = tryGoBack()
|
|
@@ -1772,6 +1796,7 @@ class ExplorerTests: XCTestCase {
|
|
|
1772
1796
|
print("OCQA_ACTION:{\"type\":\"back\",\"reason\":\"dead_end_escape\",\"from\":\"\(escapedTitle)\",\"to\":\"\(escapeJSON(postTitle))\",\"step\":\(actionCount),\"screen\":\"\(escapedTitle)\",\"narrative\":\"\(escapeJSON(recoveryNarrative("back_dead_end", screen: titleStr, to: postTitle)))\"}")
|
|
1773
1797
|
continue
|
|
1774
1798
|
}
|
|
1799
|
+
if actionCount >= maxActions { break }
|
|
1775
1800
|
// Swipe right (back gesture) as another option
|
|
1776
1801
|
let swipeStart = app.coordinate(withNormalizedOffset: CGVector(dx: 0.02, dy: 0.5))
|
|
1777
1802
|
let swipeEnd = app.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
|
|
@@ -2047,6 +2072,7 @@ class ExplorerTests: XCTestCase {
|
|
|
2047
2072
|
}
|
|
2048
2073
|
// Back didn't change screens — fall through to global nav
|
|
2049
2074
|
print("OCQA_ACTION:{\"type\":\"back\",\"reason\":\"screen_exhausted_failed\",\"screen\":\"\(escapedTitle)\",\"step\":\(actionCount),\"narrative\":\"\(escapeJSON(recoveryNarrative("back_failed", screen: titleStr)))\"}")
|
|
2075
|
+
if actionCount >= maxActions { break }
|
|
2050
2076
|
// Stuck on this screen — use global navigation (tab bar) to reach unexplored areas
|
|
2051
2077
|
let globalNav = interactable
|
|
2052
2078
|
.filter { isLikelyGlobalNavigation($0, screenBounds: screenBounds) }
|
|
@@ -3679,6 +3705,20 @@ class ExplorerTests: XCTestCase {
|
|
|
3679
3705
|
return kb.exists ? kb.frame : .zero
|
|
3680
3706
|
}
|
|
3681
3707
|
|
|
3708
|
+
/// Activity indicators are inherently indeterminate. `ProgressIndicator`, however, is also the
|
|
3709
|
+
/// XCTest type for legitimate determinate progress bars (loyalty points, upload percentage,
|
|
3710
|
+
/// onboarding completion). Only value-less/loading-valued progress indicators are hang signals.
|
|
3711
|
+
private func hasIndeterminateLoadingIndicator() -> Bool {
|
|
3712
|
+
if app.activityIndicators.allElementsBoundByIndex.contains(where: { $0.exists && $0.frame.width > 0 && $0.frame.height > 0 }) {
|
|
3713
|
+
return true
|
|
3714
|
+
}
|
|
3715
|
+
return app.progressIndicators.allElementsBoundByIndex.contains { indicator in
|
|
3716
|
+
guard indicator.exists, indicator.frame.width > 0, indicator.frame.height > 0 else { return false }
|
|
3717
|
+
let value = (indicator.value as? String ?? "").trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
|
|
3718
|
+
return value.isEmpty || value == "in progress" || value == "loading"
|
|
3719
|
+
}
|
|
3720
|
+
}
|
|
3721
|
+
|
|
3682
3722
|
/// True when the screen is in a "settled" resting state — no on-screen keyboard and no open
|
|
3683
3723
|
/// transient overlay (menu / dropdown / popover / sheet / picker wheel). A screenshot taken while
|
|
3684
3724
|
/// one of these is up is inherently ambiguous to a visual reviewer (the keyboard "covers" the
|
package/bin/tapp.js
CHANGED
|
@@ -130,6 +130,27 @@ function repeatedFlagValues(argv, name) {
|
|
|
130
130
|
return values;
|
|
131
131
|
}
|
|
132
132
|
|
|
133
|
+
function iosLaunchOptions(flags, argv) {
|
|
134
|
+
const appLaunchArgs = repeatedFlagValues(argv, "launch-arg");
|
|
135
|
+
let appLaunchEnv;
|
|
136
|
+
if (typeof flags["launch-env"] === "string") {
|
|
137
|
+
try {
|
|
138
|
+
const parsed = JSON.parse(flags["launch-env"]);
|
|
139
|
+
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object" || Object.values(parsed).some((value) => typeof value !== "string")) {
|
|
140
|
+
throw new Error("expected a JSON object with string values");
|
|
141
|
+
}
|
|
142
|
+
appLaunchEnv = parsed;
|
|
143
|
+
} catch (error) {
|
|
144
|
+
console.error(`❌ --launch-env must be a JSON object with string values: ${error.message}`);
|
|
145
|
+
process.exit(2);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return {
|
|
149
|
+
...(appLaunchArgs.length ? { appLaunchArgs } : {}),
|
|
150
|
+
...(appLaunchEnv ? { appLaunchEnv } : {}),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
133
154
|
const engineImport = () => import(path.join(packageRoot, "mcp-server", "src", "index.js"));
|
|
134
155
|
|
|
135
156
|
function requireMacFor(what) {
|
|
@@ -197,11 +218,38 @@ async function resolveTargetOrExit(engine, input) {
|
|
|
197
218
|
return resolved.bundleId;
|
|
198
219
|
}
|
|
199
220
|
|
|
221
|
+
function safeCommandUsage(verb) {
|
|
222
|
+
const usage = {
|
|
223
|
+
explore: "tapp explore [target] [--platform ios|android|web] [--actions N] [--timeout SEC] [--email VALUE] [--password VALUE] [--baseline FILE] [--json FILE]\n iOS launch configuration: [--launch-arg VALUE ...] [--launch-env '{\"KEY\":\"VALUE\"}']\n Android: [--app-id ID] [--apk FILE] [--serial ID] [--keep-data]",
|
|
224
|
+
init: "tapp init [repo] [--explore] [--refresh] [--platform PLATFORM] [--target NAME] [--url URL] [--dry-run]",
|
|
225
|
+
open: "tapp open [target] [--platform ios|android|web] [--out FILE] [--tap TEXT] [--wait-for TEXT]",
|
|
226
|
+
tree: "tapp tree [target] [--platform ios|android|web] [--json] [--tap TEXT] [--wait-for TEXT]",
|
|
227
|
+
shot: "tapp shot [--out FILE]",
|
|
228
|
+
apps: "tapp apps",
|
|
229
|
+
build: "tapp build [repo] [--scheme NAME] [--configuration NAME]",
|
|
230
|
+
flow: "tapp flow validate FILE [--platform PLATFORM] [--map FILE]\ntapp flow run FILE [--email VALUE] [--password VALUE]",
|
|
231
|
+
task: "tapp task validate FILE [--platform PLATFORM] [--map FILE]\ntapp task compile FILE --platform PLATFORM [--inputs JSON] [--out FILE]\ntapp task run FILE --platform PLATFORM [--url URL|--bundle-id ID|--app-id ID] [--inputs JSON]",
|
|
232
|
+
contract: "tapp contract validate FILE [--platform PLATFORM] [--map FILE]\ntapp contract compile FILE --platform PLATFORM [--out FILE]\ntapp contract run FILE --platform PLATFORM [--url URL|--bundle-id ID|--app-id ID]",
|
|
233
|
+
scenario: "tapp scenario validate FILE [--project-dir DIR]\ntapp scenario run FILE --platform web --url URL [--project-dir DIR]",
|
|
234
|
+
map: "tapp map build MARKERS [--platform PLATFORM] [--out FILE] [--replace]\ntapp map inspect [FILE]\ntapp map diff BEFORE AFTER [--comparable]",
|
|
235
|
+
pr: "tapp pr plan [--base REF|--changed-files FILE] [--head REF] [--platform PLATFORM] [--out FILE]\ntapp pr gate PLAN [gate target/options]\ntapp pr adopt PLAN --item ID [--project-dir DIR]",
|
|
236
|
+
plan: "tapp plan show [FILE]\ntapp plan review [FILE] --approve NAME[,NAME] --reject NAME[,NAME] --defer NAME[,NAME]\ntapp plan generate|validate|promote [FILE] [options]",
|
|
237
|
+
baseline: "tapp baseline create [repo] [--platform PLATFORM] [--target NAME] [--from GATE.json] [--replace]",
|
|
238
|
+
actor: "tapp actor set NAME --email-env ENV --password-env ENV [--project-dir DIR]\ntapp actor list [repo]",
|
|
239
|
+
app: "tapp app [repo] [--no-open] [--port PORT]",
|
|
240
|
+
report: "tapp report [captureId|latest]",
|
|
241
|
+
doctor: "tapp doctor",
|
|
242
|
+
install: "tapp install",
|
|
243
|
+
mcp: "tapp mcp",
|
|
244
|
+
};
|
|
245
|
+
return usage[verb] || `tapp ${verb}`;
|
|
246
|
+
}
|
|
247
|
+
|
|
200
248
|
// Safe help: `--help`/`-h` on ANY verb prints the command reference and does NOTHING else — never
|
|
201
249
|
// builds, launches, writes, or opens (ADR-0005 manual-testing requirement). `ci` keeps its own
|
|
202
250
|
// richer `--help` (a safe usage print in ci-gate.sh); help/version don't need interception.
|
|
203
251
|
if ((rest.includes("--help") || rest.includes("-h")) && !["help", "version", "--version", "-v", "ci"].includes(command)) {
|
|
204
|
-
console.log(
|
|
252
|
+
console.log(`Usage:\n ${safeCommandUsage(command).replaceAll("\n", "\n ")}\n\nℹ️ --help never builds, launches, writes, or opens. Full command reference:\n`);
|
|
205
253
|
command = "help";
|
|
206
254
|
rest = [];
|
|
207
255
|
}
|
|
@@ -271,7 +319,7 @@ switch (command) {
|
|
|
271
319
|
testEmail: typeof flags.email === "string" ? flags.email : undefined,
|
|
272
320
|
testPassword: typeof flags.password === "string" ? flags.password : undefined,
|
|
273
321
|
runExploration: engine?.runInitExploration,
|
|
274
|
-
onProgress: (progress) => process.stderr.write(`\r🔍 Import exploration… ${progress.action}/${progress.max || actions} actions · ${progress.states} ${platform === "web" ? "pages" : "screens"}
|
|
322
|
+
onProgress: (progress) => process.stderr.write(`\r🔍 Import exploration… ${progress.action}/${progress.max || actions} actions · ${progress.states} ${platform === "web" ? "pages reached" : platform === "ios" ? "structural states observed" : "screens reached"} `),
|
|
275
323
|
onStatus: (status) => console.error(`⏳ ${status}`),
|
|
276
324
|
outDir,
|
|
277
325
|
maxContracts,
|
|
@@ -408,6 +456,7 @@ switch (command) {
|
|
|
408
456
|
// hidden deprecated alias.
|
|
409
457
|
if (command === "qa") console.error("note: 'qa' is now 'explore' — 'qa' still works for now.\n");
|
|
410
458
|
const { flags, positionals } = parseVerbArgs(rest);
|
|
459
|
+
const launchOptions = iosLaunchOptions(flags, rest);
|
|
411
460
|
let target = positionals[0] || "";
|
|
412
461
|
let baselineFindings;
|
|
413
462
|
if (flags.baseline) {
|
|
@@ -431,7 +480,7 @@ switch (command) {
|
|
|
431
480
|
const modelPlatform = typeof flags.platform === "string" ? flags.platform.toLowerCase() : "";
|
|
432
481
|
if (modelPlatform === "ios") requireMacFor("iOS testing");
|
|
433
482
|
const onProgress = (p) =>
|
|
434
|
-
process.stderr.write(`\r🔍 Exploring… ${p.action}/${p.max || flags.actions || 60} actions · ${p.states}
|
|
483
|
+
process.stderr.write(`\r🔍 Exploring… ${p.action}/${p.max || flags.actions || 60} actions · ${p.states} states observed `);
|
|
435
484
|
const r = await engine.runExploreTarget({
|
|
436
485
|
projectDir: process.cwd(),
|
|
437
486
|
platform: modelPlatform,
|
|
@@ -440,6 +489,7 @@ switch (command) {
|
|
|
440
489
|
timeout: flags.timeout,
|
|
441
490
|
testEmail: flags.email,
|
|
442
491
|
testPassword: flags.password,
|
|
492
|
+
...launchOptions,
|
|
443
493
|
baselineFindings,
|
|
444
494
|
surface: "cli",
|
|
445
495
|
onProgress,
|
|
@@ -461,15 +511,19 @@ switch (command) {
|
|
|
461
511
|
process.exit(2);
|
|
462
512
|
}
|
|
463
513
|
if (platform === "ios") requireMacFor("iOS testing");
|
|
514
|
+
if (platform !== "ios" && Object.keys(launchOptions).length) {
|
|
515
|
+
console.error("❌ --launch-arg and --launch-env apply only to iOS targets");
|
|
516
|
+
process.exit(2);
|
|
517
|
+
}
|
|
464
518
|
if (platform === "web" && !/^https?:\/\//i.test(target)) {
|
|
465
519
|
console.error("❌ Web QA needs an http(s) URL");
|
|
466
520
|
process.exit(2);
|
|
467
521
|
}
|
|
468
522
|
const bundleId = platform === "ios" ? await resolveTargetOrExit(engine, target) : null;
|
|
469
523
|
const android = platform === "android" ? androidTarget(flags, target) : null;
|
|
470
|
-
const
|
|
524
|
+
const progressMetric = platform === "web" ? "pages reached" : platform === "ios" ? "structural states observed" : "screens reached";
|
|
471
525
|
const onProgress = (p) =>
|
|
472
|
-
process.stderr.write(`\r🔍 Exploring… ${p.action}/${p.max || flags.actions || 60} actions · ${p.states} ${
|
|
526
|
+
process.stderr.write(`\r🔍 Exploring… ${p.action}/${p.max || flags.actions || 60} actions · ${p.states} ${progressMetric} `);
|
|
473
527
|
const r = platform === "web"
|
|
474
528
|
? await engine.runQaWeb({
|
|
475
529
|
url: target,
|
|
@@ -497,7 +551,7 @@ switch (command) {
|
|
|
497
551
|
bundleId,
|
|
498
552
|
maxActions: flags.actions,
|
|
499
553
|
timeout: flags.timeout,
|
|
500
|
-
args: { testEmail: flags.email, testPassword: flags.password, baselineFindings },
|
|
554
|
+
args: { testEmail: flags.email, testPassword: flags.password, baselineFindings, ...launchOptions },
|
|
501
555
|
surface: "cli",
|
|
502
556
|
onProgress,
|
|
503
557
|
});
|
|
@@ -555,7 +609,6 @@ switch (command) {
|
|
|
555
609
|
if (target.apkPath) await driver.install(target.apkPath);
|
|
556
610
|
const snap = await driver.launch({ clearData: flags["clear-data"] === true });
|
|
557
611
|
const data = await driver.screenshot();
|
|
558
|
-
await driver.forceStop();
|
|
559
612
|
console.log(`🚀 Launched \`${target.appId}\` (Android)\n`);
|
|
560
613
|
console.log(engine.formatScreen(snap.screenTitle, snap.elements));
|
|
561
614
|
const out = typeof flags.out === "string" ? flags.out : path.join(tappHome, "shots", `${target.appId}-${Date.now()}.png`);
|
|
@@ -615,12 +668,17 @@ switch (command) {
|
|
|
615
668
|
break;
|
|
616
669
|
}
|
|
617
670
|
if (platform === "android") {
|
|
618
|
-
const
|
|
671
|
+
const input = positionals[0] || "";
|
|
672
|
+
const hasTarget = !!(input || flags["app-id"] || flags.apk);
|
|
673
|
+
const target = hasTarget
|
|
674
|
+
? androidTarget(flags, input)
|
|
675
|
+
: { serial: typeof flags.serial === "string" ? flags.serial : undefined };
|
|
619
676
|
const { AndroidDriver } = await import(path.join(packageRoot, "mcp-server", "src", "android-driver.js"));
|
|
620
677
|
const driver = new AndroidDriver(target);
|
|
621
678
|
await driver.ensureDevice();
|
|
622
|
-
|
|
623
|
-
|
|
679
|
+
if (target.apkPath) await driver.install(target.apkPath);
|
|
680
|
+
const snap = target.appId ? await driver.launch() : await driver.snapshot();
|
|
681
|
+
if (flags.json) console.log(JSON.stringify({ platform: "android", appId: target.appId || null, activity: snap.activity, screenTitle: snap.screenTitle, elements: snap.elements }, null, 2));
|
|
624
682
|
else console.log(engine.formatScreen(snap.screenTitle, snap.elements));
|
|
625
683
|
break;
|
|
626
684
|
}
|
|
@@ -118,8 +118,15 @@ export function detectAndroidScreen(elements, activity = "") {
|
|
|
118
118
|
|
|
119
119
|
export function isAndroidAppSnapshot(snapshot, appId) {
|
|
120
120
|
if (!snapshot || !appId) return false;
|
|
121
|
-
|
|
122
|
-
|
|
121
|
+
const activity = String(snapshot.activity || "");
|
|
122
|
+
const ownsActivity = activity.startsWith(`${appId}/`);
|
|
123
|
+
const packages = new Set((snapshot.elements || []).map((e) => e.package).filter(Boolean));
|
|
124
|
+
const ownsElements = packages.has(appId);
|
|
125
|
+
// dumpXml and dumpsys used to run concurrently. If an app crashed while they were sampled, a
|
|
126
|
+
// stale activity from the dead app could be paired with another app's UI tree and Tapp would map
|
|
127
|
+
// that unrelated app as the crash destination. When both signals exist, require agreement.
|
|
128
|
+
if (activity && packages.size) return ownsActivity && ownsElements;
|
|
129
|
+
return ownsActivity || ownsElements;
|
|
123
130
|
}
|
|
124
131
|
|
|
125
132
|
export class AndroidDriver {
|
|
@@ -158,6 +165,12 @@ export class AndroidDriver {
|
|
|
158
165
|
if (this.appId) await this.adb(["shell", "am", "force-stop", this.appId]);
|
|
159
166
|
}
|
|
160
167
|
|
|
168
|
+
async isProcessAlive() {
|
|
169
|
+
if (!this.appId) return false;
|
|
170
|
+
const r = await this.adb(["shell", "pidof", this.appId]);
|
|
171
|
+
return r.code === 0 && /\d/.test(String(r.stdout || ""));
|
|
172
|
+
}
|
|
173
|
+
|
|
161
174
|
async clearData() {
|
|
162
175
|
if (!this.appId) throw new Error("appId is required to clear Android app data");
|
|
163
176
|
const r = await this.adb(["shell", "pm", "clear", this.appId]);
|
|
@@ -8,6 +8,7 @@ import { semanticUiKey } from "./ui-map.js";
|
|
|
8
8
|
|
|
9
9
|
const ERROR_RE = /\b(something went wrong|internal server error|an error occurred|failed to load|unhandled exception|has stopped)\b/i;
|
|
10
10
|
const DESTRUCTIVE_RE = /\b(delete|remove|purchase|buy now|pay now|reset|erase|unsubscribe|sign out|log out|logout)\b/i;
|
|
11
|
+
const AUTH_SUBMIT_RE = /\b(sign[ -]?in|log[ -]?in|continue|submit)\b/i;
|
|
11
12
|
|
|
12
13
|
function stateHash(snap) {
|
|
13
14
|
return snap.elements.map((e) => `${androidElementKey(e)}:${e.text}:${e.x},${e.y}`).join("|");
|
|
@@ -40,6 +41,21 @@ function isDestructive(e) {
|
|
|
40
41
|
return DESTRUCTIVE_RE.test(`${e.label || ""} ${e.text || ""} ${e.description || ""} ${e.id || ""}`.replace(/[_-]+/g, " "));
|
|
41
42
|
}
|
|
42
43
|
|
|
44
|
+
export function isAndroidAuthSubmit(element) {
|
|
45
|
+
return AUTH_SUBMIT_RE.test(`${element?.label || ""} ${element?.text || ""} ${element?.description || ""} ${element?.id || ""}`.replace(/[_-]+/g, " "));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function isAndroidBlankSnapshot(snapshot) {
|
|
49
|
+
const elements = snapshot?.elements || [];
|
|
50
|
+
const meaningful = elements.some((element) =>
|
|
51
|
+
String(element.text || "").trim() ||
|
|
52
|
+
String(element.description || "").trim() ||
|
|
53
|
+
(String(element.id || "").trim() && !/^(android:)?id\/content$/i.test(String(element.id || "").trim()))
|
|
54
|
+
);
|
|
55
|
+
const interactive = elements.some((element) => element.hittable && (element.clickable || /Button|EditText|Tab|Switch|CheckBox/i.test(element.type)));
|
|
56
|
+
return !meaningful && !interactive;
|
|
57
|
+
}
|
|
58
|
+
|
|
43
59
|
export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40, timeoutSec = 300, outDir, testEmail = "", testPassword = "", clearData = true, seedTargets = [], onProgress = () => {}, driver }) {
|
|
44
60
|
const d = driver || new AndroidDriver({ appId, serial });
|
|
45
61
|
d.appId = appId;
|
|
@@ -57,6 +73,18 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
57
73
|
let issues = 0;
|
|
58
74
|
let actions = 0;
|
|
59
75
|
let snap = await d.launch({ clearData });
|
|
76
|
+
let crashReported = false;
|
|
77
|
+
|
|
78
|
+
const reportProcessExit = async (screen, step) => {
|
|
79
|
+
const alive = typeof d.isProcessAlive === "function" ? await d.isProcessAlive() : true;
|
|
80
|
+
if (alive || crashReported) return false;
|
|
81
|
+
crashReported = true;
|
|
82
|
+
emit("ISSUE", { type: "crash", severity: "critical", title: "App process exited during exploration", screen: screen || "Launch", step });
|
|
83
|
+
issues += 1;
|
|
84
|
+
return true;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
if (!isAndroidAppSnapshot(snap, appId)) await reportProcessExit("Launch", 0);
|
|
60
88
|
|
|
61
89
|
const normalizedTargets = (Array.isArray(seedTargets) ? seedTargets : []).filter((target) =>
|
|
62
90
|
target?.platform === "android" && target?.status === "planned" && target?.navigation?.status === "replayable" &&
|
|
@@ -95,7 +123,11 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
95
123
|
const before = state;
|
|
96
124
|
snap = await d.settle();
|
|
97
125
|
state = await recordState(snap);
|
|
98
|
-
if (!state) {
|
|
126
|
+
if (!state) {
|
|
127
|
+
const crashed = await reportProcessExit(before.screen, actions);
|
|
128
|
+
failure = crashed ? "The application process exited while replaying the UI Map path" : "The UI Map path left the application foreground";
|
|
129
|
+
break;
|
|
130
|
+
}
|
|
99
131
|
emit("TRANSITION", { from: before.screen, to: state.screen, action: selector, changed: before.hash !== state.hash });
|
|
100
132
|
onProgress({ action: actions, max: maxActions, states: visited.size });
|
|
101
133
|
}
|
|
@@ -113,7 +145,10 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
113
145
|
// UIAutomator can still describe the launcher or a system surface after
|
|
114
146
|
// Back/external navigation. Those are exploration boundaries, never nodes
|
|
115
147
|
// in the application-owned UI Map.
|
|
116
|
-
if (!isAndroidAppSnapshot(snap, appId))
|
|
148
|
+
if (!isAndroidAppSnapshot(snap, appId)) {
|
|
149
|
+
await reportProcessExit(visited.size ? [...visited.values()].at(-1) : "Launch", actions);
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
117
152
|
const recorded = await recordState(snap);
|
|
118
153
|
if (!recorded) break;
|
|
119
154
|
const { hash, screen, inputs } = recorded;
|
|
@@ -124,7 +159,7 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
124
159
|
emit("ISSUE", { type: "error_surface", severity: "high", title: "Visible error surface", screen, step: actions });
|
|
125
160
|
issues += 1;
|
|
126
161
|
}
|
|
127
|
-
if (snap
|
|
162
|
+
if (isAndroidBlankSnapshot(snap)) {
|
|
128
163
|
emit("ISSUE", { type: "blank_screen", severity: "high", title: "No usable controls or content", screen, step: actions });
|
|
129
164
|
issues += 1;
|
|
130
165
|
}
|
|
@@ -154,16 +189,24 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
154
189
|
});
|
|
155
190
|
if (candidate) {
|
|
156
191
|
const target = controlLabel(candidate);
|
|
192
|
+
const loginSubmit = inputs.some((input) => input.secure) && isAndroidAuthSubmit(candidate);
|
|
157
193
|
tried.add(`${hash}|tap|${target}`);
|
|
158
194
|
const before = hash;
|
|
159
195
|
const r = await d.tap(target, snap);
|
|
160
196
|
actions += 1;
|
|
161
197
|
snap = await d.settle();
|
|
162
198
|
const after = stateHash(snap);
|
|
163
|
-
emit("ACTION", { type: "tap", target, reason: "untried_control", step: actions, screen, status: r.status });
|
|
164
|
-
if (!isAndroidAppSnapshot(snap, appId))
|
|
199
|
+
emit("ACTION", { type: loginSubmit ? "login_submit" : "tap", target, reason: "untried_control", step: actions, screen, status: r.status });
|
|
200
|
+
if (!isAndroidAppSnapshot(snap, appId)) {
|
|
201
|
+
await reportProcessExit(screen, actions);
|
|
202
|
+
break;
|
|
203
|
+
}
|
|
165
204
|
emit("TRANSITION", { from: screen, to: snap.screenTitle, action: target, changed: before !== after });
|
|
166
|
-
|
|
205
|
+
const authFailed = loginSubmit && inputDescriptors(snap.elements).some((input) => input.secure);
|
|
206
|
+
if (r.status === "ok" && authFailed) {
|
|
207
|
+
emit("ISSUE", { type: "auth_failed", severity: "high", title: "Sign-in attempt remained on the login screen", screen, target, step: actions });
|
|
208
|
+
issues += 1;
|
|
209
|
+
} else if (r.status === "ok" && before === after && candidate.clickable) {
|
|
167
210
|
emit("ISSUE", { type: "unresponsive_element", severity: "medium", title: `Control did not respond: ${target}`, screen, target, step: actions });
|
|
168
211
|
issues += 1;
|
|
169
212
|
}
|
|
@@ -179,7 +222,10 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
179
222
|
actions += 1;
|
|
180
223
|
const next = await d.settle();
|
|
181
224
|
emit("ACTION", { type: "back", target: "system back", reason: "state_exhausted", step: actions, screen });
|
|
182
|
-
if (!isAndroidAppSnapshot(next, appId))
|
|
225
|
+
if (!isAndroidAppSnapshot(next, appId)) {
|
|
226
|
+
await reportProcessExit(screen, actions);
|
|
227
|
+
break;
|
|
228
|
+
}
|
|
183
229
|
emit("TRANSITION", { from: screen, to: next.screenTitle, action: "back", changed: stateHash(next) !== hash });
|
|
184
230
|
if (stateHash(next) !== hash) { snap = next; continue; }
|
|
185
231
|
}
|
|
@@ -187,11 +233,7 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
187
233
|
}
|
|
188
234
|
|
|
189
235
|
const timedOut = Date.now() >= deadline;
|
|
190
|
-
|
|
191
|
-
emit("ISSUE", { type: "explore_timeout", severity: "medium", title: `Exploration timed out after ${timeoutSec}s`, screen: snap.screenTitle, step: actions });
|
|
192
|
-
issues += 1;
|
|
193
|
-
}
|
|
194
|
-
emit("COMPLETE", { actions, states: visited.size, issues, screens: [...new Set(visited.values())].join(","), outcome: timedOut ? "timeout" : "complete" });
|
|
236
|
+
emit("COMPLETE", { actions, states: visited.size, issues, screens: [...new Set(visited.values())].join(","), outcome: timedOut ? "timeout" : "complete", timedOut, ...(timedOut ? { timeoutSeconds: timeoutSec } : {}) });
|
|
195
237
|
onProgress({ action: actions, max: maxActions, states: visited.size });
|
|
196
238
|
return { markersPath, outDir, actions, states: visited.size, issues, timedOut, seedTargets: normalizedTargets };
|
|
197
239
|
}
|
|
@@ -402,7 +402,7 @@ function renderMarkdown(report, regression, flows, scenarios, contracts, prPlan,
|
|
|
402
402
|
} else if (gate.policy === "gate") {
|
|
403
403
|
lines.push("");
|
|
404
404
|
lines.push("### Baseline — 🟡 not active yet");
|
|
405
|
-
lines.push("No baseline was supplied
|
|
405
|
+
lines.push("No baseline was supplied. This gate still enforces absolute blockers and reviewed suite failures, but it cannot identify new regressions until a baseline is saved.");
|
|
406
406
|
}
|
|
407
407
|
if (prPlan) {
|
|
408
408
|
lines.push("");
|
|
@@ -487,7 +487,7 @@ function renderMarkdown(report, regression, flows, scenarios, contracts, prPlan,
|
|
|
487
487
|
}
|
|
488
488
|
|
|
489
489
|
const args = parseArgs(process.argv.slice(2));
|
|
490
|
-
const report = buildQaReport(args.markers, { platform: args.platform || "ios" });
|
|
490
|
+
const report = buildQaReport(args.markers, { platform: args.platform || "ios", target: args.label || null });
|
|
491
491
|
if (!report) {
|
|
492
492
|
// Required evidence could not be obtained — this is inconclusive (fails closed), not a gate FAIL
|
|
493
493
|
// and not a usage error. See the outcome model in report.js (GATE_EXIT).
|
|
@@ -561,7 +561,7 @@ try {
|
|
|
561
561
|
const decision = evaluateGate({ report, regression, flows, scenarios, contracts, prPlan, baseline, failOn: args.failOn });
|
|
562
562
|
const gate = {
|
|
563
563
|
...decision,
|
|
564
|
-
target: args.targetKey || null,
|
|
564
|
+
target: args.targetKey || report.target || null,
|
|
565
565
|
revision: gitRevision(args.projectDir),
|
|
566
566
|
checked: report.checkedFor,
|
|
567
567
|
notChecked: report.notChecked,
|
|
@@ -49,7 +49,7 @@ function collectShots(captureDir) {
|
|
|
49
49
|
}
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
export function writeHtmlReport(captureDir, { report, label = "" } = {}) {
|
|
52
|
+
export function writeHtmlReport(captureDir, { report, label = "", recordingWarning = "" } = {}) {
|
|
53
53
|
const r = report || buildQaReport(path.join(captureDir, "ocqa-markers.txt"));
|
|
54
54
|
if (!r) return null;
|
|
55
55
|
|
|
@@ -78,7 +78,16 @@ export function writeHtmlReport(captureDir, { report, label = "" } = {}) {
|
|
|
78
78
|
|
|
79
79
|
const videoHtml = video
|
|
80
80
|
? `<h2>Recording — the full exploration</h2>\n<video controls preload="metadata" style="width:100%;border:1px solid #d0d7de;border-radius:8px" src="${esc(video)}"></video>`
|
|
81
|
-
:
|
|
81
|
+
: recordingWarning
|
|
82
|
+
? `<h2>Recording</h2>\n<p class="warning">Unavailable: ${esc(recordingWarning)}. Screenshots were still captured.</p>`
|
|
83
|
+
: "";
|
|
84
|
+
|
|
85
|
+
const scopeList = (items) => (items || []).map((item) => `<li>${esc(item)}</li>`).join("\n");
|
|
86
|
+
const scopeHtml = `<section class="scope">
|
|
87
|
+
<div><h2>Checked this run</h2><ul>${scopeList(r.checkedFor) || "<li class='dim'>No automated checks completed.</li>"}</ul></div>
|
|
88
|
+
<div><h2>Not checked this run</h2><ul>${scopeList(r.notChecked) || "<li class='dim'>No additional limitations recorded.</li>"}</ul></div>
|
|
89
|
+
${r.conditionsNotReached?.length ? `<div><h2>Conditions not reached</h2><ul>${scopeList(r.conditionsNotReached)}</ul></div>` : ""}
|
|
90
|
+
</section>`;
|
|
82
91
|
|
|
83
92
|
const html = `<!doctype html>
|
|
84
93
|
<html lang="en">
|
|
@@ -90,6 +99,10 @@ export function writeHtmlReport(captureDir, { report, label = "" } = {}) {
|
|
|
90
99
|
h1 { font-size: 1.6rem; margin-bottom: 0.2rem; }
|
|
91
100
|
.meta { color: #57606a; margin-bottom: 1.2rem; }
|
|
92
101
|
.headline { background: #f6f8fa; border-radius: 8px; padding: 0.9rem 1.1rem; margin: 1rem 0; }
|
|
102
|
+
.warning { background: #fff8c5; border: 1px solid #d4a72c; border-radius: 8px; padding: 0.75rem 1rem; }
|
|
103
|
+
.scope { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 0.8rem 1.5rem; }
|
|
104
|
+
.scope h2 { font-size: 1.05rem; margin-bottom: 0.35rem; }
|
|
105
|
+
.scope ul { margin-top: 0; padding-left: 1.2rem; }
|
|
93
106
|
.sev { color: #fff; border-radius: 4px; padding: 0.05rem 0.45rem; font-size: 0.78rem; font-weight: 600; margin-right: 0.4rem; }
|
|
94
107
|
ul.findings { padding-left: 1.1rem; } ul.findings li { margin-bottom: 0.6rem; }
|
|
95
108
|
.ai { color: #57606a; font-size: 0.88rem; margin: 0.15rem 0 0 0.2rem; }
|
|
@@ -105,6 +118,7 @@ export function writeHtmlReport(captureDir, { report, label = "" } = {}) {
|
|
|
105
118
|
<div class="meta">${esc(label)} · evidence page (observation, not a release decision)</div>
|
|
106
119
|
${r.platform === "web" ? `<div class="meta">Deterministic basis: ${r.deterministicFindingCounts?.total || 0} deterministic finding(s); ${r.sampledFindingCounts?.total || 0} sampled probe finding(s) are advisory.</div>` : ""}
|
|
107
120
|
<div class="headline">${esc(r.headline)}</div>
|
|
121
|
+
${scopeHtml}
|
|
108
122
|
<h2>Findings</h2>
|
|
109
123
|
<ul class="findings">
|
|
110
124
|
${findingsHtml}
|
package/mcp-server/src/index.js
CHANGED
|
@@ -984,7 +984,10 @@ async function runExploreStreaming(bundleId, actions, timeout, env, onProgress)
|
|
|
984
984
|
const before = new Set(listCaptureRuns(80).map((r) => r.id));
|
|
985
985
|
const proc = spawn("bash", cmdArgs, { cwd: repoRoot, env: { ...process.env, ...env } });
|
|
986
986
|
proc.stdout.on("data", () => {});
|
|
987
|
-
|
|
987
|
+
let captureStderr = "";
|
|
988
|
+
proc.stderr.on("data", (chunk) => {
|
|
989
|
+
captureStderr = (captureStderr + chunk.toString("utf8")).slice(-8_000);
|
|
990
|
+
});
|
|
988
991
|
const closed = new Promise((res) => proc.on("close", (code) => res(code ?? 1)));
|
|
989
992
|
|
|
990
993
|
let captureDir = null;
|
|
@@ -1037,7 +1040,18 @@ async function runExploreStreaming(bundleId, actions, timeout, env, onProgress)
|
|
|
1037
1040
|
await closed;
|
|
1038
1041
|
const created = listCaptureRuns(80).find((r) => !before.has(r.id))
|
|
1039
1042
|
|| (captureDir ? { id: path.basename(captureDir), path: captureDir, relativePath: path.relative(repoRoot, captureDir) } : null);
|
|
1040
|
-
return { created, timedOut };
|
|
1043
|
+
return { created, timedOut, captureStderr };
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
export function recordingUnavailableReason(stderr = "") {
|
|
1047
|
+
const text = String(stderr);
|
|
1048
|
+
if (/resource busy|host recording is already in progress/i.test(text)) {
|
|
1049
|
+
return "the simulator recorder is busy with another host recording";
|
|
1050
|
+
}
|
|
1051
|
+
if (/could not start simulator video recording/i.test(text)) {
|
|
1052
|
+
return "the simulator could not start video recording";
|
|
1053
|
+
}
|
|
1054
|
+
return "the simulator did not produce a recording";
|
|
1041
1055
|
}
|
|
1042
1056
|
|
|
1043
1057
|
// Grab the booted simulator's current screen and return it downscaled + JPEG-compressed so the
|
|
@@ -1318,7 +1332,7 @@ export function qaNextSteps(report, surface = "mcp") {
|
|
|
1318
1332
|
return next;
|
|
1319
1333
|
}
|
|
1320
1334
|
|
|
1321
|
-
function formatQaReport(report, { regression, inputHint, timedOut, bundleId, aiConfigured, reportHtml, recording, uiMap, surface = "mcp" } = {}) {
|
|
1335
|
+
function formatQaReport(report, { regression, inputHint, timedOut, bundleId, aiConfigured, reportHtml, recording, recordingWarning, uiMap, surface = "mcp" } = {}) {
|
|
1322
1336
|
const c = report.findingCounts || {};
|
|
1323
1337
|
const badge = observationBadge(report);
|
|
1324
1338
|
const sevBits = ["critical", "high", "medium", "low"]
|
|
@@ -1338,6 +1352,7 @@ function formatQaReport(report, { regression, inputHint, timedOut, bundleId, aiC
|
|
|
1338
1352
|
if (uiMap) L.push(`**UI Map** — ${uiMap.nodeCount} states · ${uiMap.edgeCount} transitions · ${uiMap.controlCount} semantic controls · ${uiMap.path}`);
|
|
1339
1353
|
if (reportHtml) L.push(`**Evidence** — 📄 ${reportHtml} (screenshots of every screen + findings, shareable)`);
|
|
1340
1354
|
if (recording) L.push(`**Recording** — 🎬 ${recording} (full exploration, embedded in the evidence page)`);
|
|
1355
|
+
else if (recordingWarning) L.push(`**Recording** — ⚠️ unavailable: ${recordingWarning}; screenshots were still captured`);
|
|
1341
1356
|
L.push(`**Issues** — ${c.total ? `${c.total}${sevBits ? ` (${sevBits})` : ""}` : "none found ✨"}`);
|
|
1342
1357
|
if (Array.isArray(report.findings) && report.findings.length) {
|
|
1343
1358
|
L.push("");
|
|
@@ -1367,7 +1382,7 @@ function formatQaReport(report, { regression, inputHint, timedOut, bundleId, aiC
|
|
|
1367
1382
|
L.push("");
|
|
1368
1383
|
L.push(`> ℹ️ ${inputHint}`);
|
|
1369
1384
|
}
|
|
1370
|
-
// The
|
|
1385
|
+
// The observation's scope: enumerate exactly what ran and what remains unchecked.
|
|
1371
1386
|
if (Array.isArray(report.checkedFor) && report.checkedFor.length) {
|
|
1372
1387
|
L.push("");
|
|
1373
1388
|
L.push(`> ✅ Checked: ${report.checkedFor.join(" · ")}`);
|
|
@@ -1381,11 +1396,10 @@ function formatQaReport(report, { regression, inputHint, timedOut, bundleId, aiC
|
|
|
1381
1396
|
const next = qaNextSteps(report, surface);
|
|
1382
1397
|
L.push("");
|
|
1383
1398
|
L.push(`**Next** — ${next.join(" · ")}`);
|
|
1384
|
-
// The gate hook belongs at the moment the user thinks "I want
|
|
1385
|
-
// i.e. right after a verdict that found something, or after they hand-diffed a baseline.
|
|
1399
|
+
// The gate hook belongs at the moment the user thinks "I want these checks on every PR".
|
|
1386
1400
|
if ((report.findings && report.findings.length) || regression) {
|
|
1387
1401
|
L.push("");
|
|
1388
|
-
L.push("> 🚦 Teams:
|
|
1402
|
+
L.push("> 🚦 Teams: run these checks plus reviewed release contracts as a merge gate on every PR — https://github.com/aarwitz/tapp#ci-gate");
|
|
1389
1403
|
}
|
|
1390
1404
|
return L.join("\n");
|
|
1391
1405
|
}
|
|
@@ -1466,7 +1480,7 @@ export async function runQaWeb({ url, maxActions, timeout, testEmail, testPasswo
|
|
|
1466
1480
|
} catch (err) {
|
|
1467
1481
|
return { error: String(err.message || err) };
|
|
1468
1482
|
}
|
|
1469
|
-
const report = buildQaReport(webResult.markersPath, { platform: "web" });
|
|
1483
|
+
const report = buildQaReport(webResult.markersPath, { platform: "web", target: url.trim() });
|
|
1470
1484
|
if (!report) return { error: "Web exploration produced no markers", details: { capture: { id, path: outDir } } };
|
|
1471
1485
|
const backend = remoteAiOptedIn() ? resolveModelBackend() : null;
|
|
1472
1486
|
if (backend && report.findings.length) {
|
|
@@ -1509,7 +1523,7 @@ export async function runQaAndroid({ appId, apkPath, serial, maxActions, timeout
|
|
|
1509
1523
|
} catch (error) {
|
|
1510
1524
|
return { error: error.message || String(error), details: { capture: { id, path: outDir } } };
|
|
1511
1525
|
}
|
|
1512
|
-
const report = buildQaReport(androidResult.markersPath, { platform: "android" });
|
|
1526
|
+
const report = buildQaReport(androidResult.markersPath, { platform: "android", target: appId.trim() });
|
|
1513
1527
|
if (!report) return { error: "Android exploration produced no markers", details: { capture: { id, path: outDir } } };
|
|
1514
1528
|
const backend = remoteAiOptedIn() ? resolveModelBackend() : null;
|
|
1515
1529
|
if (backend && report.findings.length) {
|
|
@@ -1541,10 +1555,10 @@ export async function runQaIos({ bundleId, maxActions, timeout, args = {}, surfa
|
|
|
1541
1555
|
const timeoutSec = Math.max(30, Math.min(3600, asInteger(timeout, 600)));
|
|
1542
1556
|
const env = explorationEnvFromArgs(args);
|
|
1543
1557
|
|
|
1544
|
-
const { created, timedOut } = await runExploreStreaming(bundleId, actions, timeoutSec, env, onProgress);
|
|
1558
|
+
const { created, timedOut, captureStderr } = await runExploreStreaming(bundleId, actions, timeoutSec, env, onProgress);
|
|
1545
1559
|
if (!created) return { error: "Exploration produced no capture run", details: { timedOut } };
|
|
1546
1560
|
|
|
1547
|
-
const report = buildQaReport(path.join(created.path, "ocqa-markers.txt"));
|
|
1561
|
+
const report = buildQaReport(path.join(created.path, "ocqa-markers.txt"), { platform: "ios", target: bundleId });
|
|
1548
1562
|
if (!report) {
|
|
1549
1563
|
return {
|
|
1550
1564
|
error: "No markers parsed from exploration (the app may not have launched)",
|
|
@@ -1574,13 +1588,14 @@ export async function runQaIos({ bundleId, maxActions, timeout, args = {}, surfa
|
|
|
1574
1588
|
// Cross-run regression vs. a caller-supplied baseline (the CI gate).
|
|
1575
1589
|
const regression = computeRegression(report.findings, args.baselineFindings);
|
|
1576
1590
|
const uiMap = await writeRunUiMap({ markersPath: path.join(created.path, "ocqa-markers.txt"), platform: "ios", target: bundleId, runId: created.id, outDir: created.path });
|
|
1591
|
+
const recording =
|
|
1592
|
+
["exploration.webm", "exploration.mov"].map((f) => path.join(created.path, f)).find((p) => fs.existsSync(p)) || null;
|
|
1593
|
+
const recordingWarning = recording ? null : recordingUnavailableReason(captureStderr);
|
|
1577
1594
|
let reportHtml = null;
|
|
1578
1595
|
try {
|
|
1579
1596
|
const { writeHtmlReport } = await import("./html-report.js");
|
|
1580
|
-
reportHtml = writeHtmlReport(created.path, { report, label: bundleId });
|
|
1597
|
+
reportHtml = writeHtmlReport(created.path, { report, label: bundleId, recordingWarning });
|
|
1581
1598
|
} catch { /* evidence page is best-effort */ }
|
|
1582
|
-
const recording =
|
|
1583
|
-
["exploration.webm", "exploration.mov"].map((f) => path.join(created.path, f)).find((p) => fs.existsSync(p)) || null;
|
|
1584
1599
|
const structured = {
|
|
1585
1600
|
...report,
|
|
1586
1601
|
regression,
|
|
@@ -1588,11 +1603,12 @@ export async function runQaIos({ bundleId, maxActions, timeout, args = {}, surfa
|
|
|
1588
1603
|
inputHint,
|
|
1589
1604
|
reportHtml,
|
|
1590
1605
|
recording,
|
|
1606
|
+
recordingWarning,
|
|
1591
1607
|
capture: { id: created.id, path: created.path, relativePath: created.relativePath },
|
|
1592
1608
|
timedOut,
|
|
1593
1609
|
autoBooted: sim.autoBooted || false,
|
|
1594
1610
|
};
|
|
1595
|
-
const text = formatQaReport(report, { regression, inputHint, timedOut, bundleId, aiConfigured: !!backend, reportHtml, recording, uiMap: uiMap.error ? null : uiMap, surface });
|
|
1611
|
+
const text = formatQaReport(report, { regression, inputHint, timedOut, bundleId, aiConfigured: !!backend, reportHtml, recording, recordingWarning, uiMap: uiMap.error ? null : uiMap, surface });
|
|
1596
1612
|
return { structured, text };
|
|
1597
1613
|
}
|
|
1598
1614
|
|
|
@@ -1753,6 +1769,8 @@ export async function runExploreTarget({
|
|
|
1753
1769
|
timeout,
|
|
1754
1770
|
testEmail,
|
|
1755
1771
|
testPassword,
|
|
1772
|
+
appLaunchArgs,
|
|
1773
|
+
appLaunchEnv,
|
|
1756
1774
|
baselineFindings,
|
|
1757
1775
|
surface = "cli",
|
|
1758
1776
|
onProgress = () => {},
|
|
@@ -1776,6 +1794,10 @@ export async function runExploreTarget({
|
|
|
1776
1794
|
catch (error) { return { error: error.message || String(error) }; }
|
|
1777
1795
|
const selectedPlatform = selected.platform;
|
|
1778
1796
|
|
|
1797
|
+
if (selectedPlatform !== "ios" && ((Array.isArray(appLaunchArgs) && appLaunchArgs.length) || (appLaunchEnv && Object.keys(appLaunchEnv).length))) {
|
|
1798
|
+
return { error: "appLaunchArgs/appLaunchEnv apply only to iOS targets." };
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1779
1801
|
if (selectedPlatform === "web") {
|
|
1780
1802
|
const ownedUrl = String(selected.runtime?.ownedUrl || "").trim();
|
|
1781
1803
|
if (/^https?:\/\//i.test(ownedUrl)) {
|
|
@@ -1818,7 +1840,7 @@ export async function runExploreTarget({
|
|
|
1818
1840
|
const resolved = await resolveAppTarget(root, { cwd: root, onStatus, scheme, configuration });
|
|
1819
1841
|
if (resolved.error) return resolved;
|
|
1820
1842
|
if (resolved.via) onStatus(`Target ${resolved.bundleId} — ${resolved.via}`);
|
|
1821
|
-
return runQaIos({ bundleId: resolved.bundleId, maxActions, timeout, args: { testEmail, testPassword, baselineFindings }, surface, onProgress });
|
|
1843
|
+
return runQaIos({ bundleId: resolved.bundleId, maxActions, timeout, args: { testEmail, testPassword, baselineFindings, appLaunchArgs, appLaunchEnv }, surface, onProgress });
|
|
1822
1844
|
}
|
|
1823
1845
|
|
|
1824
1846
|
function openLocalPort() {
|
|
@@ -2949,12 +2971,12 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
2949
2971
|
// MCP concerns: auth, arg validation, and progress notifications.
|
|
2950
2972
|
const progressToken = request.params && request.params._meta ? request.params._meta.progressToken : undefined;
|
|
2951
2973
|
const budget = Math.max(1, Math.min(1000, asInteger(args.maxActions, 60)));
|
|
2952
|
-
const notifyProgress = (
|
|
2974
|
+
const notifyProgress = (metric) => (p) => {
|
|
2953
2975
|
if (progressToken === undefined) return;
|
|
2954
2976
|
const total = p.max || budget;
|
|
2955
2977
|
server.notification({
|
|
2956
2978
|
method: "notifications/progress",
|
|
2957
|
-
params: { progressToken, progress: p.action || 0, total, message: `🔍 Exploring… ${p.action}/${total} actions · ${p.states} ${
|
|
2979
|
+
params: { progressToken, progress: p.action || 0, total, message: `🔍 Exploring… ${p.action}/${total} actions · ${p.states} ${metric}` },
|
|
2958
2980
|
}).catch(() => {});
|
|
2959
2981
|
};
|
|
2960
2982
|
if (wantsWeb) {
|
|
@@ -2965,7 +2987,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
2965
2987
|
testEmail: args.testEmail,
|
|
2966
2988
|
testPassword: args.testPassword,
|
|
2967
2989
|
baselineFindings: args.baselineFindings,
|
|
2968
|
-
onProgress: notifyProgress("pages"),
|
|
2990
|
+
onProgress: notifyProgress("pages reached"),
|
|
2969
2991
|
});
|
|
2970
2992
|
if (r.error) return errorResult(r.error, r.details || {});
|
|
2971
2993
|
return richResult(r.text, r.structured);
|
|
@@ -2982,14 +3004,14 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
2982
3004
|
testPassword: args.testPassword,
|
|
2983
3005
|
baselineFindings: args.baselineFindings,
|
|
2984
3006
|
clearData: args.clearData !== false,
|
|
2985
|
-
onProgress: notifyProgress("screens"),
|
|
3007
|
+
onProgress: notifyProgress("screens reached"),
|
|
2986
3008
|
});
|
|
2987
3009
|
if (r.error) return errorResult(r.error, r.details || {});
|
|
2988
3010
|
return richResult(r.text, r.structured);
|
|
2989
3011
|
}
|
|
2990
3012
|
|
|
2991
3013
|
let lastProgress = null;
|
|
2992
|
-
const iosProgress = notifyProgress("
|
|
3014
|
+
const iosProgress = notifyProgress("structural states observed");
|
|
2993
3015
|
const r = await runQaIos({
|
|
2994
3016
|
bundleId: String(args.appBundleId).trim(),
|
|
2995
3017
|
maxActions: args.maxActions,
|
|
@@ -3686,7 +3708,6 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
|
3686
3708
|
if (isNonEmptyString(args.apkPath)) await driver.install(path.resolve(args.apkPath));
|
|
3687
3709
|
const snap = await driver.launch({ clearData: args.clearData === true });
|
|
3688
3710
|
const data = await driver.screenshot();
|
|
3689
|
-
await driver.forceStop().catch(() => {});
|
|
3690
3711
|
return {
|
|
3691
3712
|
content: [
|
|
3692
3713
|
{ type: "text", text: `🚀 Launched \`${appId}\` (Android)\n\n` + formatScreen(snap.screenTitle, snap.elements) },
|
|
@@ -428,16 +428,17 @@ export function adoptPrCoverageProposal({ projectDir, prPlanPath, item, releaseP
|
|
|
428
428
|
if (!ground || !fs.existsSync(mapPath)) throw new Error("Coverage proposal requires the repository's persistent UI Map");
|
|
429
429
|
const map = JSON.parse(fs.readFileSync(mapPath, "utf8"));
|
|
430
430
|
const node = (map.nodes || []).find((candidate) => candidate.id === ground.id && candidate.status !== "proposed");
|
|
431
|
-
|
|
431
|
+
const refreshAdvice = "refresh the persistent UI Map with `tapp init --explore --refresh`, rerun `tapp pr gate`, then retry `tapp pr adopt`";
|
|
432
|
+
if (!node) throw new Error(`Coverage proposal UI Map node is stale or missing: ${ground.id}; ${refreshAdvice}`);
|
|
432
433
|
if (target.navigation?.route && !(node.routes || []).some((route) => route.platform === target.platform && route.path === target.navigation.route && route.replayable === true)) {
|
|
433
|
-
throw new Error(`Coverage proposal route is stale in the persistent UI Map: ${target.navigation.route}`);
|
|
434
|
+
throw new Error(`Coverage proposal route is stale in the persistent UI Map: ${target.navigation.route}; ${refreshAdvice}`);
|
|
434
435
|
}
|
|
435
436
|
if (target.navigation?.mode === "ui-map-path") {
|
|
436
437
|
const currentNavigation = replayableUiMapNavigation(map, node.id, target.platform);
|
|
437
438
|
const expectedEdges = (target.navigation.steps || []).map((step) => step.edgeId);
|
|
438
439
|
const currentEdges = (currentNavigation.steps || []).map((step) => step.edgeId);
|
|
439
440
|
if (currentNavigation.status !== "replayable" || JSON.stringify(expectedEdges) !== JSON.stringify(currentEdges)) {
|
|
440
|
-
throw new Error(`Coverage proposal UI Map path is stale for ${node.name}`);
|
|
441
|
+
throw new Error(`Coverage proposal UI Map path is stale for ${node.name}; ${refreshAdvice}`);
|
|
441
442
|
}
|
|
442
443
|
}
|
|
443
444
|
const releasePlan = JSON.parse(fs.readFileSync(targetPath, "utf8"));
|
package/mcp-server/src/report.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Pure report/gate logic shared by the MCP server (index.js) and the CI gate CLI
|
|
2
|
-
// (ci-report.js). Turns a capture's OCQA markers into
|
|
3
|
-
//
|
|
2
|
+
// (ci-report.js). Turns a capture's OCQA markers into a scoreless exploration observation,
|
|
3
|
+
// and applies explicit policy separately in the CI gate. No shell, no server —
|
|
4
4
|
// keep it dependency-free so the CI path stays importable and testable.
|
|
5
5
|
import fs from "fs";
|
|
6
6
|
import path from "path";
|
|
@@ -133,9 +133,9 @@ export function observationSummary(report) {
|
|
|
133
133
|
return `${report?.screensExplored || 0} screens · ${report?.actionsPerformed || 0} actions · ${n} finding(s) · observation only`;
|
|
134
134
|
}
|
|
135
135
|
|
|
136
|
-
// Turn a capture's OCQA markers into
|
|
137
|
-
//
|
|
138
|
-
export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
136
|
+
// Turn a capture's OCQA markers into a scoreless observation with deduped findings and an
|
|
137
|
+
// explicit coverage floor. Release judgment is applied later by evaluateGate.
|
|
138
|
+
export function buildQaReport(markersFilePath, { platform = "ios", target = null } = {}) {
|
|
139
139
|
const base = parseOcqaMarkers(markersFilePath);
|
|
140
140
|
if (!base) return null;
|
|
141
141
|
|
|
@@ -145,6 +145,7 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
145
145
|
const inputsByScreen = new Map();
|
|
146
146
|
const screenElementCounts = {}; // screen -> max elements observed (content-collapse detection)
|
|
147
147
|
let anySecure = false;
|
|
148
|
+
let loginAttempted = false;
|
|
148
149
|
let actions = 0;
|
|
149
150
|
|
|
150
151
|
for (const line of raw.split(/\r?\n/)) {
|
|
@@ -163,6 +164,17 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
163
164
|
}
|
|
164
165
|
} else if (t.startsWith("OCQA_ACTION:")) {
|
|
165
166
|
actions += 1;
|
|
167
|
+
try {
|
|
168
|
+
const action = JSON.parse(t.slice("OCQA_ACTION:".length));
|
|
169
|
+
if (action?.type === "login" || String(action?.type || "").startsWith("login_")) loginAttempted = true;
|
|
170
|
+
} catch {
|
|
171
|
+
/* ignore malformed */
|
|
172
|
+
}
|
|
173
|
+
} else if (
|
|
174
|
+
t === "OCQA_STATE:login_preamble_submitted" ||
|
|
175
|
+
t === "OCQA_STATE:login_preamble_two_step_submitted"
|
|
176
|
+
) {
|
|
177
|
+
loginAttempted = true;
|
|
166
178
|
} else if (t.startsWith("OCQA_STATE:{")) {
|
|
167
179
|
try {
|
|
168
180
|
const s = JSON.parse(t.slice("OCQA_STATE:".length));
|
|
@@ -202,7 +214,12 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
202
214
|
const missingResources = new Set(normalizedIssues
|
|
203
215
|
.filter((issue) => issue.type === "missing_asset" && issue.target)
|
|
204
216
|
.map((issue) => issue.target));
|
|
217
|
+
// Older native captures represented the caller's wall-clock budget as a high-severity app
|
|
218
|
+
// finding. A timeout makes the evidence partial/inconclusive; it does not prove an app
|
|
219
|
+
// performance defect (the harness has a separate app_hang detector for that).
|
|
220
|
+
const timeBudgetExhausted = base.complete?.timedOut === true || normalizedIssues.some((issue) => issue.type === "explore_timeout");
|
|
205
221
|
const reportIssues = normalizedIssues.filter((issue) =>
|
|
222
|
+
issue.type !== "explore_timeout" &&
|
|
206
223
|
!(issue.type === "network_error" && issue.target && missingResources.has(issue.target) && /^Request failed:/i.test(String(issue.title || ""))));
|
|
207
224
|
|
|
208
225
|
// Dedup by stable signature (type|screen|target) so repeated detections count once —
|
|
@@ -226,8 +243,13 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
226
243
|
findings.sort((a, b) => severityRank(a.severity) - severityRank(b.severity));
|
|
227
244
|
|
|
228
245
|
const screensExplored = screens.size || base.uniqueScreens.length;
|
|
229
|
-
|
|
230
|
-
|
|
246
|
+
// Some native recovery operations are counted by the harness budget but intentionally do not
|
|
247
|
+
// emit a public action narrative. Preserve the larger authoritative completion count instead of
|
|
248
|
+
// understating coverage whenever at least one narrated action exists.
|
|
249
|
+
const actionsPerformed = Math.max(
|
|
250
|
+
actions,
|
|
251
|
+
base.complete && typeof base.complete === "object" ? base.complete.actions || 0 : 0,
|
|
252
|
+
);
|
|
231
253
|
const crit = findings.filter((f) => f.severity === "critical").length;
|
|
232
254
|
const high = findings.filter((f) => f.severity === "high").length;
|
|
233
255
|
const med = findings.filter((f) => f.severity === "medium").length;
|
|
@@ -244,9 +266,22 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
244
266
|
// Coverage floor: exploration is inconclusive if the app wasn't actually exercised. Exploration
|
|
245
267
|
// OBSERVES — it does not render a ship verdict or score (ADR-0005). Judgment (pass/fail/
|
|
246
268
|
// inconclusive) is the gate's job (evaluateGate), computed from these findings + coverage + policy.
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
269
|
+
// A one-page web target can still be swept exhaustively: page errors, requests, links, assets,
|
|
270
|
+
// placeholder anchors, and visible controls do not require a second route. Native exploration
|
|
271
|
+
// retains the stronger multi-screen/action floor. A credentialless single-screen login remains
|
|
272
|
+
// inconclusive so a login wall can never turn into a clean pass.
|
|
273
|
+
const coverageFloorMet = platform === "web"
|
|
274
|
+
? screensExplored >= 1 && actionsPerformed >= 1
|
|
275
|
+
: screensExplored >= 2 && actionsPerformed >= 3;
|
|
276
|
+
const credentiallessLoginWall = anySecure && !loginAttempted && screensExplored <= 1;
|
|
277
|
+
const inconclusive = !coverageFloorMet || credentiallessLoginWall || timeBudgetExhausted;
|
|
278
|
+
const stopReason = credentiallessLoginWall ? "login-wall-no-credentials"
|
|
279
|
+
: timeBudgetExhausted ? "time-budget-exhausted"
|
|
280
|
+
: coverageFloorMet ? "completed" : "coverage-floor-not-met";
|
|
281
|
+
|
|
282
|
+
const headline = timeBudgetExhausted
|
|
283
|
+
? `Inconclusive — exploration reached its ${base.complete?.timeoutSeconds || "configured"}s time budget after ${actionsPerformed} action(s) across ${screensExplored} screen(s). Findings are partial; this is not an app performance finding. Increase --timeout or request fewer actions.`
|
|
284
|
+
: inconclusive
|
|
250
285
|
? `Inconclusive — only ${screensExplored} screen(s) / ${actionsPerformed} action(s) explored. The app may have crashed on launch, be stuck behind a sign-in wall, or otherwise prevent exploration. Absence of issues is NOT a pass.`
|
|
251
286
|
: findings.length === 0
|
|
252
287
|
? platform === "web"
|
|
@@ -254,7 +289,7 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
254
289
|
: "No issues surfaced in the exercised surfaces. An observation, not a release decision."
|
|
255
290
|
: `${findings.length} issue(s) surfaced for review (${crit} critical, ${high} high, ${med} medium, ${low} low). An observation, not a release decision.`;
|
|
256
291
|
|
|
257
|
-
// The
|
|
292
|
+
// The observation's honesty label: exactly which defect classes this run checked, which
|
|
258
293
|
// it structurally could NOT check, and which conditions never came up — so "checked" is
|
|
259
294
|
// never claimed for a state the run didn't reach. Platform-aware: a web run doesn't
|
|
260
295
|
// inherit iOS keyboard assertions and vice versa.
|
|
@@ -300,9 +335,12 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
300
335
|
"content & reachability regressions require a baseline" ,
|
|
301
336
|
];
|
|
302
337
|
}
|
|
303
|
-
// "Failed sign-ins" is only a claim when
|
|
304
|
-
|
|
338
|
+
// "Failed sign-ins" is only a claim when credentials were actually submitted. Merely seeing a
|
|
339
|
+
// password field proves that a login surface was reached, not that authentication was exercised.
|
|
340
|
+
if (loginAttempted) checkedFor.splice(2, 0, "failed sign-ins");
|
|
341
|
+
else if (anySecure) notChecked.push("sign-in behavior (login form reached, no test credentials supplied)");
|
|
305
342
|
else conditionsNotReached.push("sign-in (no login form encountered this run)");
|
|
343
|
+
if (timeBudgetExhausted) notChecked.push("the full requested action budget (run reached its wall-clock timeout)");
|
|
306
344
|
|
|
307
345
|
return {
|
|
308
346
|
// An ExplorationRun observation: findings + coverage + evidence, NO ship verdict or score
|
|
@@ -313,7 +351,7 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
313
351
|
// ended; coverage/evidence/uiMap/comparison are the structured observation. uiMap and comparison
|
|
314
352
|
// are populated by consumers that build the map / diff a baseline (null in the bare observation).
|
|
315
353
|
runStatus: inconclusive ? "limited" : "completed",
|
|
316
|
-
stopReason
|
|
354
|
+
stopReason,
|
|
317
355
|
headline,
|
|
318
356
|
inconclusive,
|
|
319
357
|
coverage: { screensExplored, actionsPerformed, screens: Array.from(screens) },
|
|
@@ -324,6 +362,7 @@ export function buildQaReport(markersFilePath, { platform = "ios" } = {}) {
|
|
|
324
362
|
notChecked,
|
|
325
363
|
conditionsNotReached,
|
|
326
364
|
platform,
|
|
365
|
+
target: typeof target === "string" && target.trim() ? target.trim() : null,
|
|
327
366
|
screensExplored,
|
|
328
367
|
actionsPerformed,
|
|
329
368
|
findingCounts: { critical: crit, high, medium: med, low, total: findings.length },
|
|
@@ -453,7 +492,7 @@ export function computeRegression(current, baseline) {
|
|
|
453
492
|
// the evidence." Exit codes are the CI contract; precedence is fail > inconclusive > pass.
|
|
454
493
|
export const GATE_EXIT = { pass: 0, fail: 1, error: 2, inconclusive: 3 };
|
|
455
494
|
// Bump when the gate's decision semantics change (NOT the npm version). Recorded on every GateRun.
|
|
456
|
-
export const GATE_POLICY_VERSION = "
|
|
495
|
+
export const GATE_POLICY_VERSION = "2";
|
|
457
496
|
|
|
458
497
|
// Pure gate evaluator: frozen evidence + policy → a GateRun decision. Extracted verbatim from the
|
|
459
498
|
// former inline logic in ci-report.js so the `[char]` characterization tests keep passing — the
|
|
@@ -484,6 +523,16 @@ export function evaluateGate({ report, regression = null, flows = [], scenarios
|
|
|
484
523
|
const needsReview = suites.filter((s) => classify(s) === "needs-review");
|
|
485
524
|
if (needsReview.length) inconclusive(`${needsReview.length} ${label}(s) contain assert_ai (model-observed); the deterministic gate cannot decide them — review, or add an explicit probabilistic policy`);
|
|
486
525
|
}
|
|
526
|
+
// A submitted sign-in that remains on the login surface is an explicit exercised guarantee,
|
|
527
|
+
// not ordinary pre-existing UI debt. Letting it pass without a baseline would produce the
|
|
528
|
+
// contradictory public result "failed sign-in detected" + gate PASS. Keep sampled probes out,
|
|
529
|
+
// but fail every deterministic auth failure under every gate policy (including with a baseline).
|
|
530
|
+
const authFailures = (report.findings || []).filter((finding) =>
|
|
531
|
+
finding?.type === "auth_failed" &&
|
|
532
|
+
finding?.authority !== "model-observed" &&
|
|
533
|
+
finding?.evaluationTier !== "sampled"
|
|
534
|
+
);
|
|
535
|
+
if (authFailures.length) fail(`${authFailures.length} deterministic sign-in attempt(s) failed`);
|
|
487
536
|
// Selected-but-unexecuted work is missing evidence, not an observed violation → inconclusive.
|
|
488
537
|
if (prPlan?.execution?.notRun) inconclusive(`${prPlan.execution.notRun} selected release contract(s) did not run`);
|
|
489
538
|
if (prPlan?.execution?.explorationFailed) inconclusive(`${prPlan.execution.explorationFailed} planned PR exploration target(s) failed or were not reached`);
|
|
@@ -45,7 +45,7 @@ export function webPlaceholderLinkFindings(links = []) {
|
|
|
45
45
|
const seen = new Set();
|
|
46
46
|
for (const link of links || []) {
|
|
47
47
|
const rawHref = String(link?.rawHref || "").trim().toLowerCase();
|
|
48
|
-
const placeholder = rawHref === "#" || /^javascript:(?:void\(0\);?|;?)$/.test(rawHref);
|
|
48
|
+
const placeholder = rawHref === "" || rawHref === "#" || /^javascript:(?:void\(0\);?|;?)$/.test(rawHref);
|
|
49
49
|
if (!placeholder || link?.handlerHint) continue;
|
|
50
50
|
const label = String(link?.label || "").replace(/\s+/g, " ").trim().slice(0, 100);
|
|
51
51
|
const fingerprint = String(link?.fingerprint || "link").replace(/\s+/g, " ").trim().slice(0, 100) || "link";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aarwitz/tapp",
|
|
3
|
-
"version": "0.17.0-rc.
|
|
3
|
+
"version": "0.17.0-rc.2",
|
|
4
4
|
"mcpName": "io.github.aarwitz/tapp",
|
|
5
5
|
"description": "Release contracts, autonomous QA, and evidence-backed CI gates for iOS, Android, and web.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -87,7 +87,7 @@
|
|
|
87
87
|
"mobile"
|
|
88
88
|
],
|
|
89
89
|
"scripts": {
|
|
90
|
-
"test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/cloud-runner.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
|
|
90
|
+
"test": "node --test tests/report.test.js tests/regression.test.js tests/engine.test.js tests/project-config.test.js tests/application-model.test.js tests/ui-map.test.js tests/task-runtime.test.js tests/release-contract.test.js tests/pr-selection.test.js tests/flow-runtime.test.js tests/web-explorer.test.js tests/web-session.test.js tests/web-flow.test.js tests/scenario-runtime.test.js tests/android-driver.test.js tests/android-explorer.test.js tests/android-flow.test.js tests/android-primitives-protocol.test.js tests/managed-web.test.js tests/product-operations.test.js tests/browser-product.test.js tests/browser-onboarding.test.js tests/managed-operation.test.js tests/cloud-runner.test.js tests/ci-setup.test.js tests/ci-install.test.js tests/cli.test.js tests/action.test.js tests/package-surface.test.js tests/ci-gate.test.js tests/ci-report.test.js tests/desktop-protocol.test.js tests/ios-flow-protocol.test.js",
|
|
91
91
|
"test:browser-journey": "node --test tests/browser-journey.test.js",
|
|
92
92
|
"test:browser-native": "TAPP_RUN_NATIVE_BROWSER=1 node --test tests/browser-native-journey.test.js"
|
|
93
93
|
}
|
package/scripts/ci-gate.sh
CHANGED
|
@@ -3,9 +3,11 @@
|
|
|
3
3
|
#
|
|
4
4
|
# Boots a simulator if needed, installs the app build under test, runs the autonomous exploration
|
|
5
5
|
# harness, replays every committed Flow (deterministic E2E tests), diffs the findings against a
|
|
6
|
-
# stored baseline, writes a GitHub Actions step summary, and exits
|
|
7
|
-
#
|
|
8
|
-
#
|
|
6
|
+
# stored baseline, writes a GitHub Actions step summary, and exits with the public outcome contract:
|
|
7
|
+
# pass 0, deterministic fail 1, infrastructure/usage error 2, inconclusive evidence 3. Absolute
|
|
8
|
+
# blockers and reviewed Flow/Scenario/Contract failures are enforced even without a baseline;
|
|
9
|
+
# baseline comparisons additionally catch new high/critical regressions. Wrapped by ../action.yml
|
|
10
|
+
# for GitHub Actions; equally usable from any other CI or locally.
|
|
9
11
|
#
|
|
10
12
|
# Usage:
|
|
11
13
|
# scripts/ci-gate.sh [--platform ios] --app <path/to/App.app> [--bundle-id <com.example.app>]
|
|
@@ -35,7 +37,7 @@ set -uo pipefail
|
|
|
35
37
|
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
36
38
|
|
|
37
39
|
usage() {
|
|
38
|
-
sed -n '2
|
|
40
|
+
sed -n '2,/^set -uo pipefail$/p' "$0" | sed '$d; s/^# \{0,1\}//'
|
|
39
41
|
}
|
|
40
42
|
|
|
41
43
|
PLATFORM="ios" APP_PATH="" BUNDLE_ID="" APK_PATH="" APP_ID="" URL="" WEB_TARGET="" TARGET_KEY="" SERIAL="" ACTIONS=40 TIMEOUT=600 FLOWS="" SCENARIOS="" CONTRACTS="" PROJECT_DIR="" BASELINE="" FAIL_ON="gate" JSON_OUT="" MD_OUT="" DEVICE="iPhone 16 Pro" PR_BASE="" PR_HEAD="HEAD" CHANGED_FILES_FILE="" PR_PLAN_OUT=""
|
|
@@ -82,7 +84,6 @@ fi
|
|
|
82
84
|
TAPP_PROJECT_ARTIFACTS=""
|
|
83
85
|
if [[ -n "$PROJECT_DIR" ]]; then
|
|
84
86
|
[[ -d "$PROJECT_DIR/.tapp" ]] && TAPP_PROJECT_ARTIFACTS="$PROJECT_DIR/.tapp"
|
|
85
|
-
[[ -z "$TAPP_PROJECT_ARTIFACTS" && -d "$PROJECT_DIR/.autotap" ]] && TAPP_PROJECT_ARTIFACTS="$PROJECT_DIR/.autotap"
|
|
86
87
|
fi
|
|
87
88
|
[[ -z "$FLOWS" && -n "$TAPP_PROJECT_ARTIFACTS" && -d "$TAPP_PROJECT_ARTIFACTS/flows" ]] && FLOWS="$TAPP_PROJECT_ARTIFACTS/flows/*.yml"
|
|
88
89
|
[[ "$PLATFORM" == "web" && -z "$SCENARIOS" && -n "$TAPP_PROJECT_ARTIFACTS" && -d "$TAPP_PROJECT_ARTIFACTS/scenarios" ]] && SCENARIOS="$TAPP_PROJECT_ARTIFACTS/scenarios/*.yml"
|
package/scripts/platform-gate.js
CHANGED
|
@@ -37,7 +37,10 @@ for (let i = 2; i < process.argv.length; i += 1) {
|
|
|
37
37
|
else if (key === "--json-out") args.jsonOut = value;
|
|
38
38
|
else if (key === "--md-out") args.mdOut = value;
|
|
39
39
|
else if (key === "--pr-plan") args.prPlan = value;
|
|
40
|
-
else
|
|
40
|
+
else {
|
|
41
|
+
console.error(`Unknown argument: ${key}`);
|
|
42
|
+
process.exit(2);
|
|
43
|
+
}
|
|
41
44
|
}
|
|
42
45
|
if (!["web", "android"].includes(args.platform)) throw new Error("--platform must be web|android");
|
|
43
46
|
if (args.platform === "web" && !args.url && !args.projectDir) throw new Error("Web gate requires --url or --project-dir for managed build/start");
|
|
@@ -94,7 +97,7 @@ if (args.prPlan) {
|
|
|
94
97
|
}
|
|
95
98
|
|
|
96
99
|
let managedRuntime = null;
|
|
97
|
-
let exitCode =
|
|
100
|
+
let exitCode = 2;
|
|
98
101
|
try {
|
|
99
102
|
if (args.platform === "web" && !args.url) {
|
|
100
103
|
const started = await startManagedWebTarget({
|
|
@@ -106,7 +109,7 @@ try {
|
|
|
106
109
|
if (started.error) {
|
|
107
110
|
console.error(`❌ ${started.error}`);
|
|
108
111
|
if (started.details?.remediation) console.error(` ${started.details.remediation}`);
|
|
109
|
-
|
|
112
|
+
exitCode = 2;
|
|
110
113
|
} else {
|
|
111
114
|
managedRuntime = started;
|
|
112
115
|
args.url = started.url;
|
|
@@ -114,7 +117,7 @@ try {
|
|
|
114
117
|
}
|
|
115
118
|
}
|
|
116
119
|
if (args.platform === "web" && !args.url) {
|
|
117
|
-
exitCode =
|
|
120
|
+
exitCode = 2;
|
|
118
121
|
} else {
|
|
119
122
|
const qa = args.platform === "web"
|
|
120
123
|
? await runQaWeb({ url: args.url, maxActions: args.actions, timeout: args.timeout, testEmail: process.env.OCQA_TEST_EMAIL, testPassword: process.env.OCQA_TEST_PASSWORD, seedTargets: prExplorationTargets })
|
|
@@ -122,7 +125,7 @@ try {
|
|
|
122
125
|
testEmail: process.env.OCQA_TEST_EMAIL, testPassword: process.env.OCQA_TEST_PASSWORD, seedTargets: prExplorationTargets });
|
|
123
126
|
if (qa.error) {
|
|
124
127
|
console.error(`❌ ${qa.error}`);
|
|
125
|
-
exitCode =
|
|
128
|
+
exitCode = 2;
|
|
126
129
|
} else {
|
|
127
130
|
const captureDir = qa.structured.capture.path;
|
|
128
131
|
const markers = path.join(captureDir, "ocqa-markers.txt");
|
|
@@ -177,6 +180,9 @@ try {
|
|
|
177
180
|
exitCode = report.status ?? 1;
|
|
178
181
|
}
|
|
179
182
|
}
|
|
183
|
+
} catch (error) {
|
|
184
|
+
console.error(`❌ ${error?.message || String(error)}`);
|
|
185
|
+
exitCode = 2;
|
|
180
186
|
} finally {
|
|
181
187
|
if (managedRuntime) {
|
|
182
188
|
await stopManagedWebTarget(managedRuntime);
|
package/scripts/quick-capture.sh
CHANGED
|
@@ -262,58 +262,92 @@ case "$MODE" in
|
|
|
262
262
|
echo "Running autonomous exploration ($MAX_ACTIONS actions, timeout: ${EXPLORE_TIMEOUT}s, app: $APP_BUNDLE)..."
|
|
263
263
|
ensure_harness_built "$SIM_NAME"
|
|
264
264
|
|
|
265
|
-
#
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
if
|
|
273
|
-
|
|
274
|
-
|
|
265
|
+
# XCUITest can block inside `app.launch()` until the outer watchdog when the target dies in its
|
|
266
|
+
# initializer, producing a misleading timeout after minutes. For an ordinary unconfigured
|
|
267
|
+
# launch, probe the target directly first and verify that the returned PID survives a short
|
|
268
|
+
# settle window. (Configured launch args/env skip this probe because simctl would not reproduce
|
|
269
|
+
# that launch contract.) The harness still performs its own fresh launch for healthy apps.
|
|
270
|
+
PREFLIGHT_CRASH=0
|
|
271
|
+
PREFLIGHT_OUTPUT=""
|
|
272
|
+
if [[ -z "${OCQA_APP_LAUNCH_ARGS_JSON:-}" && -z "${OCQA_APP_LAUNCH_ENV_JSON:-}" ]]; then
|
|
273
|
+
xcrun simctl terminate "$UDID" "$APP_BUNDLE" >/dev/null 2>&1 || true
|
|
274
|
+
PREFLIGHT_OUTPUT="$(xcrun simctl launch "$UDID" "$APP_BUNDLE" 2>&1 || true)"
|
|
275
|
+
PREFLIGHT_PID="$(echo "$PREFLIGHT_OUTPUT" | sed -n 's/.*: \([0-9][0-9]*\)$/\1/p' | tail -1)"
|
|
276
|
+
if [[ -n "$PREFLIGHT_PID" ]]; then
|
|
277
|
+
sleep 2
|
|
278
|
+
if ! kill -0 "$PREFLIGHT_PID" 2>/dev/null; then PREFLIGHT_CRASH=1; fi
|
|
279
|
+
fi
|
|
280
|
+
xcrun simctl terminate "$UDID" "$APP_BUNDLE" >/dev/null 2>&1 || true
|
|
275
281
|
fi
|
|
276
282
|
|
|
277
|
-
# Run exploration with watchdog timeout to avoid silent hangs.
|
|
278
283
|
local_output_file="$CAPTURE_DIR/harness-output.txt"
|
|
279
|
-
run_harness_test "testAutonomousExploration" "$SIM_NAME" "$APP_BUNDLE" "$MAX_ACTIONS" "$EXPLORE_TIMEOUT" > "$local_output_file" 2>&1 &
|
|
280
|
-
HARNESS_PID=$!
|
|
281
|
-
|
|
282
|
-
START_TS=$(date +%s)
|
|
283
284
|
TIMED_OUT=0
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
285
|
+
RECORD_PID=""
|
|
286
|
+
if [[ "$PREFLIGHT_CRASH" -eq 1 ]]; then
|
|
287
|
+
OUTPUT="OCQA_ISSUE:{\"type\":\"crash\",\"severity\":\"critical\",\"title\":\"App crashed during launch preflight\",\"screen\":\"Launch\",\"step\":0}
|
|
288
|
+
OCQA_COMPLETE:{\"actions\":0,\"states\":0,\"issues\":1,\"screens\":\"\",\"outcome\":\"launch_crash\"}"
|
|
289
|
+
printf '%s\n%s\n' "$PREFLIGHT_OUTPUT" "$OUTPUT" > "$local_output_file"
|
|
290
|
+
echo "WARNING: Target process exited during launch preflight; recorded a crash instead of waiting for the exploration timeout." >&2
|
|
291
|
+
else
|
|
292
|
+
|
|
293
|
+
# Start video recording in background
|
|
294
|
+
cleanup_stale_recorders "$UDID"
|
|
295
|
+
if xcrun simctl io "$UDID" recordVideo --codec=h264 "$CAPTURE_DIR/exploration.mov" & then
|
|
296
|
+
RECORD_PID=$!
|
|
297
|
+
fi
|
|
298
|
+
sleep 0.5
|
|
299
|
+
if ! kill -0 "$RECORD_PID" 2>/dev/null; then
|
|
300
|
+
echo "WARNING: Could not start simulator video recording. Continuing without video." >&2
|
|
301
|
+
RECORD_PID=""
|
|
293
302
|
fi
|
|
294
|
-
sleep 2
|
|
295
|
-
done
|
|
296
303
|
|
|
297
|
-
|
|
298
|
-
|
|
304
|
+
# Run exploration with watchdog timeout to avoid silent hangs.
|
|
305
|
+
run_harness_test "testAutonomousExploration" "$SIM_NAME" "$APP_BUNDLE" "$MAX_ACTIONS" "$EXPLORE_TIMEOUT" > "$local_output_file" 2>&1 &
|
|
306
|
+
HARNESS_PID=$!
|
|
307
|
+
|
|
308
|
+
START_TS=$(date +%s)
|
|
309
|
+
while kill -0 "$HARNESS_PID" 2>/dev/null; do
|
|
310
|
+
NOW_TS=$(date +%s)
|
|
311
|
+
ELAPSED=$((NOW_TS - START_TS))
|
|
312
|
+
if [[ "$ELAPSED" -ge "$EXPLORE_TIMEOUT" ]]; then
|
|
313
|
+
TIMED_OUT=1
|
|
314
|
+
kill -TERM "$HARNESS_PID" 2>/dev/null || true
|
|
315
|
+
sleep 2
|
|
316
|
+
kill -KILL "$HARNESS_PID" 2>/dev/null || true
|
|
317
|
+
break
|
|
318
|
+
fi
|
|
319
|
+
sleep 2
|
|
320
|
+
done
|
|
321
|
+
|
|
322
|
+
wait "$HARNESS_PID" 2>/dev/null || true
|
|
323
|
+
OUTPUT="$(cat "$local_output_file" 2>/dev/null || true)"
|
|
299
324
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
325
|
+
# Stop recording
|
|
326
|
+
if [[ -n "$RECORD_PID" ]]; then
|
|
327
|
+
kill -INT "$RECORD_PID" 2>/dev/null || true
|
|
328
|
+
wait "$RECORD_PID" 2>/dev/null || true
|
|
329
|
+
fi
|
|
330
|
+
sleep 1
|
|
304
331
|
fi
|
|
305
|
-
sleep 1
|
|
306
332
|
|
|
307
333
|
# Parse OCQA_ markers
|
|
308
334
|
echo "$OUTPUT" | grep "^OCQA_" > "$CAPTURE_DIR/ocqa-markers.txt" || true
|
|
309
335
|
if [[ "$TIMED_OUT" -eq 1 ]]; then
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
336
|
+
# The caller's wall-clock budget expiring means the evidence is partial; it is not proof that
|
|
337
|
+
# the app itself is slow or hung. Preserve observed coverage in the terminal marker and let
|
|
338
|
+
# report.js classify the run as inconclusive instead of inventing an app finding.
|
|
339
|
+
TIMED_ACTIONS="$(echo "$OUTPUT" | sed -n 's/^OCQA_PROGRESS:{"action":\([0-9][0-9]*\).*/\1/p' | tail -1)"
|
|
340
|
+
TIMED_STATES="$(echo "$OUTPUT" | sed -n 's/^OCQA_PROGRESS:.*"states":\([0-9][0-9]*\).*/\1/p' | tail -1)"
|
|
341
|
+
TIMED_ISSUES="$(echo "$OUTPUT" | grep -c '^OCQA_ISSUE:' || true)"
|
|
342
|
+
TIMED_ACTIONS="${TIMED_ACTIONS:-0}"
|
|
343
|
+
TIMED_STATES="${TIMED_STATES:-0}"
|
|
344
|
+
TIMED_ISSUES="${TIMED_ISSUES:-0}"
|
|
345
|
+
echo "OCQA_COMPLETE:{\"actions\":$TIMED_ACTIONS,\"states\":$TIMED_STATES,\"issues\":$TIMED_ISSUES,\"screens\":\"\",\"timedOut\":true,\"timeoutSeconds\":$EXPLORE_TIMEOUT}" >> "$CAPTURE_DIR/ocqa-markers.txt"
|
|
346
|
+
echo "WARNING: Exploration reached its ${EXPLORE_TIMEOUT}s time budget; evidence is partial" >&2
|
|
313
347
|
fi
|
|
314
348
|
echo "$OUTPUT" > "$CAPTURE_DIR/full-output.txt"
|
|
315
349
|
|
|
316
|
-
COMPLETE_LINE=$(
|
|
350
|
+
COMPLETE_LINE=$(grep "OCQA_COMPLETE" "$CAPTURE_DIR/ocqa-markers.txt" | tail -1 || true)
|
|
317
351
|
if [[ -n "$COMPLETE_LINE" ]]; then
|
|
318
352
|
echo ""
|
|
319
353
|
echo "Exploration complete: $COMPLETE_LINE"
|