@aarwitz/tapp 0.16.5 → 0.17.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +35 -21
- package/Harness/OCQAHarnessUITests/ExplorerTests.swift +100 -31
- package/README.md +70 -67
- package/bin/tapp.js +265 -74
- package/browser/app.js +12 -6
- package/docs/BROWSER-PRODUCT.md +75 -0
- package/docs/PRODUCT-ENGINE.md +107 -0
- package/docs/application-model.md +278 -0
- package/docs/scenarios.md +95 -0
- package/mcp-server/src/android-driver.js +88 -3
- package/mcp-server/src/android-explorer.js +83 -14
- package/mcp-server/src/application-model.js +13 -9
- package/mcp-server/src/browser-product.js +1 -1
- package/mcp-server/src/ci-report.js +84 -62
- package/mcp-server/src/ci-setup.js +35 -5
- package/mcp-server/src/enrich.js +1 -1
- package/mcp-server/src/html-report.js +41 -7
- package/mcp-server/src/index.js +198 -72
- package/mcp-server/src/pr-selection.js +4 -3
- package/mcp-server/src/product-execution.js +1 -1
- package/mcp-server/src/product-operations.js +96 -7
- package/mcp-server/src/project-config.js +1 -2
- package/mcp-server/src/project-paths.js +5 -17
- package/mcp-server/src/release-contract.js +3 -3
- package/mcp-server/src/report.js +185 -51
- package/mcp-server/src/task-runtime.js +1 -1
- package/mcp-server/src/web-explorer.js +1 -1
- package/package.json +2 -2
- package/scripts/ci-gate.sh +11 -9
- package/scripts/flow_ai_judge.py +1 -1
- package/scripts/platform-gate.js +11 -5
- package/scripts/quick-capture.sh +72 -38
- package/scripts/run-flow.sh +1 -1
|
@@ -41,6 +41,16 @@ function runFile(command, args, { encoding = "utf8", timeout = 30_000, maxBuffer
|
|
|
41
41
|
});
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
export function parseLatestAndroidCrashExitInfo(output) {
|
|
45
|
+
const blocks = String(output || "").split(/ApplicationExitInfo #\d+:/).slice(1);
|
|
46
|
+
for (const block of blocks) {
|
|
47
|
+
const reason = block.match(/\breason=(4|5)\s+\((?:APP CRASH|NATIVE CRASH)/);
|
|
48
|
+
const identity = block.match(/\btimestamp=([^\n]+?)\s+pid=(\d+)\b/);
|
|
49
|
+
if (reason && identity) return `${identity[1].trim()}|${identity[2]}|${reason[1]}`;
|
|
50
|
+
}
|
|
51
|
+
return null;
|
|
52
|
+
}
|
|
53
|
+
|
|
44
54
|
function entityDecode(value) {
|
|
45
55
|
return String(value || "")
|
|
46
56
|
.replaceAll(""", '"').replaceAll("'", "'")
|
|
@@ -118,8 +128,22 @@ export function detectAndroidScreen(elements, activity = "") {
|
|
|
118
128
|
|
|
119
129
|
export function isAndroidAppSnapshot(snapshot, appId) {
|
|
120
130
|
if (!snapshot || !appId) return false;
|
|
121
|
-
|
|
122
|
-
|
|
131
|
+
const activity = String(snapshot.activity || "");
|
|
132
|
+
const ownsActivity = activity.startsWith(`${appId}/`);
|
|
133
|
+
const packages = new Set((snapshot.elements || []).map((e) => e.package).filter(Boolean));
|
|
134
|
+
const ownsElements = packages.has(appId);
|
|
135
|
+
// dumpXml and dumpsys used to run concurrently. If an app crashed while they were sampled, a
|
|
136
|
+
// stale activity from the dead app could be paired with another app's UI tree and Tapp would map
|
|
137
|
+
// that unrelated app as the crash destination. When both signals exist, require agreement.
|
|
138
|
+
if (activity && packages.size) return ownsActivity && ownsElements;
|
|
139
|
+
return ownsActivity || ownsElements;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function isSystemOverlayOverApp(snapshot, appId) {
|
|
143
|
+
if (!snapshot || !appId || !String(snapshot.activity || "").startsWith(`${appId}/`)) return false;
|
|
144
|
+
const packages = new Set((snapshot.elements || []).map((element) => element.package).filter(Boolean));
|
|
145
|
+
return packages.size > 0 && !packages.has(appId)
|
|
146
|
+
&& [...packages].every((name) => name === "android" || name.startsWith("com.android.systemui"));
|
|
123
147
|
}
|
|
124
148
|
|
|
125
149
|
export class AndroidDriver {
|
|
@@ -158,6 +182,19 @@ export class AndroidDriver {
|
|
|
158
182
|
if (this.appId) await this.adb(["shell", "am", "force-stop", this.appId]);
|
|
159
183
|
}
|
|
160
184
|
|
|
185
|
+
async isProcessAlive() {
|
|
186
|
+
if (!this.appId) return false;
|
|
187
|
+
const r = await this.adb(["shell", "pidof", this.appId]);
|
|
188
|
+
return r.code === 0 && /\d/.test(String(r.stdout || ""));
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
async latestCrashExitInfo() {
|
|
192
|
+
if (!this.appId) return null;
|
|
193
|
+
const r = await this.adb(["shell", "dumpsys", "activity", "exit-info", this.appId]);
|
|
194
|
+
if (r.code !== 0) return null;
|
|
195
|
+
return parseLatestAndroidCrashExitInfo(r.stdout);
|
|
196
|
+
}
|
|
197
|
+
|
|
161
198
|
async clearData() {
|
|
162
199
|
if (!this.appId) throw new Error("appId is required to clear Android app data");
|
|
163
200
|
const r = await this.adb(["shell", "pm", "clear", this.appId]);
|
|
@@ -175,7 +212,42 @@ export class AndroidDriver {
|
|
|
175
212
|
const r = await this.adb(["shell", "am", "start", "-W", "-n", component], { timeout: 30_000 });
|
|
176
213
|
if (r.code !== 0 || !/Status:\s*ok/i.test(String(r.stdout))) throw new Error((r.stderr || r.stdout || `Could not launch ${this.appId}`).trim());
|
|
177
214
|
await sleep(600);
|
|
178
|
-
return this.
|
|
215
|
+
return this.waitForOwnedSnapshot();
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async closeSystemDialogs() {
|
|
219
|
+
// A crash dialog from the previous app can outlive that process and cover a
|
|
220
|
+
// newly launched app. The shell broadcast only closes system-owned surfaces;
|
|
221
|
+
// it does not clear crash history or interact with the app under test.
|
|
222
|
+
await this.adb(["shell", "am", "broadcast", "-a", "android.intent.action.CLOSE_SYSTEM_DIALOGS"]);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
async waitForOwnedSnapshot(timeoutMs = 12_000) {
|
|
226
|
+
const deadline = Date.now() + timeoutMs;
|
|
227
|
+
let latest;
|
|
228
|
+
let consecutiveOwned = 0;
|
|
229
|
+
let overlayDismissed = false;
|
|
230
|
+
do {
|
|
231
|
+
latest = await this.snapshot();
|
|
232
|
+
if (isAndroidAppSnapshot(latest, this.appId)) {
|
|
233
|
+
consecutiveOwned += 1;
|
|
234
|
+
// One mixed dumpsys/UIAutomator sample caused cross-run contamination.
|
|
235
|
+
// Require agreement twice before handing the surface to exploration.
|
|
236
|
+
if (consecutiveOwned >= 2) return latest;
|
|
237
|
+
} else {
|
|
238
|
+
consecutiveOwned = 0;
|
|
239
|
+
if (!overlayDismissed && isSystemOverlayOverApp(latest, this.appId)) {
|
|
240
|
+
overlayDismissed = true;
|
|
241
|
+
await this.closeSystemDialogs();
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
if (Date.now() >= deadline) break;
|
|
245
|
+
// Activity and UIAutomator are intentionally required to agree. During app
|
|
246
|
+
// launch they may momentarily describe opposite sides of the transition;
|
|
247
|
+
// retry that mixed snapshot instead of turning it into zero-screen evidence.
|
|
248
|
+
await sleep(150);
|
|
249
|
+
} while (Date.now() < deadline);
|
|
250
|
+
return latest;
|
|
179
251
|
}
|
|
180
252
|
|
|
181
253
|
async currentActivity() {
|
|
@@ -210,6 +282,19 @@ export class AndroidDriver {
|
|
|
210
282
|
return { screenTitle: detectAndroidScreen(elements, activity), elements, activity, xml };
|
|
211
283
|
}
|
|
212
284
|
|
|
285
|
+
async observeActivityTransition(beforeActivity, timeoutMs = 700) {
|
|
286
|
+
const initial = String(beforeActivity || "");
|
|
287
|
+
if (!initial) return false;
|
|
288
|
+
const deadline = Date.now() + timeoutMs;
|
|
289
|
+
do {
|
|
290
|
+
const current = await this.currentActivity();
|
|
291
|
+
if (current && current !== initial) return true;
|
|
292
|
+
if (Date.now() >= deadline) break;
|
|
293
|
+
await sleep(60);
|
|
294
|
+
} while (Date.now() < deadline);
|
|
295
|
+
return false;
|
|
296
|
+
}
|
|
297
|
+
|
|
213
298
|
async screenshot(filePath) {
|
|
214
299
|
const r = await this.adb(["exec-out", "screencap", "-p"], { encoding: "buffer", timeout: 30_000, maxBuffer: 32 * 1024 * 1024 });
|
|
215
300
|
if (r.code !== 0 || !r.stdout?.length) throw new Error("Android screenshot failed");
|
|
@@ -8,6 +8,8 @@ import { semanticUiKey } from "./ui-map.js";
|
|
|
8
8
|
|
|
9
9
|
const ERROR_RE = /\b(something went wrong|internal server error|an error occurred|failed to load|unhandled exception|has stopped)\b/i;
|
|
10
10
|
const DESTRUCTIVE_RE = /\b(delete|remove|purchase|buy now|pay now|reset|erase|unsubscribe|sign out|log out|logout)\b/i;
|
|
11
|
+
const AUTH_SUBMIT_RE = /\b(sign[ -]?in|log[ -]?in|continue|submit)\b/i;
|
|
12
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
11
13
|
|
|
12
14
|
function stateHash(snap) {
|
|
13
15
|
return snap.elements.map((e) => `${androidElementKey(e)}:${e.text}:${e.x},${e.y}`).join("|");
|
|
@@ -40,8 +42,24 @@ function isDestructive(e) {
|
|
|
40
42
|
return DESTRUCTIVE_RE.test(`${e.label || ""} ${e.text || ""} ${e.description || ""} ${e.id || ""}`.replace(/[_-]+/g, " "));
|
|
41
43
|
}
|
|
42
44
|
|
|
43
|
-
export
|
|
45
|
+
export function isAndroidAuthSubmit(element) {
|
|
46
|
+
return AUTH_SUBMIT_RE.test(`${element?.label || ""} ${element?.text || ""} ${element?.description || ""} ${element?.id || ""}`.replace(/[_-]+/g, " "));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function isAndroidBlankSnapshot(snapshot) {
|
|
50
|
+
const elements = snapshot?.elements || [];
|
|
51
|
+
const meaningful = elements.some((element) =>
|
|
52
|
+
String(element.text || "").trim() ||
|
|
53
|
+
String(element.description || "").trim() ||
|
|
54
|
+
(String(element.id || "").trim() && !/^(android:)?id\/content$/i.test(String(element.id || "").trim()))
|
|
55
|
+
);
|
|
56
|
+
const interactive = elements.some((element) => element.hittable && (element.clickable || /Button|EditText|Tab|Switch|CheckBox/i.test(element.type)));
|
|
57
|
+
return !meaningful && !interactive;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40, timeoutSec = 300, outDir, testEmail = "", testPassword = "", clearData = true, seedTargets = [], onProgress = () => {}, driver, screenshotDelayMs }) {
|
|
44
61
|
const d = driver || new AndroidDriver({ appId, serial });
|
|
62
|
+
const visualSettleMs = Number.isFinite(screenshotDelayMs) ? Math.max(0, screenshotDelayMs) : (driver ? 0 : 350);
|
|
45
63
|
d.appId = appId;
|
|
46
64
|
await d.ensureDevice();
|
|
47
65
|
if (apkPath) await d.install(apkPath);
|
|
@@ -56,7 +74,33 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
56
74
|
const visited = new Map();
|
|
57
75
|
let issues = 0;
|
|
58
76
|
let actions = 0;
|
|
77
|
+
const crashExitBaseline = typeof d.latestCrashExitInfo === "function" ? await d.latestCrashExitInfo() : null;
|
|
59
78
|
let snap = await d.launch({ clearData });
|
|
79
|
+
let crashReported = false;
|
|
80
|
+
|
|
81
|
+
const reportProcessExit = async (screen, step) => {
|
|
82
|
+
if (typeof d.isProcessAlive !== "function") return false;
|
|
83
|
+
// Android may reveal the launcher before the crashing process disappears from
|
|
84
|
+
// pidof. A one-shot liveness sample made identical crashes scheduler-dependent.
|
|
85
|
+
// Poll only after app ownership is already lost: external intents and ordinary
|
|
86
|
+
// Back boundaries keep the originating process alive and remain boundaries.
|
|
87
|
+
let alive = await d.isProcessAlive();
|
|
88
|
+
let latestCrash = typeof d.latestCrashExitInfo === "function" ? await d.latestCrashExitInfo() : null;
|
|
89
|
+
const exitDeadline = Date.now() + 2_000;
|
|
90
|
+
while (alive && (!latestCrash || latestCrash === crashExitBaseline) && Date.now() < exitDeadline) {
|
|
91
|
+
await sleep(150);
|
|
92
|
+
alive = await d.isProcessAlive();
|
|
93
|
+
latestCrash = typeof d.latestCrashExitInfo === "function" ? await d.latestCrashExitInfo() : null;
|
|
94
|
+
}
|
|
95
|
+
const recordedCrash = !!latestCrash && latestCrash !== crashExitBaseline;
|
|
96
|
+
if ((alive && !recordedCrash) || crashReported) return false;
|
|
97
|
+
crashReported = true;
|
|
98
|
+
emit("ISSUE", { type: "crash", severity: "critical", title: "App process exited during exploration", screen: screen || "Launch", step });
|
|
99
|
+
issues += 1;
|
|
100
|
+
return true;
|
|
101
|
+
};
|
|
102
|
+
|
|
103
|
+
if (!isAndroidAppSnapshot(snap, appId)) await reportProcessExit("Launch", 0);
|
|
60
104
|
|
|
61
105
|
const normalizedTargets = (Array.isArray(seedTargets) ? seedTargets : []).filter((target) =>
|
|
62
106
|
target?.platform === "android" && target?.status === "planned" && target?.navigation?.status === "replayable" &&
|
|
@@ -69,6 +113,10 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
69
113
|
const screen = snapshot.screenTitle;
|
|
70
114
|
if (!visited.has(hash)) {
|
|
71
115
|
visited.set(hash, screen);
|
|
116
|
+
// UIAutomator can expose a fully populated hierarchy a fraction before
|
|
117
|
+
// SurfaceFlinger composites the first app frame. Give real devices one
|
|
118
|
+
// bounded draw interval so retained PNG evidence matches the tree.
|
|
119
|
+
if (visualSettleMs) await sleep(visualSettleMs);
|
|
72
120
|
await d.screenshot(path.join(screenshots, `${String(visited.size).padStart(2, "0")}-${screen.replace(/[^a-z0-9]+/gi, "-").toLowerCase()}.png`)).catch(() => {});
|
|
73
121
|
}
|
|
74
122
|
const inputs = inputDescriptors(snapshot.elements);
|
|
@@ -95,7 +143,11 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
95
143
|
const before = state;
|
|
96
144
|
snap = await d.settle();
|
|
97
145
|
state = await recordState(snap);
|
|
98
|
-
if (!state) {
|
|
146
|
+
if (!state) {
|
|
147
|
+
const crashed = await reportProcessExit(before.screen, actions);
|
|
148
|
+
failure = crashed ? "The application process exited while replaying the UI Map path" : "The UI Map path left the application foreground";
|
|
149
|
+
break;
|
|
150
|
+
}
|
|
99
151
|
emit("TRANSITION", { from: before.screen, to: state.screen, action: selector, changed: before.hash !== state.hash });
|
|
100
152
|
onProgress({ action: actions, max: maxActions, states: visited.size });
|
|
101
153
|
}
|
|
@@ -113,7 +165,10 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
113
165
|
// UIAutomator can still describe the launcher or a system surface after
|
|
114
166
|
// Back/external navigation. Those are exploration boundaries, never nodes
|
|
115
167
|
// in the application-owned UI Map.
|
|
116
|
-
if (!isAndroidAppSnapshot(snap, appId))
|
|
168
|
+
if (!isAndroidAppSnapshot(snap, appId)) {
|
|
169
|
+
await reportProcessExit(visited.size ? [...visited.values()].at(-1) : "Launch", actions);
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
117
172
|
const recorded = await recordState(snap);
|
|
118
173
|
if (!recorded) break;
|
|
119
174
|
const { hash, screen, inputs } = recorded;
|
|
@@ -124,7 +179,7 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
124
179
|
emit("ISSUE", { type: "error_surface", severity: "high", title: "Visible error surface", screen, step: actions });
|
|
125
180
|
issues += 1;
|
|
126
181
|
}
|
|
127
|
-
if (snap
|
|
182
|
+
if (isAndroidBlankSnapshot(snap)) {
|
|
128
183
|
emit("ISSUE", { type: "blank_screen", severity: "high", title: "No usable controls or content", screen, step: actions });
|
|
129
184
|
issues += 1;
|
|
130
185
|
}
|
|
@@ -154,16 +209,31 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
154
209
|
});
|
|
155
210
|
if (candidate) {
|
|
156
211
|
const target = controlLabel(candidate);
|
|
212
|
+
const loginSubmit = inputs.some((input) => input.secure) && isAndroidAuthSubmit(candidate);
|
|
157
213
|
tried.add(`${hash}|tap|${target}`);
|
|
158
214
|
const before = hash;
|
|
159
215
|
const r = await d.tap(target, snap);
|
|
160
216
|
actions += 1;
|
|
161
|
-
|
|
217
|
+
// UIAutomator can miss a short-lived Activity that opens and cleanly
|
|
218
|
+
// returns before its next hierarchy dump. Observe the cheaper Activity
|
|
219
|
+
// signal in parallel so a real transient response is not called dead.
|
|
220
|
+
const activityEffect = r.status === "ok" && typeof d.observeActivityTransition === "function"
|
|
221
|
+
? d.observeActivityTransition(snap.activity)
|
|
222
|
+
: Promise.resolve(false);
|
|
223
|
+
const [settledSnap, activityEffectObserved] = await Promise.all([d.settle(), activityEffect]);
|
|
224
|
+
snap = settledSnap;
|
|
162
225
|
const after = stateHash(snap);
|
|
163
|
-
emit("ACTION", { type: "tap", target, reason: "untried_control", step: actions, screen, status: r.status });
|
|
164
|
-
if (!isAndroidAppSnapshot(snap, appId))
|
|
226
|
+
emit("ACTION", { type: loginSubmit ? "login_submit" : "tap", target, reason: "untried_control", step: actions, screen, status: r.status });
|
|
227
|
+
if (!isAndroidAppSnapshot(snap, appId)) {
|
|
228
|
+
await reportProcessExit(screen, actions);
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
165
231
|
emit("TRANSITION", { from: screen, to: snap.screenTitle, action: target, changed: before !== after });
|
|
166
|
-
|
|
232
|
+
const authFailed = loginSubmit && inputDescriptors(snap.elements).some((input) => input.secure);
|
|
233
|
+
if (r.status === "ok" && authFailed) {
|
|
234
|
+
emit("ISSUE", { type: "auth_failed", severity: "high", title: "Sign-in attempt remained on the login screen", screen, target, step: actions });
|
|
235
|
+
issues += 1;
|
|
236
|
+
} else if (r.status === "ok" && before === after && candidate.clickable && !activityEffectObserved) {
|
|
167
237
|
emit("ISSUE", { type: "unresponsive_element", severity: "medium", title: `Control did not respond: ${target}`, screen, target, step: actions });
|
|
168
238
|
issues += 1;
|
|
169
239
|
}
|
|
@@ -179,7 +249,10 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
179
249
|
actions += 1;
|
|
180
250
|
const next = await d.settle();
|
|
181
251
|
emit("ACTION", { type: "back", target: "system back", reason: "state_exhausted", step: actions, screen });
|
|
182
|
-
if (!isAndroidAppSnapshot(next, appId))
|
|
252
|
+
if (!isAndroidAppSnapshot(next, appId)) {
|
|
253
|
+
await reportProcessExit(screen, actions);
|
|
254
|
+
break;
|
|
255
|
+
}
|
|
183
256
|
emit("TRANSITION", { from: screen, to: next.screenTitle, action: "back", changed: stateHash(next) !== hash });
|
|
184
257
|
if (stateHash(next) !== hash) { snap = next; continue; }
|
|
185
258
|
}
|
|
@@ -187,11 +260,7 @@ export async function exploreAndroid({ appId, apkPath, serial, maxActions = 40,
|
|
|
187
260
|
}
|
|
188
261
|
|
|
189
262
|
const timedOut = Date.now() >= deadline;
|
|
190
|
-
|
|
191
|
-
emit("ISSUE", { type: "explore_timeout", severity: "medium", title: `Exploration timed out after ${timeoutSec}s`, screen: snap.screenTitle, step: actions });
|
|
192
|
-
issues += 1;
|
|
193
|
-
}
|
|
194
|
-
emit("COMPLETE", { actions, states: visited.size, issues, screens: [...new Set(visited.values())].join(","), outcome: timedOut ? "timeout" : "complete" });
|
|
263
|
+
emit("COMPLETE", { actions, states: visited.size, issues, screens: [...new Set(visited.values())].join(","), outcome: timedOut ? "timeout" : "complete", timedOut, ...(timedOut ? { timeoutSeconds: timeoutSec } : {}) });
|
|
195
264
|
onProgress({ action: actions, max: maxActions, states: visited.size });
|
|
196
265
|
return { markersPath, outDir, actions, states: visited.size, issues, timedOut, seedTargets: normalizedTargets };
|
|
197
266
|
}
|
|
@@ -114,8 +114,8 @@ function applyRuntimeTargetValidation(root, targets, validation) {
|
|
|
114
114
|
build: { container, scheme, configuration },
|
|
115
115
|
evidence: {
|
|
116
116
|
...(captureId ? { capture: portableEvidenceReference(`tapp-capture:${captureId}`) } : {}),
|
|
117
|
-
verdict: String(validation.evidence?.verdict || "unknown"),
|
|
118
117
|
inconclusive: validation.evidence?.inconclusive === true,
|
|
118
|
+
findingCount: Number(validation.evidence?.findingCount || 0),
|
|
119
119
|
...(validation.evidence?.observedAt ? { observedAt: String(validation.evidence.observedAt) } : {}),
|
|
120
120
|
},
|
|
121
121
|
detail: "Tapp built this repository target with the recorded scheme, installed it, launched it, and produced UI Map evidence.",
|
|
@@ -149,8 +149,8 @@ function persistedTargetValidations(root, outDir) {
|
|
|
149
149
|
},
|
|
150
150
|
evidence: {
|
|
151
151
|
captureId: capture.startsWith("tapp-capture:") ? capture.slice("tapp-capture:".length) : "",
|
|
152
|
-
verdict: validation.evidence?.verdict,
|
|
153
152
|
inconclusive: validation.evidence?.inconclusive === true,
|
|
153
|
+
findingCount: Number(validation.evidence?.findingCount || 0),
|
|
154
154
|
observedAt: validation.evidence?.observedAt,
|
|
155
155
|
},
|
|
156
156
|
}];
|
|
@@ -424,7 +424,7 @@ function applicationName(root, targets) {
|
|
|
424
424
|
return pkg?.name || (targets.length === 1 ? targets[0].name : path.basename(root));
|
|
425
425
|
}
|
|
426
426
|
|
|
427
|
-
export async function inspectApplicationRepository({ projectDir, ownedUrl = "", platform = "", targetValidation = null, outDir = ".tapp" } = {}) {
|
|
427
|
+
export async function inspectApplicationRepository({ projectDir, ownedUrl = "", platform = "", targetValidation = null, defaultTargetId = "", outDir = ".tapp" } = {}) {
|
|
428
428
|
const root = fs.realpathSync(path.resolve(projectDir || process.cwd()));
|
|
429
429
|
const inventory = walk(root);
|
|
430
430
|
let targets = [
|
|
@@ -567,9 +567,14 @@ export async function inspectApplicationRepository({ projectDir, ownedUrl = "",
|
|
|
567
567
|
for (const error of taskErrors) requirements.push({ id: stableId("task-error", error.path), severity: "blocking", status: "invalid", message: error.error, remediation: `Fix ${error.path} before generation.` });
|
|
568
568
|
for (const error of contractErrors) requirements.push({ id: stableId("contract-error", error.path), severity: "blocking", status: "invalid", message: error.error, remediation: `Fix ${error.path} before trusting the release plan.` });
|
|
569
569
|
|
|
570
|
+
const recordedDefaultTargetId = targets.some((target) => target.id === defaultTargetId)
|
|
571
|
+
? defaultTargetId
|
|
572
|
+
: targets.length === 1 ? targets[0].id : "";
|
|
570
573
|
const model = {
|
|
571
574
|
schemaVersion: 1, kind: "tapp-application-model",
|
|
572
|
-
|
|
575
|
+
// defaultTargetId is recorded only when selection is unambiguous or the user explicitly chose
|
|
576
|
+
// a target during source-connected exploration. Detection order is not user intent.
|
|
577
|
+
application: { name: applicationName(root, targets), repositoryRoot: ".", platforms: [...new Set(targets.map((target) => target.platform))].sort(), targetIds: targets.map((target) => target.id), defaultTargetId: recordedDefaultTargetId },
|
|
573
578
|
targets,
|
|
574
579
|
actors,
|
|
575
580
|
entities,
|
|
@@ -1415,8 +1420,8 @@ export function recordContractProposalValidation(plan, { id = "", name = "", pla
|
|
|
1415
1420
|
export function recordGeneratedTaskProposalValidation({ projectDir, item, platform, evidence = "", detail = "" } = {}) {
|
|
1416
1421
|
if (!["ios", "android", "web"].includes(platform)) throw new Error("platform must be ios|android|web");
|
|
1417
1422
|
const root = fs.realpathSync(path.resolve(projectDir || process.cwd()));
|
|
1418
|
-
const proposalMarkers = [
|
|
1419
|
-
const reviewedMarkers = [
|
|
1423
|
+
const proposalMarkers = [`${path.sep}.tapp${path.sep}proposals${path.sep}tasks${path.sep}`];
|
|
1424
|
+
const reviewedMarkers = [`${path.sep}.tapp${path.sep}tasks${path.sep}`];
|
|
1420
1425
|
const updated = [];
|
|
1421
1426
|
for (const taskPath of item?.generation?.taskPaths || []) {
|
|
1422
1427
|
const absolute = path.resolve(root, taskPath);
|
|
@@ -1461,8 +1466,7 @@ export function mergeGeneratedTaskProposalValidation(plan, updates = []) {
|
|
|
1461
1466
|
}
|
|
1462
1467
|
|
|
1463
1468
|
function promotedDestination(root, source, kind) {
|
|
1464
|
-
const marker = [
|
|
1465
|
-
.map((directory) => `${path.sep}${directory}${path.sep}proposals${path.sep}${kind}${path.sep}`)
|
|
1469
|
+
const marker = [`${path.sep}.tapp${path.sep}proposals${path.sep}${kind}${path.sep}`]
|
|
1466
1470
|
.find((candidate) => source.includes(candidate));
|
|
1467
1471
|
if (!marker) throw new Error(`Proposal ${kind.slice(0, -1)} is outside .tapp/proposals/${kind}: ${relative(root, source)}`);
|
|
1468
1472
|
const index = source.indexOf(marker);
|
|
@@ -1505,7 +1509,7 @@ export async function promoteValidatedProposals(plan, { projectDir, ids = [] } =
|
|
|
1505
1509
|
for (const taskPath of item.generation.taskPaths || []) {
|
|
1506
1510
|
const source = path.resolve(root, taskPath);
|
|
1507
1511
|
if (!fs.existsSync(source)) throw new Error(`Generated Task is missing: ${taskPath}`);
|
|
1508
|
-
if (!
|
|
1512
|
+
if (!String(source).includes(`${path.sep}.tapp${path.sep}proposals${path.sep}tasks${path.sep}`)) continue;
|
|
1509
1513
|
const destination = promotedDestination(root, source, "tasks");
|
|
1510
1514
|
moves.set(source, destination);
|
|
1511
1515
|
let task = taskRecords.get(source);
|
|
@@ -109,7 +109,7 @@ function assetPath(pathname) {
|
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
function captureRoot() {
|
|
112
|
-
return path.join(process.env.TAPP_HOME ||
|
|
112
|
+
return path.join(process.env.TAPP_HOME || path.join(os.homedir(), ".tapp"), "captures");
|
|
113
113
|
}
|
|
114
114
|
|
|
115
115
|
function capturePath(captureId, relative = "report.html") {
|
|
@@ -16,17 +16,19 @@
|
|
|
16
16
|
// [--pr-plan <plan.json>] # selected PR contract execution manifest
|
|
17
17
|
// [--project-dir <repo> --maintenance-url <url>]
|
|
18
18
|
// # optional disposable web patch replay
|
|
19
|
-
// [--fail-on <gate|
|
|
19
|
+
// [--fail-on <gate|absolute|any>] # default: gate
|
|
20
20
|
//
|
|
21
21
|
// Gate policy (--fail-on):
|
|
22
22
|
// gate fail when the run introduced NEW high/critical findings vs. the baseline
|
|
23
|
-
// (no baseline ⇒ falls back to `
|
|
24
|
-
// pre-existing debt doesn't block, regressions and broken
|
|
25
|
-
//
|
|
26
|
-
//
|
|
23
|
+
// (no baseline ⇒ falls back to `absolute`), or when any suite failed. The default:
|
|
24
|
+
// pre-existing debt doesn't block, regressions and broken suites do.
|
|
25
|
+
// absolute fail on any current-run deterministic findings-block (critical / risk threshold) or an
|
|
26
|
+
// inconclusive run, or any failed suite — no baseline needed.
|
|
27
|
+
// any fail on any finding at all, or any suite failure. Strictest.
|
|
27
28
|
import fs from "fs";
|
|
28
29
|
import path from "node:path";
|
|
29
|
-
import {
|
|
30
|
+
import { execSync } from "node:child_process";
|
|
31
|
+
import { buildQaReport, computeRegression, computeContentCollapse, computeReachabilityLoss, evaluateGate, GATE_EXIT } from "./report.js";
|
|
30
32
|
import { writeHtmlReport } from "./html-report.js";
|
|
31
33
|
import { buildUiMapFromMarkers, writeUiMap } from "./ui-map.js";
|
|
32
34
|
import { proposeSelectorMaintenance, validateWebMaintenanceProposal } from "./maintenance-proposal.js";
|
|
@@ -59,13 +61,31 @@ function parseArgs(argv) {
|
|
|
59
61
|
console.error("Required: --markers <ocqa-markers.txt>");
|
|
60
62
|
process.exit(2);
|
|
61
63
|
}
|
|
62
|
-
if (!["gate", "
|
|
63
|
-
console.error(`--fail-on must be gate|
|
|
64
|
+
if (!["gate", "absolute", "any"].includes(args.failOn)) {
|
|
65
|
+
console.error(`--fail-on must be gate|absolute|any, got: ${args.failOn}`);
|
|
64
66
|
process.exit(2);
|
|
65
67
|
}
|
|
66
68
|
return args;
|
|
67
69
|
}
|
|
68
70
|
|
|
71
|
+
// A GateRun records the revision it judged. A commit SHA alone doesn't represent a dirty local
|
|
72
|
+
// checkout, so record both. Best-effort: CI env → git → nulls (never throws / fails the gate).
|
|
73
|
+
function gitRevision(repoDir) {
|
|
74
|
+
// Inspect the TARGET repository (--project-dir) when given, not the process cwd, so a gate run
|
|
75
|
+
// launched from elsewhere doesn't record the wrong repo's SHA. GITHUB_SHA still wins in CI.
|
|
76
|
+
const opts = { stdio: ["ignore", "pipe", "ignore"], ...(repoDir ? { cwd: repoDir } : {}) };
|
|
77
|
+
const sh = (cmd) => execSync(cmd, opts).toString().trim();
|
|
78
|
+
let sha = process.env.GITHUB_SHA || null;
|
|
79
|
+
let dirty = false;
|
|
80
|
+
try {
|
|
81
|
+
if (!sha) sha = sh("git rev-parse HEAD") || null;
|
|
82
|
+
dirty = sh("git status --porcelain").length > 0;
|
|
83
|
+
} catch {
|
|
84
|
+
/* not a git checkout / git unavailable */
|
|
85
|
+
}
|
|
86
|
+
return { sha, dirty };
|
|
87
|
+
}
|
|
88
|
+
|
|
69
89
|
async function validateMaintenancePlan(plan, args) {
|
|
70
90
|
if (!plan || plan.platform !== "web" || !args.projectDir || !args.maintenanceUrl) return plan;
|
|
71
91
|
let remaining = 3;
|
|
@@ -101,7 +121,7 @@ async function validateMaintenancePlan(plan, args) {
|
|
|
101
121
|
// Port of flow_lib.py report() / FlowRunnerService.parseReport — kept in sync deliberately.
|
|
102
122
|
function parseFlowLog(logPath) {
|
|
103
123
|
const name = logPath.split("/").pop().replace(/\.log$/, "");
|
|
104
|
-
if (!fs.existsSync(logPath)) return { name, passed: false, total: 0, failed: 0, steps: [], missing: true };
|
|
124
|
+
if (!fs.existsSync(logPath)) return { name, passed: false, total: 0, failed: 0, steps: [], missing: true, modelObserved: false, deterministicFailed: false };
|
|
105
125
|
const steps = [];
|
|
106
126
|
let total = 0, executed = 0, failed = 0, passed = false, sawResult = false, flowName = null, kind = "flow", contract = "", criticality = "";
|
|
107
127
|
for (const raw of fs.readFileSync(logPath, "utf8").split(/\r?\n/)) {
|
|
@@ -132,7 +152,12 @@ function parseFlowLog(logPath) {
|
|
|
132
152
|
failed = steps.filter((s) => s.status === "fail").length;
|
|
133
153
|
passed = steps.length > 0 && failed === 0;
|
|
134
154
|
}
|
|
135
|
-
|
|
155
|
+
// Structural evidence authority (ADR-0005): assert_ai steps are model-observed. A deterministic
|
|
156
|
+
// step failure is real; a suite that only carries a model assertion cannot be decided by the
|
|
157
|
+
// default deterministic gate. evaluateGate reads these flags, never the raw action string.
|
|
158
|
+
const modelObserved = steps.some((s) => s.action === "assert_ai");
|
|
159
|
+
const deterministicFailed = steps.some((s) => s.action !== "assert_ai" && s.status === "fail");
|
|
160
|
+
return { name: flowName || name, kind, ...(contract ? { contract, criticality } : {}), passed, total, executed, failed, steps, modelObserved, deterministicFailed };
|
|
136
161
|
}
|
|
137
162
|
|
|
138
163
|
function loadBaseline(baselinePath) {
|
|
@@ -338,16 +363,19 @@ function enrichPrPlan(plan, contracts, currentUiMap = null, markersPath = "", pr
|
|
|
338
363
|
}
|
|
339
364
|
|
|
340
365
|
const SEV_ICON = { critical: "🟥", high: "🟧", medium: "🟨", low: "🟩" };
|
|
366
|
+
const GATE_BADGE = { pass: "🟢 PASS", fail: "🔴 FAIL", inconclusive: "🟡 INCONCLUSIVE" };
|
|
341
367
|
|
|
342
368
|
function renderMarkdown(report, regression, flows, scenarios, contracts, prPlan, gate) {
|
|
343
369
|
const lines = [];
|
|
344
|
-
|
|
370
|
+
// The header shows the GATE decision (pass/fail/inconclusive), not a ship verdict — exploration
|
|
371
|
+
// only observes; the gate judges (ADR-0005).
|
|
372
|
+
lines.push(`## tapp release check — ${GATE_BADGE[gate.outcome] || gate.outcome}`);
|
|
345
373
|
lines.push("");
|
|
346
374
|
lines.push(report.headline);
|
|
347
375
|
lines.push("");
|
|
348
|
-
lines.push(
|
|
376
|
+
lines.push(`${report.screensExplored} screens · ${report.actionsPerformed} actions · ${report.findingCounts.total} finding(s)`);
|
|
349
377
|
if (report.platform === "web") {
|
|
350
|
-
lines.push(`**
|
|
378
|
+
lines.push(`**Deterministic basis:** ${report.deterministicFindingCounts?.total || 0} deterministic finding(s); ${report.sampledFindingCounts?.total || 0} sampled probe finding(s) are advisory.`);
|
|
351
379
|
}
|
|
352
380
|
if (report.uiMap) lines.push(`**UI Map:** ${report.uiMap.nodeCount} states · ${report.uiMap.edgeCount} transitions · ${report.uiMap.controlCount} semantic controls`);
|
|
353
381
|
if (report.findings.length) {
|
|
@@ -359,18 +387,22 @@ function renderMarkdown(report, regression, flows, scenarios, contracts, prPlan,
|
|
|
359
387
|
}
|
|
360
388
|
}
|
|
361
389
|
if (regression) {
|
|
362
|
-
|
|
390
|
+
// This is the GATE's report, so it may judge the regression. computeRegression is comparison-only,
|
|
391
|
+
// so derive the new-high/critical count here (mirrors evaluateGate).
|
|
392
|
+
const newCritical = regression.newFindings.filter((f) => f.severity === "critical").length;
|
|
393
|
+
const newHigh = regression.newFindings.filter((f) => f.severity === "high").length;
|
|
394
|
+
const regFailed = newCritical + newHigh > 0;
|
|
363
395
|
lines.push("");
|
|
364
|
-
lines.push(`### Since baseline — ${
|
|
396
|
+
lines.push(`### Since baseline — ${regFailed ? "🔴 regression gate FAILED" : "🟢 regression gate passed"}`);
|
|
365
397
|
lines.push(`+${regression.counts.new} new · ${regression.counts.persisting} persisting · ${regression.counts.resolved} resolved` +
|
|
366
|
-
(
|
|
398
|
+
(regFailed ? ` — **${newCritical} new critical, ${newHigh} new high**` : ""));
|
|
367
399
|
for (const f of regression.newFindings) {
|
|
368
400
|
lines.push(`- NEW ${SEV_ICON[f.severity] || ""} ${f.severity}: ${f.title} (${f.screen ?? "—"})`);
|
|
369
401
|
}
|
|
370
402
|
} else if (gate.policy === "gate") {
|
|
371
403
|
lines.push("");
|
|
372
404
|
lines.push("### Baseline — 🟡 not active yet");
|
|
373
|
-
lines.push("No baseline was supplied
|
|
405
|
+
lines.push("No baseline was supplied. This gate still enforces absolute blockers and reviewed suite failures, but it cannot identify new regressions until a baseline is saved.");
|
|
374
406
|
}
|
|
375
407
|
if (prPlan) {
|
|
376
408
|
lines.push("");
|
|
@@ -444,15 +476,23 @@ function renderMarkdown(report, regression, flows, scenarios, contracts, prPlan,
|
|
|
444
476
|
}
|
|
445
477
|
}
|
|
446
478
|
lines.push("");
|
|
447
|
-
|
|
479
|
+
const badge = GATE_BADGE[gate.outcome] || (gate.failed ? "🔴 FAIL" : "🟢 PASS");
|
|
480
|
+
lines.push(`**Gate (${gate.policy}): ${badge}**${gate.reasons.length ? " — " + gate.reasons.join("; ") : ""}`);
|
|
481
|
+
// The gate is only authoritative about what it actually ran — record the scope explicitly.
|
|
482
|
+
const rev = gate.revision?.sha ? `${String(gate.revision.sha).slice(0, 12)}${gate.revision.dirty ? "-dirty" : ""}` : "unknown";
|
|
483
|
+
lines.push(`_target: ${gate.target || "—"} · revision: ${rev} · policy: ${gate.policy} v${gate.policyVersion || "?"}_`);
|
|
484
|
+
if (Array.isArray(gate.checked) && gate.checked.length) lines.push(`_Checked: ${gate.checked.join(" · ")}_`);
|
|
485
|
+
if (Array.isArray(gate.notChecked) && gate.notChecked.length) lines.push(`_Not checked: ${gate.notChecked.join(" · ")}_`);
|
|
448
486
|
return lines.join("\n");
|
|
449
487
|
}
|
|
450
488
|
|
|
451
489
|
const args = parseArgs(process.argv.slice(2));
|
|
452
|
-
const report = buildQaReport(args.markers, { platform: args.platform || "ios" });
|
|
490
|
+
const report = buildQaReport(args.markers, { platform: args.platform || "ios", target: args.label || null });
|
|
453
491
|
if (!report) {
|
|
454
|
-
|
|
455
|
-
|
|
492
|
+
// Required evidence could not be obtained — this is inconclusive (fails closed), not a gate FAIL
|
|
493
|
+
// and not a usage error. See the outcome model in report.js (GATE_EXIT).
|
|
494
|
+
console.error(`No OCQA markers found at ${args.markers} — the exploration did not run (inconclusive).`);
|
|
495
|
+
process.exit(GATE_EXIT.inconclusive);
|
|
456
496
|
}
|
|
457
497
|
let currentUiMap = null;
|
|
458
498
|
if (args.htmlDir) {
|
|
@@ -491,18 +531,12 @@ if (collapsed.length) {
|
|
|
491
531
|
report.findings.push(...collapsed);
|
|
492
532
|
report.findingCounts.high += collapsed.length;
|
|
493
533
|
report.findingCounts.total += collapsed.length;
|
|
494
|
-
report.
|
|
495
|
-
report.
|
|
496
|
-
//
|
|
497
|
-
// baseline
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
report.releaseScore = report.confidence;
|
|
501
|
-
}
|
|
502
|
-
if (report.verdict === "ready") {
|
|
503
|
-
report.verdict = Number.isFinite(report.confidence) && report.confidence < 50 ? "blocked" : "caution";
|
|
504
|
-
}
|
|
505
|
-
report.headline = `Proceed with caution — ${collapsed.length} screen(s) regressed vs. baseline (content collapsed or became unreachable).`;
|
|
534
|
+
report.deterministicFindingCounts.high += collapsed.length;
|
|
535
|
+
report.deterministicFindingCounts.total += collapsed.length;
|
|
536
|
+
// Collapsed/unreachable screens are deterministic regressions: they raise the deterministic finding
|
|
537
|
+
// counts (so findingsBlock sees them) and count as new-vs-baseline (so the gate fails on them).
|
|
538
|
+
// No score/verdict to mutate — exploration is scoreless; the gate renders the outcome.
|
|
539
|
+
report.headline = `${collapsed.length} screen(s) regressed vs. baseline (content collapsed or became unreachable).`;
|
|
506
540
|
}
|
|
507
541
|
const regression = computeRegression(report.findings, baseline?.findings ?? null);
|
|
508
542
|
const runs = args.flowLogs.map(parseFlowLog);
|
|
@@ -519,34 +553,19 @@ try {
|
|
|
519
553
|
process.exit(2);
|
|
520
554
|
}
|
|
521
555
|
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
const
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
if (report.inconclusive) reasons.push("run was inconclusive (coverage floor not met)");
|
|
536
|
-
} else {
|
|
537
|
-
if (regression?.gate.failed) {
|
|
538
|
-
reasons.push(`${regression.gate.newCritical} new critical + ${regression.gate.newHigh} new high vs. baseline`);
|
|
539
|
-
}
|
|
540
|
-
// A regression gate must also catch regressions in EXPLORABILITY, not just in findings:
|
|
541
|
-
// a change that makes the app crash at launch (or reintroduces a login wall) produces an
|
|
542
|
-
// inconclusive run with zero new findings — that must never pass. (Found via corpus
|
|
543
|
-
// bug-seeding: a seeded crash-at-startup sailed through on the findings diff alone.)
|
|
544
|
-
if (report.verdict === "blocked") reasons.push("verdict is blocked");
|
|
545
|
-
if (report.inconclusive && !baseline.inconclusive) {
|
|
546
|
-
reasons.push("run became inconclusive vs. baseline (app may no longer launch/explore)");
|
|
547
|
-
}
|
|
548
|
-
}
|
|
549
|
-
const gate = { policy: args.failOn, failed: reasons.length > 0, reasons };
|
|
556
|
+
// The gate decision now lives in a pure, unit-tested evaluator (report.js). A regression gate
|
|
557
|
+
// must catch regressions in EXPLORABILITY, not just findings: a change that makes the app crash at
|
|
558
|
+
// launch (or reintroduces a login wall) produces an inconclusive run with zero new findings — that
|
|
559
|
+
// must never pass (found via corpus bug-seeding). evaluateGate encodes that as an `inconclusive`
|
|
560
|
+
// outcome (exit 3), distinct from a deterministic `fail` (exit 1).
|
|
561
|
+
const decision = evaluateGate({ report, regression, flows, scenarios, contracts, prPlan, baseline, failOn: args.failOn });
|
|
562
|
+
const gate = {
|
|
563
|
+
...decision,
|
|
564
|
+
target: args.targetKey || report.target || null,
|
|
565
|
+
revision: gitRevision(args.projectDir),
|
|
566
|
+
checked: report.checkedFor,
|
|
567
|
+
notChecked: report.notChecked,
|
|
568
|
+
};
|
|
550
569
|
|
|
551
570
|
const md = renderMarkdown(report, regression, flows, scenarios, contracts, prPlan, gate);
|
|
552
571
|
console.log(md);
|
|
@@ -563,4 +582,7 @@ if (args.htmlDir) {
|
|
|
563
582
|
const html = writeHtmlReport(args.htmlDir, { report, label: args.label || "CI run" });
|
|
564
583
|
if (html) console.log(`\nEvidence report: ${html}`);
|
|
565
584
|
}
|
|
566
|
-
|
|
585
|
+
// Outcome → exit code (ADR-0005): pass 0 · fail 1 · error 2 · inconclusive 3. fail and inconclusive
|
|
586
|
+
// both block a merge; distinct codes let CI tell "a regression was observed" from "we couldn't be
|
|
587
|
+
// sure" and keep infra/usage errors (2) separate.
|
|
588
|
+
process.exit(gate.exitCode);
|