@bugmole/cli 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +18 -4
- package/package.json +4 -4
- package/scripts/bugmole.ts +60 -0
- package/scripts/ensure-playwright.cjs +87 -0
- package/scripts/ensure-playwright.test.ts +36 -0
- package/scripts/sync-launcher-version.cjs +24 -0
- package/scripts/sync-launcher-version.test.ts +28 -0
- package/src/registry/migrations/0034_signup_attribution.sql +19 -0
- package/src/registry/migrations/0035_domain_verify_file.sql +6 -0
- package/src/registry-worker/core.ts +2 -0
- package/src/registry-worker/domains/domains.test.ts +64 -1
- package/src/registry-worker/domains/routes.ts +32 -15
- package/src/registry-worker/domains.ts +40 -2
- package/src/registry-worker/features.ts +2 -0
- package/src/registry-worker/feedback/feedback.test.ts +47 -0
- package/src/registry-worker/feedback/website.ts +77 -0
- package/src/registry-worker/index.ts +25 -7
- package/src/registry-worker/org/routes.test.ts +24 -1
- package/src/registry-worker/org/routes.ts +5 -4
- package/src/registry-worker/runner/runner.test.ts +38 -0
- package/src/registry-worker/runner/tokens.ts +4 -1
- package/src/registry-worker/signup/attribution.test.ts +147 -0
- package/src/registry-worker/signup/attribution.ts +194 -0
- package/src/runtime/ai-exploration.test.ts +34 -1
- package/src/runtime/ai-exploration.ts +97 -3
- package/src/runtime/appium-driver.ts +6 -0
- package/src/runtime/discovery-task.ts +24 -5
- package/src/runtime/flow-language.ts +21 -0
- package/src/runtime/journey-editor.ts +39 -0
- package/src/runtime/journey-graph.test.ts +42 -0
- package/src/runtime/playwright-driver.test.ts +145 -0
- package/src/runtime/playwright-driver.ts +237 -8
- package/src/runtime/run-job.test.ts +21 -0
- package/src/runtime/run-job.ts +8 -0
- package/src/runtime/site-discovery.test.ts +135 -1
- package/src/runtime/site-discovery.ts +173 -24
- package/src/runtime/web-suite.test.ts +65 -0
- package/src/runtime/web-suite.ts +3 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
// Sign-up attribution: which affiliate code (?ref=) and campaign (utm_* tags)
|
|
2
|
+
// brought a new user. bugmole.com keeps the first visit's tags in the bm_attr
|
|
3
|
+
// cookie on .bugmole.com; the dashboard reads it after sign-in and posts it
|
|
4
|
+
// here once. First touch wins, and only accounts created in the last day can
|
|
5
|
+
// be attributed, so visiting an affiliate link later can't claim an old user.
|
|
6
|
+
// Nothing here is sent anywhere else; Scents reads the table.
|
|
7
|
+
import type { Env, Principal } from "../core.js";
|
|
8
|
+
import { requestJson, response } from "../core.js";
|
|
9
|
+
import { registerRoutes } from "../routes/index.js";
|
|
10
|
+
|
|
11
|
+
/** How long after an account first appears it may still be attributed. */
|
|
12
|
+
export const ATTRIBUTION_WINDOW_HOURS = 24;
|
|
13
|
+
/** How long after signing up someone may still type an affiliate's referral code. */
|
|
14
|
+
export const REFERRAL_CODE_WINDOW_DAYS = 30;
|
|
15
|
+
|
|
16
|
+
// The dashboard's own identity for calls made without a person (webhooks).
|
|
17
|
+
const DASHBOARD_SERVICE_ID = "bugmole-dashboard-service";
|
|
18
|
+
const UTM_FIELDS = ["utm_source", "utm_medium", "utm_campaign", "utm_content", "utm_term"] as const;
|
|
19
|
+
// The cookie lives 90 days; allow a little over that, and a little clock skew.
|
|
20
|
+
const MAX_AGE_MS = 100 * 86_400_000;
|
|
21
|
+
const CLOCK_SKEW_MS = 5 * 60_000;
|
|
22
|
+
|
|
23
|
+
export type SignupAttribution = {
|
|
24
|
+
ref: string | null;
|
|
25
|
+
utm_source: string | null;
|
|
26
|
+
utm_medium: string | null;
|
|
27
|
+
utm_campaign: string | null;
|
|
28
|
+
utm_content: string | null;
|
|
29
|
+
utm_term: string | null;
|
|
30
|
+
landing_path: string | null;
|
|
31
|
+
referrer_host: string | null;
|
|
32
|
+
first_seen_at: string | null;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
function text(value: unknown, max: number): string | null {
|
|
36
|
+
if (typeof value !== "string") return null;
|
|
37
|
+
const trimmed = value.trim();
|
|
38
|
+
// Control characters mean the value was tampered with, not typed into a link.
|
|
39
|
+
if (!trimmed || /[\u0000-\u001f\u007f]/u.test(trimmed)) return null;
|
|
40
|
+
return trimmed.slice(0, max);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function ref(value: unknown): string | null {
|
|
44
|
+
if (typeof value !== "string") return null;
|
|
45
|
+
const code = value.trim().toLowerCase();
|
|
46
|
+
return /^[a-z0-9_-]{1,40}$/u.test(code) ? code : null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function landingPath(value: unknown): string | null {
|
|
50
|
+
if (typeof value !== "string") return null;
|
|
51
|
+
const pathOnly = value.trim().split(/[?#]/u)[0];
|
|
52
|
+
// A path on bugmole.com: starts with one slash, no spaces.
|
|
53
|
+
if (!/^\/(?!\/)[^\s\u0000-\u001f\u007f]*$/u.test(pathOnly)) return null;
|
|
54
|
+
return pathOnly.slice(0, 200);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function referrerHost(value: unknown): string | null {
|
|
58
|
+
if (typeof value !== "string") return null;
|
|
59
|
+
const host = value.trim().toLowerCase().replace(/\.$/u, "");
|
|
60
|
+
if (host.length > 100 || !/^[a-z0-9-]+(\.[a-z0-9-]+)*$/u.test(host)) return null;
|
|
61
|
+
// Only outside sites count as referrers.
|
|
62
|
+
if (host === "bugmole.com" || host.endsWith(".bugmole.com")) return null;
|
|
63
|
+
return host;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function firstSeenAt(value: unknown, now: number): string | null {
|
|
67
|
+
if (typeof value !== "string" || value.length > 40) return null;
|
|
68
|
+
const at = Date.parse(value);
|
|
69
|
+
if (!Number.isFinite(at) || at > now + CLOCK_SKEW_MS || at < now - MAX_AGE_MS) return null;
|
|
70
|
+
return new Date(at).toISOString();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Keeps only values that match the contract; anything else becomes null.
|
|
75
|
+
* Returns null when nothing usable is left.
|
|
76
|
+
*/
|
|
77
|
+
export function normalizeAttribution(input: unknown, now = Date.now()): SignupAttribution | null {
|
|
78
|
+
if (!input || typeof input !== "object" || Array.isArray(input)) return null;
|
|
79
|
+
const source = input as Record<string, unknown>;
|
|
80
|
+
const result: SignupAttribution = {
|
|
81
|
+
ref: ref(source.ref),
|
|
82
|
+
utm_source: null,
|
|
83
|
+
utm_medium: null,
|
|
84
|
+
utm_campaign: null,
|
|
85
|
+
utm_content: null,
|
|
86
|
+
utm_term: null,
|
|
87
|
+
landing_path: landingPath(source.landing_path),
|
|
88
|
+
referrer_host: referrerHost(source.referrer_host),
|
|
89
|
+
first_seen_at: firstSeenAt(source.first_seen_at, now),
|
|
90
|
+
};
|
|
91
|
+
for (const field of UTM_FIELDS) result[field] = text(source[field], 100);
|
|
92
|
+
const { first_seen_at: _when, ...facts } = result;
|
|
93
|
+
return Object.values(facts).some((value) => value !== null) ? result : null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async function recordAttribution(request: Request, env: Env, principal: Principal): Promise<Response> {
|
|
97
|
+
// Only a person signed in on the dashboard, never an API key or runner.
|
|
98
|
+
if (principal.apiKeyId || principal.runner || !principal.scopes.includes("*") || principal.aegisId === DASHBOARD_SERVICE_ID) {
|
|
99
|
+
return response({ error: "Only the dashboard records sign-up attribution" }, 403);
|
|
100
|
+
}
|
|
101
|
+
const attribution = normalizeAttribution(await requestJson(request));
|
|
102
|
+
if (!attribution) return response({ recorded: false, reason: "nothing_to_record" });
|
|
103
|
+
const values = [
|
|
104
|
+
attribution.ref,
|
|
105
|
+
...UTM_FIELDS.map((field) => attribution[field]),
|
|
106
|
+
attribution.landing_path,
|
|
107
|
+
attribution.referrer_host,
|
|
108
|
+
attribution.first_seen_at,
|
|
109
|
+
];
|
|
110
|
+
const [, inserted] = await env.DB.batch([
|
|
111
|
+
// The dashboard can call this before anything else has created the row.
|
|
112
|
+
env.DB.prepare("INSERT INTO aegis_users (id) VALUES (?) ON CONFLICT(id) DO NOTHING").bind(principal.aegisId),
|
|
113
|
+
env.DB.prepare(`
|
|
114
|
+
INSERT INTO signup_attribution
|
|
115
|
+
(aegis_id, ref, utm_source, utm_medium, utm_campaign, utm_content, utm_term, landing_path, referrer_host, first_seen_at)
|
|
116
|
+
SELECT id, ?, ?, ?, ?, ?, ?, ?, ?, ?
|
|
117
|
+
FROM aegis_users
|
|
118
|
+
WHERE id = ? AND created_at > datetime('now', ?)
|
|
119
|
+
ON CONFLICT(aegis_id) DO NOTHING
|
|
120
|
+
`).bind(...values, principal.aegisId, `-${ATTRIBUTION_WINDOW_HOURS} hours`),
|
|
121
|
+
]);
|
|
122
|
+
if (inserted?.meta?.changes === 1) return response({ recorded: true }, 201);
|
|
123
|
+
const existing = await env.DB.prepare("SELECT 1 AS found FROM signup_attribution WHERE aegis_id = ?")
|
|
124
|
+
.bind(principal.aegisId).first<{ found: number }>();
|
|
125
|
+
return response({ recorded: false, reason: existing ? "already_recorded" : "account_too_old" });
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function signedInPerson(principal: Principal): boolean {
|
|
129
|
+
return !principal.apiKeyId && !principal.runner && principal.scopes.includes("*") && principal.aegisId !== DASHBOARD_SERVICE_ID;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Whether `code` is a real affiliate's code, asked of Scents (which owns the affiliate list). */
|
|
133
|
+
export async function isAffiliateCode(env: Env, code: string): Promise<boolean | null> {
|
|
134
|
+
if (!env.SCENTS) return null;
|
|
135
|
+
try {
|
|
136
|
+
const res = await env.SCENTS.fetch(`https://scents.bugmole.com/public/ref/${encodeURIComponent(code)}`);
|
|
137
|
+
if (!res.ok) return null;
|
|
138
|
+
return ((await res.json()) as { valid?: boolean }).valid === true;
|
|
139
|
+
} catch {
|
|
140
|
+
return null;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** The signed-in person's referral: the code that brought them, and whether they may still type one. */
|
|
145
|
+
async function myReferral(env: Env, principal: Principal): Promise<Response> {
|
|
146
|
+
const row = await env.DB.prepare(
|
|
147
|
+
`SELECT u.created_at, a.ref FROM aegis_users u LEFT JOIN signup_attribution a ON a.aegis_id = u.id WHERE u.id = ?`,
|
|
148
|
+
).bind(principal.aegisId).first<{ created_at: string | null; ref: string | null }>();
|
|
149
|
+
const created = row?.created_at ? Date.parse(`${row.created_at.replace(" ", "T")}Z`) : Date.now();
|
|
150
|
+
const deadline = new Date(created + REFERRAL_CODE_WINDOW_DAYS * 86_400_000);
|
|
151
|
+
return response({ ref: row?.ref ?? null, canEnterCode: !row?.ref && Date.now() < deadline.getTime(), deadline: deadline.toISOString() });
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Someone types an affiliate's referral code (they may have heard it, or
|
|
156
|
+
* declined the cookie that would have carried the link's ?ref=). Once per
|
|
157
|
+
* person, within 30 days of signing up, only if nothing credited them yet,
|
|
158
|
+
* and only a code that belongs to an affiliate, so a typo can't lock in.
|
|
159
|
+
*/
|
|
160
|
+
async function enterReferralCode(request: Request, env: Env, principal: Principal): Promise<Response> {
|
|
161
|
+
const code = ref(((await requestJson(request)) as { code?: unknown } | null)?.code);
|
|
162
|
+
if (!code) return response({ error: "Enter the referral code: letters, numbers, - or _" }, 400);
|
|
163
|
+
const valid = await isAffiliateCode(env, code);
|
|
164
|
+
if (valid === null) return response({ error: "Couldn't check that code right now. Try again in a minute." }, 503);
|
|
165
|
+
if (!valid) return response({ error: "That referral code doesn't exist. Check the spelling." }, 404);
|
|
166
|
+
const [, written] = await env.DB.batch([
|
|
167
|
+
env.DB.prepare("INSERT INTO aegis_users (id) VALUES (?) ON CONFLICT(id) DO NOTHING").bind(principal.aegisId),
|
|
168
|
+
env.DB.prepare(`
|
|
169
|
+
INSERT INTO signup_attribution (aegis_id, ref)
|
|
170
|
+
SELECT id, ? FROM aegis_users WHERE id = ? AND created_at > datetime('now', ?)
|
|
171
|
+
ON CONFLICT(aegis_id) DO UPDATE SET ref = excluded.ref WHERE signup_attribution.ref IS NULL
|
|
172
|
+
`).bind(code, principal.aegisId, `-${REFERRAL_CODE_WINDOW_DAYS} days`),
|
|
173
|
+
]);
|
|
174
|
+
if (written?.meta?.changes === 1) return response({ recorded: true, ref: code }, 201);
|
|
175
|
+
const existing = await env.DB.prepare("SELECT ref FROM signup_attribution WHERE aegis_id = ?").bind(principal.aegisId).first<{ ref: string | null }>();
|
|
176
|
+
return response(
|
|
177
|
+
existing?.ref
|
|
178
|
+
? { error: `Your account is already credited to code ${existing.ref}.`, ref: existing.ref }
|
|
179
|
+
: { error: `Referral codes can only be added in the first ${REFERRAL_CODE_WINDOW_DAYS} days after signing up.` },
|
|
180
|
+
409,
|
|
181
|
+
);
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
registerRoutes(async ({ request, env, principal, url }) => {
|
|
185
|
+
if (url.pathname === "/api/attribution/me" || url.pathname === "/api/attribution/code") {
|
|
186
|
+
if (!signedInPerson(principal)) return response({ error: "Only the dashboard can do this" }, 403);
|
|
187
|
+
if (url.pathname === "/api/attribution/me" && request.method === "GET") return myReferral(env, principal);
|
|
188
|
+
if (url.pathname === "/api/attribution/code" && request.method === "POST") return enterReferralCode(request, env, principal);
|
|
189
|
+
return response({ error: "Method not allowed" }, 405);
|
|
190
|
+
}
|
|
191
|
+
if (url.pathname !== "/api/attribution") return null;
|
|
192
|
+
if (request.method !== "POST") return response({ error: "Method not allowed" }, 405);
|
|
193
|
+
return recordAttribution(request, env, principal);
|
|
194
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import test from "node:test";
|
|
3
|
-
import { explorationPrompt, parseExploration } from "./ai-exploration.js";
|
|
3
|
+
import { explorationPrompt, parseCheckedExploration, parseExploration } from "./ai-exploration.js";
|
|
4
4
|
import type { DiscoveryResult } from "./site-discovery.js";
|
|
5
5
|
|
|
6
6
|
const result: DiscoveryResult = {
|
|
@@ -30,3 +30,36 @@ test("parseExploration only accepts well-formed journeys with at least two steps
|
|
|
30
30
|
assert.equal(flows[0].name, "Sign up then buy");
|
|
31
31
|
assert.equal(flows[0].kind, "journey");
|
|
32
32
|
});
|
|
33
|
+
|
|
34
|
+
// The real case that prompted the check: app.bugmole.com's sign-in page has one
|
|
35
|
+
// button and no links, and the model wrote journeys through a "Forgot
|
|
36
|
+
// password?" link and a sign-up page that don't exist.
|
|
37
|
+
const signInOnly: DiscoveryResult = {
|
|
38
|
+
baseUrl: "https://app.test",
|
|
39
|
+
pages: [{ route: "/login", url: "https://app.test/login", title: "Bugmole", heading: "Welcome back", path: [], links: [], forms: [], controls: ["Continue with Aegis →"] }],
|
|
40
|
+
flows: [],
|
|
41
|
+
events: [],
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
test("the prompt shows the buttons discovery saw, and forbids inventing others", () => {
|
|
45
|
+
const prompt = explorationPrompt(signInOnly);
|
|
46
|
+
assert.match(prompt, /"buttons":\["Continue with Aegis →"\]/);
|
|
47
|
+
assert.match(prompt, /Never invent a page, link or button/);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("journeys through controls or pages discovery never saw are set aside with the reason", () => {
|
|
51
|
+
const reply = JSON.stringify({
|
|
52
|
+
journeys: [
|
|
53
|
+
{ name: "Forgot password", steps: [{ openLink: "/login" }, { tapOn: "Forgot password?" }, { assertVisible: "Reset your password" }] },
|
|
54
|
+
{ name: "Sign up", steps: [{ openLink: "/signup" }, { assertVisible: "Create account" }] },
|
|
55
|
+
{ name: "Hidden check", steps: [{ openLink: "/login" }, { assertVisible: "Remember me" }, { tapOn: "Continue with Aegis" }] },
|
|
56
|
+
{ name: "Continue to Aegis", steps: [{ openLink: "/login" }, { assertVisible: "Welcome back" }, { tapOn: "Continue with Aegis" }, { assertVisible: "Sign in to Aegis" }] },
|
|
57
|
+
],
|
|
58
|
+
});
|
|
59
|
+
const { flows, setAside } = parseCheckedExploration(reply, signInOnly);
|
|
60
|
+
assert.deepEqual(flows.map((flow) => flow.name), ["Continue to Aegis"], "the last assertion may be new text: it is the outcome");
|
|
61
|
+
assert.deepEqual(setAside.map((item) => item.name), ["Forgot password", "Sign up", "Hidden check"]);
|
|
62
|
+
assert.match(setAside[0].reason, /taps “Forgot password\?”/);
|
|
63
|
+
assert.match(setAside[1].reason, /opens \/signup/);
|
|
64
|
+
assert.match(setAside[2].reason, /expects “Remember me”/);
|
|
65
|
+
});
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import YAML from "yaml";
|
|
5
5
|
import { parseMaestroStyleFlow } from "./flow-language.js";
|
|
6
6
|
import { extractJson } from "./platform-ai.js";
|
|
7
|
-
import { slugify, type DiscoveredFlow, type DiscoveryResult } from "./site-discovery.js";
|
|
7
|
+
import { slugify, type DiscoveredFlow, type DiscoveredPage, type DiscoveryResult } from "./site-discovery.js";
|
|
8
8
|
|
|
9
9
|
const MAX_JOURNEYS = 8;
|
|
10
10
|
const MAX_STEPS = 40;
|
|
@@ -19,12 +19,13 @@ export function explorationPrompt(result: DiscoveryResult, persona?: Exploration
|
|
|
19
19
|
reachedBy: page.path,
|
|
20
20
|
links: page.links.slice(0, 15).map((link) => link.label),
|
|
21
21
|
forms: page.forms.map((form) => ({ name: form.name, fields: form.fields.map((field) => `${field.label} (${field.type}${field.required ? ", required" : ""})`), submit: form.submit })),
|
|
22
|
+
...(page.controls?.length ? { buttons: page.controls.slice(0, 20) } : {}),
|
|
22
23
|
}));
|
|
23
24
|
return [
|
|
24
25
|
"You are a QA engineer. Below is a map of a web app a crawler just explored.",
|
|
25
26
|
...(persona ? [`Decide what's worth testing from this persona's point of view — ${persona.name}: ${persona.description}`] : []),
|
|
26
27
|
`Write up to ${MAX_JOURNEYS} end-to-end user journeys worth testing that span several pages (for example: sign up, then buy something).`,
|
|
27
|
-
"Only use pages, links, buttons and fields that appear in the map. Don't repeat single-page visits; those already exist.",
|
|
28
|
+
"Only use pages, links, buttons and fields that appear in the map, with their exact text. Never invent a page, link or button; if the map is too small for a journey, return fewer journeys or none. Don't repeat single-page visits; those already exist.",
|
|
28
29
|
"Use obvious sample data (qa@example.com, Bugmole). Write passwords as \"{{secret.APP_PASSWORD}}\". Never submit destructive actions (delete, cancel, unsubscribe).",
|
|
29
30
|
"Steps: openLink: \"/path\"; tapOn: \"<visible text>\"; inputText: \"<text>\"; assertVisible: \"<text>\"; scrollUntilVisible: { element: \"<text>\" }.",
|
|
30
31
|
"Every journey starts with openLink and ends with an assertVisible that proves it worked.",
|
|
@@ -35,7 +36,99 @@ export function explorationPrompt(result: DiscoveryResult, persona?: Exploration
|
|
|
35
36
|
].join("\n");
|
|
36
37
|
}
|
|
37
38
|
|
|
38
|
-
|
|
39
|
+
const normalize = (value: string) => value.toLowerCase().replace(/\s+/g, " ").replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Everything a visitor could see or type into on the explored pages, and the
|
|
43
|
+
* routes that exist. A journey step outside this is something the model made
|
|
44
|
+
* up, however plausible ("Forgot password?" on a page that has no such link).
|
|
45
|
+
*/
|
|
46
|
+
function observedOn(pages: readonly DiscoveredPage[], baseUrl: string) {
|
|
47
|
+
const words = new Set<string>();
|
|
48
|
+
const add = (value?: string) => {
|
|
49
|
+
const normal = value ? normalize(value) : "";
|
|
50
|
+
if (normal) words.add(normal);
|
|
51
|
+
};
|
|
52
|
+
const routes = new Set<string>(["/"]);
|
|
53
|
+
for (const page of pages) {
|
|
54
|
+
routes.add(page.route);
|
|
55
|
+
add(page.title);
|
|
56
|
+
add(page.heading);
|
|
57
|
+
for (const link of page.links) {
|
|
58
|
+
add(link.label);
|
|
59
|
+
add(link.menu);
|
|
60
|
+
routes.add(link.route);
|
|
61
|
+
}
|
|
62
|
+
for (const form of page.forms) {
|
|
63
|
+
add(form.name);
|
|
64
|
+
add(form.submit);
|
|
65
|
+
for (const field of form.fields) add(field.label);
|
|
66
|
+
}
|
|
67
|
+
for (const control of page.controls ?? []) add(control);
|
|
68
|
+
}
|
|
69
|
+
const seen = (text: string) => {
|
|
70
|
+
const target = normalize(text);
|
|
71
|
+
if (!target) return false;
|
|
72
|
+
for (const word of words) {
|
|
73
|
+
if (word === target || (target.length >= 3 && word.includes(target)) || (word.length >= 3 && target.includes(word))) return true;
|
|
74
|
+
}
|
|
75
|
+
return false;
|
|
76
|
+
};
|
|
77
|
+
const routeExists = (link: string) => {
|
|
78
|
+
try {
|
|
79
|
+
const url = new URL(link, baseUrl);
|
|
80
|
+
if (url.origin !== new URL(baseUrl).origin) return false;
|
|
81
|
+
return routes.has(url.pathname) || routes.has(url.pathname.replace(/\/$/, "") || "/");
|
|
82
|
+
} catch {
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
return { seen, routeExists };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Why a journey can't be trusted, or null. Taps, scrolls and assertions along
|
|
91
|
+
* the way must name something observed, and every openLink a real route. The
|
|
92
|
+
* last assertVisible may name new text: it is the outcome the journey proves
|
|
93
|
+
* (a confirmation after a form), which discovery, being read-only, never saw.
|
|
94
|
+
*/
|
|
95
|
+
function unobservedStep(steps: readonly unknown[], observed: ReturnType<typeof observedOn>): string | null {
|
|
96
|
+
for (const [index, raw] of steps.entries()) {
|
|
97
|
+
const step = raw as Record<string, unknown>;
|
|
98
|
+
if (typeof step?.openLink === "string" && !observed.routeExists(step.openLink)) return `opens ${step.openLink}, which wasn't found`;
|
|
99
|
+
const tap = typeof step?.tapOn === "string" ? step.tapOn : undefined;
|
|
100
|
+
if (tap && !observed.seen(tap)) return `taps “${tap}”, which isn't on any explored page`;
|
|
101
|
+
const scroll = (step?.scrollUntilVisible as { element?: unknown } | undefined)?.element;
|
|
102
|
+
if (typeof scroll === "string" && !observed.seen(scroll)) return `looks for “${scroll}”, which isn't on any explored page`;
|
|
103
|
+
const assertion = typeof step?.assertVisible === "string" ? step.assertVisible : undefined;
|
|
104
|
+
if (assertion && index < steps.length - 1 && !observed.seen(assertion)) return `expects “${assertion}”, which isn't on any explored page`;
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export type CheckedExploration = { flows: DiscoveredFlow[]; setAside: Array<{ name: string; reason: string }> };
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Parses the model's journeys and keeps only those built from what discovery
|
|
113
|
+
* actually saw on `pages`; the rest are set aside with the reason.
|
|
114
|
+
*/
|
|
115
|
+
export function parseCheckedExploration(reply: string, result: Pick<DiscoveryResult, "baseUrl" | "pages" | "flows">): CheckedExploration {
|
|
116
|
+
const observed = observedOn(result.pages, result.baseUrl);
|
|
117
|
+
const setAside: CheckedExploration["setAside"] = [];
|
|
118
|
+
const flows = parseExploration(reply, result.baseUrl, result.flows, (name, steps) => {
|
|
119
|
+
const reason = unobservedStep(steps, observed);
|
|
120
|
+
if (reason) setAside.push({ name, reason });
|
|
121
|
+
return !reason;
|
|
122
|
+
});
|
|
123
|
+
return { flows, setAside };
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function parseExploration(
|
|
127
|
+
reply: string,
|
|
128
|
+
baseUrl: string,
|
|
129
|
+
existing: readonly DiscoveredFlow[],
|
|
130
|
+
accept: (name: string, steps: readonly unknown[]) => boolean = () => true,
|
|
131
|
+
): DiscoveredFlow[] {
|
|
39
132
|
const data = extractJson(reply) as { journeys?: unknown } | null;
|
|
40
133
|
if (!data || !Array.isArray(data.journeys)) return [];
|
|
41
134
|
const taken = new Set(existing.map((flow) => flow.id));
|
|
@@ -44,6 +137,7 @@ export function parseExploration(reply: string, baseUrl: string, existing: reado
|
|
|
44
137
|
const name = typeof journey?.name === "string" ? journey.name.trim().slice(0, 80) : "";
|
|
45
138
|
const steps = Array.isArray(journey?.steps) ? journey.steps.slice(0, MAX_STEPS) : [];
|
|
46
139
|
if (!name || steps.length < 2) continue;
|
|
140
|
+
if (!accept(name, steps)) continue;
|
|
47
141
|
const content = `url: ${baseUrl}\nname: ${JSON.stringify(name)}\n---\n${YAML.stringify(steps)}`;
|
|
48
142
|
let parsed;
|
|
49
143
|
try {
|
|
@@ -139,6 +139,12 @@ export class AppiumDriver implements AppDriver {
|
|
|
139
139
|
}
|
|
140
140
|
counter.n += step.steps.length;
|
|
141
141
|
if (lastError) throw lastError;
|
|
142
|
+
} else if (step.type === "assertLayout" || step.type === "assertLinks") {
|
|
143
|
+
// Page checks read the page's layout and links, which a device
|
|
144
|
+
// cloud session doesn't expose; they run in the browser targets.
|
|
145
|
+
steps.push({ sequenceNumber: index, status: "passed", label: describeStepLabel(step), message: "Not checked on device clouds; checked in the browser targets", durationMs: 0 });
|
|
146
|
+
log.push(`Step ${index + 1}: ${describeStepForLog(step)} — skipped (checked in browsers, not on device clouds)`);
|
|
147
|
+
continue;
|
|
142
148
|
} else {
|
|
143
149
|
await this.step(session!, step, mode, flow, context, outDir, artifacts);
|
|
144
150
|
}
|
|
@@ -6,8 +6,8 @@ import { createProjectObjectStore } from "../storage/create-object-store.js";
|
|
|
6
6
|
import { projectPrefix } from "../storage/keys.js";
|
|
7
7
|
import { readJourneyGraph, writeJourneyGraph } from "./journey-editor.js";
|
|
8
8
|
import type { JourneyEdge, JourneyNode, UserJourney } from "./journey-graph.js";
|
|
9
|
-
import { discoverSite, type DiscoveryResult } from "./site-discovery.js";
|
|
10
|
-
import { explorationPrompt,
|
|
9
|
+
import { discoverSite, stoppedAtSignIn, type DiscoveryResult } from "./site-discovery.js";
|
|
10
|
+
import { explorationPrompt, parseCheckedExploration, type ExplorationPersona } from "./ai-exploration.js";
|
|
11
11
|
import { modelCaller } from "./platform-ai.js";
|
|
12
12
|
|
|
13
13
|
/** Discovery tasks queued as "Explore with AI …" also have a model write multi-page journeys. */
|
|
@@ -248,7 +248,21 @@ export async function runDiscoveryTask(
|
|
|
248
248
|
await send("running");
|
|
249
249
|
},
|
|
250
250
|
});
|
|
251
|
-
|
|
251
|
+
// Everything reachable was a sign-in screen. Say so plainly, and don't
|
|
252
|
+
// ask AI to imagine the app behind it: that spends credits on journeys
|
|
253
|
+
// through pages nobody has seen.
|
|
254
|
+
const signInWall = stoppedAtSignIn(result);
|
|
255
|
+
if (signInWall) {
|
|
256
|
+
result.events.push({
|
|
257
|
+
at: new Date().toISOString(),
|
|
258
|
+
kind: "skip",
|
|
259
|
+
text: "Stopped at a sign-in page: Bugmole Cloud can't sign in to your app yet, so everything behind it is unexplored. "
|
|
260
|
+
+ "To explore signed in, run discovery from your machine or CI with a sign-in flow: bugmole --mode discover --sign-in <flow>.",
|
|
261
|
+
});
|
|
262
|
+
latest = progressOf(url, result);
|
|
263
|
+
await send("running");
|
|
264
|
+
}
|
|
265
|
+
if (discoveryUsesAi(task) && !signInWall) {
|
|
252
266
|
const caller = modelCaller(cfg, { projectId: cfg.project?.id ?? "default", requestId: task.id });
|
|
253
267
|
const note = (text: string, kind: "flows" | "error") => {
|
|
254
268
|
result.events.push({ at: new Date().toISOString(), kind, text });
|
|
@@ -262,9 +276,13 @@ export async function runDiscoveryTask(
|
|
|
262
276
|
await note(persona ? `Asking AI for multi-page journeys as ${persona.name}…` : "Asking AI for multi-page journeys…", "flows");
|
|
263
277
|
try {
|
|
264
278
|
const reply = await caller(explorationPrompt(result, persona ?? undefined), "exploration");
|
|
265
|
-
const journeys =
|
|
279
|
+
const { flows: journeys, setAside } = parseCheckedExploration(reply.text, result);
|
|
266
280
|
result.flows.push(...journeys);
|
|
267
281
|
await note(`AI wrote ${journeys.length} journey${journeys.length === 1 ? "" : "s"}`, "flows");
|
|
282
|
+
for (const journey of setAside.slice(0, 5)) {
|
|
283
|
+
await note(`Set aside “${journey.name}”: it ${journey.reason}`, "error");
|
|
284
|
+
}
|
|
285
|
+
if (setAside.length > 5) await note(`Set aside ${setAside.length - 5} more that used things discovery didn't see`, "error");
|
|
268
286
|
} catch (error) {
|
|
269
287
|
await note(`AI exploration failed: ${error instanceof Error ? error.message : String(error)}`, "error");
|
|
270
288
|
}
|
|
@@ -273,7 +291,8 @@ export async function runDiscoveryTask(
|
|
|
273
291
|
const planIds = await saveDiscovery(cfg, client, result);
|
|
274
292
|
latest = progressOf(url, result, planIds);
|
|
275
293
|
await send("completed", {
|
|
276
|
-
completionSummary: `Explored ${result.pages.length} page(s) of ${url} and discovered ${result.flows.length} flow(s)
|
|
294
|
+
completionSummary: `Explored ${result.pages.length} page(s) of ${url} and discovered ${result.flows.length} flow(s).`
|
|
295
|
+
+ (signInWall ? " Stopped at a sign-in page; the app behind it is unexplored." : ""),
|
|
277
296
|
});
|
|
278
297
|
} catch (error) {
|
|
279
298
|
await send("error", { errorMessage: error instanceof Error ? error.message : String(error) });
|
|
@@ -25,6 +25,18 @@ export type FlowStep =
|
|
|
25
25
|
* sitting on top of a link or button and silently swallowing the tap.
|
|
26
26
|
*/
|
|
27
27
|
| { type: "assertClickable"; text: string; index: number; timeoutMs?: number }
|
|
28
|
+
/**
|
|
29
|
+
* The page looks right at this screen size: it doesn't scroll sideways,
|
|
30
|
+
* no text spills out of the card or box it sits in, and none is cut off
|
|
31
|
+
* without an ellipsis. What a person notices first on a phone.
|
|
32
|
+
*/
|
|
33
|
+
| { type: "assertLayout" }
|
|
34
|
+
/**
|
|
35
|
+
* Every link on the page leads somewhere. Same-site links must answer;
|
|
36
|
+
* with `external`, other sites' links are checked too (a 404, 410 or
|
|
37
|
+
* server error fails; sites that refuse robots are not counted).
|
|
38
|
+
*/
|
|
39
|
+
| { type: "assertLinks"; external: boolean }
|
|
28
40
|
| { type: "scrollUntilVisible"; text: string }
|
|
29
41
|
| { type: "wait"; ms: number }
|
|
30
42
|
| { type: "takeScreenshot"; name: string }
|
|
@@ -128,6 +140,8 @@ function parseStep(raw: unknown): FlowStep {
|
|
|
128
140
|
if (raw === "launchApp") return { type: "launchApp" };
|
|
129
141
|
if (raw === "stopApp") return { type: "stopApp" };
|
|
130
142
|
if (raw === "eraseText") return { type: "eraseText" };
|
|
143
|
+
if (raw === "assertLayout") return { type: "assertLayout" };
|
|
144
|
+
if (raw === "assertLinks") return { type: "assertLinks", external: false };
|
|
131
145
|
if (raw === "requestTempEmail") return { type: "requestTempEmail", as: "tempEmail" };
|
|
132
146
|
if (raw === "openEmailLink") return { type: "openEmailLink", as: "tempEmail" };
|
|
133
147
|
// Maestro waits for animations before asserting; the web driver's
|
|
@@ -187,6 +201,11 @@ function parseStep(raw: unknown): FlowStep {
|
|
|
187
201
|
}
|
|
188
202
|
return { type: "assertVisible", text: requireText(value.visible, "extendedWaitUntil"), timeoutMs };
|
|
189
203
|
}
|
|
204
|
+
if ("assertLayout" in record) return { type: "assertLayout" };
|
|
205
|
+
if ("assertLinks" in record) {
|
|
206
|
+
const value = asRecord(record.assertLinks);
|
|
207
|
+
return { type: "assertLinks", external: Boolean(value?.external) };
|
|
208
|
+
}
|
|
190
209
|
if ("scrollUntilVisible" in record) {
|
|
191
210
|
const value = asRecord(record.scrollUntilVisible);
|
|
192
211
|
return { type: "scrollUntilVisible", text: requireText(value ? value.element : record.scrollUntilVisible, "scrollUntilVisible") };
|
|
@@ -351,6 +370,8 @@ export function describeStepLabel(step: FlowStep): string {
|
|
|
351
370
|
case "assertVisible": return `See “${step.text}”`;
|
|
352
371
|
case "assertNotVisible": return `No “${step.text}”`;
|
|
353
372
|
case "assertClickable": return `“${step.text}” is clickable`;
|
|
373
|
+
case "assertLayout": return "Nothing spills off the screen";
|
|
374
|
+
case "assertLinks": return step.external ? "Every link works" : "Every link on the site works";
|
|
354
375
|
case "scrollUntilVisible": return `Scroll to “${step.text}”`;
|
|
355
376
|
case "wait": return `Wait ${step.ms}ms`;
|
|
356
377
|
case "takeScreenshot": return `Screenshot “${step.name}”`;
|
|
@@ -98,6 +98,45 @@ function nextRevision(journey: UserJourney): UserJourney {
|
|
|
98
98
|
};
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
+
/**
|
|
102
|
+
* A run that passed proves the flow it ran. Discovery saves what it finds as
|
|
103
|
+
* draft journeys, and only stable ones are drawn in the dashboard's journey
|
|
104
|
+
* graph, but nothing promoted them: a cloud run has no agent to call
|
|
105
|
+
* stabilizeJourney. This does it for the drafts discovery wrote, reading the
|
|
106
|
+
* stored graph (a cloud container has no local copy). Journeys an agent or a
|
|
107
|
+
* person made are left for them to stabilize. Returns the journey promoted.
|
|
108
|
+
*/
|
|
109
|
+
export async function stabilizeAfterPassingRun(
|
|
110
|
+
cfg: JourneyConfig,
|
|
111
|
+
run: { id: string; planId?: string | null; journeyId?: string | null },
|
|
112
|
+
): Promise<string | null> {
|
|
113
|
+
const store = storeFor(cfg);
|
|
114
|
+
const key = `${projectPrefix(projectId(cfg))}/journeys/graph.json`;
|
|
115
|
+
if (!(await store.has(key))) return null;
|
|
116
|
+
const parsed = JSON.parse(await store.getText(key)) as JourneyGraph;
|
|
117
|
+
const graph = repairJourneyGraph({
|
|
118
|
+
...emptyJourneyGraph(projectId(cfg)),
|
|
119
|
+
...parsed,
|
|
120
|
+
journeys: Array.isArray(parsed.journeys) ? parsed.journeys : [],
|
|
121
|
+
nodes: Array.isArray(parsed.nodes) ? parsed.nodes : [],
|
|
122
|
+
edges: Array.isArray(parsed.edges) ? parsed.edges : [],
|
|
123
|
+
}, projectId(cfg));
|
|
124
|
+
const current = graph.journeys.find((journey) =>
|
|
125
|
+
(run.journeyId && journey.id === run.journeyId) || (run.planId && journey.metadata?.planId === run.planId));
|
|
126
|
+
if (!current || current.stability === "stable" || current.metadata?.generatedBy !== "site-discovery") return null;
|
|
127
|
+
const journey: UserJourney = {
|
|
128
|
+
...current,
|
|
129
|
+
status: "published",
|
|
130
|
+
stability: "stable",
|
|
131
|
+
linkedToGraph: true,
|
|
132
|
+
verificationRunId: run.id,
|
|
133
|
+
verifiedAt: new Date().toISOString(),
|
|
134
|
+
};
|
|
135
|
+
graph.journeys = graph.journeys.map((item) => item.id === current.id ? journey : item);
|
|
136
|
+
await writeJourneyGraph(cfg, graph);
|
|
137
|
+
return journey.id;
|
|
138
|
+
}
|
|
139
|
+
|
|
101
140
|
export async function stabilizeJourney(
|
|
102
141
|
cfg: JourneyConfig,
|
|
103
142
|
input: { journeyId: string; verificationRunId: string; verifiedAt?: string },
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
addJourneyTransition,
|
|
10
10
|
createJourney,
|
|
11
11
|
reuseJourneyStep,
|
|
12
|
+
stabilizeAfterPassingRun,
|
|
12
13
|
stabilizeJourney,
|
|
13
14
|
nextJourneyTask,
|
|
14
15
|
readJourneyGraph,
|
|
@@ -255,3 +256,44 @@ test("requires runtime transition and terminal-goal completion for source paths"
|
|
|
255
256
|
assert.equal(nextJourneyTask(cfg, "top-up-flow").status, "exhausted");
|
|
256
257
|
await fs.promises.rm(root, { recursive: true, force: true });
|
|
257
258
|
});
|
|
259
|
+
|
|
260
|
+
test("a passing run promotes the explored draft it proved, from the stored graph alone", async () => {
|
|
261
|
+
const root = fs.mkdtempSync(path.join(os.tmpdir(), "bugmole-promote-"));
|
|
262
|
+
const storage = { provider: "local", localDirectory: path.join(root, "artifacts") };
|
|
263
|
+
const writer = { project: { id: "demo" }, runtime: { spec_dir: path.join(root, "spec"), artifacts_dir: path.join(root, "artifacts") }, storage };
|
|
264
|
+
// A cloud container: same project storage, no local copy of the graph.
|
|
265
|
+
const container = { project: { id: "demo" }, runtime: { spec_dir: path.join(root, "container"), artifacts_dir: path.join(root, "artifacts") }, storage };
|
|
266
|
+
try {
|
|
267
|
+
const graph = emptyJourneyGraph("demo");
|
|
268
|
+
graph.nodes = [{ id: "screen:/login", title: "Welcome back", kind: "screen", route: "/login" }];
|
|
269
|
+
const draft = (id: string, generatedBy?: string) => ({
|
|
270
|
+
id,
|
|
271
|
+
name: id,
|
|
272
|
+
actor: "visitor",
|
|
273
|
+
startNodeId: "screen:/login",
|
|
274
|
+
nodeIds: ["screen:/login"],
|
|
275
|
+
edgeIds: [],
|
|
276
|
+
revision: 1,
|
|
277
|
+
status: "draft" as const,
|
|
278
|
+
sourceOfTruth: "canonical_flow" as const,
|
|
279
|
+
stability: "discovered" as const,
|
|
280
|
+
linkedToGraph: false,
|
|
281
|
+
metadata: generatedBy ? { generatedBy, planId: id } : {},
|
|
282
|
+
});
|
|
283
|
+
graph.journeys = [draft("discovered-welcome-back", "site-discovery"), draft("hand-made")];
|
|
284
|
+
await writeJourneyGraph(writer, graph);
|
|
285
|
+
|
|
286
|
+
assert.equal(await stabilizeAfterPassingRun(container, { id: "run_1", planId: "discovered-welcome-back", journeyId: "discovered-welcome-back" }), "discovered-welcome-back");
|
|
287
|
+
const stored = JSON.parse(fs.readFileSync(path.join(root, "artifacts", "projects", "demo", "journeys", "graph.json"), "utf8"));
|
|
288
|
+
const promoted = stored.journeys.find((journey: { id: string }) => journey.id === "discovered-welcome-back");
|
|
289
|
+
assert.equal(promoted.stability, "stable");
|
|
290
|
+
assert.equal(promoted.linkedToGraph, true);
|
|
291
|
+
assert.equal(promoted.verificationRunId, "run_1");
|
|
292
|
+
|
|
293
|
+
assert.equal(await stabilizeAfterPassingRun(container, { id: "run_2", journeyId: "hand-made" }), null, "journeys a person or agent made are theirs to stabilize");
|
|
294
|
+
assert.equal(await stabilizeAfterPassingRun(container, { id: "run_3", journeyId: "discovered-welcome-back" }), null, "already stable");
|
|
295
|
+
assert.equal(await stabilizeAfterPassingRun(container, { id: "run_4", journeyId: "missing" }), null);
|
|
296
|
+
} finally {
|
|
297
|
+
fs.rmSync(root, { recursive: true, force: true });
|
|
298
|
+
}
|
|
299
|
+
});
|