@ihyeon/flow-qa 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ko.md +9 -3
- package/README.md +9 -3
- package/THIRD_PARTY_NOTICES.md +1 -1
- package/dist/src/cli.js +42 -8
- package/dist/src/compiler.js +11 -3
- package/dist/src/engine.js +62 -10
- package/dist/src/i18n.js +11 -0
- package/dist/src/mobile-runtime.js +66 -0
- package/dist/src/planner.js +19 -7
- package/dist/src/report.js +4 -1
- package/dist/src/schema.js +21 -1
- package/dist/src/targets.js +20 -0
- package/dist/src/ui/public/app.js +32 -14
- package/dist/src/ui/public/i18n.js +18 -3
- package/dist/src/ui/public/index.html +4 -2
- package/dist/src/ui/public/style.css +1 -1
- package/dist/src/ui/server.js +35 -15
- package/docs/architecture.md +7 -5
- package/docs/mobile.md +59 -0
- package/docs/releases.md +9 -0
- package/docs/ui.md +7 -7
- package/package.json +4 -2
package/dist/src/schema.js
CHANGED
|
@@ -12,6 +12,9 @@ export const locatorSchema = z.discriminatedUnion('by', [
|
|
|
12
12
|
]);
|
|
13
13
|
export const stepSchema = z.discriminatedUnion('type', [
|
|
14
14
|
z.object({ type: z.literal('open'), path: text.refine((v) => v.startsWith('/') && !v.startsWith('//') && !v.includes('\\'), 'Use an app-relative path starting with /.') }).strict(),
|
|
15
|
+
z.object({ type: z.literal('launchApp') }).strict(),
|
|
16
|
+
z.object({ type: z.literal('back') }).strict(),
|
|
17
|
+
z.object({ type: z.literal('swipe'), direction: z.enum(['up', 'down', 'left', 'right']) }).strict(),
|
|
15
18
|
z.object({ type: z.literal('act'), instruction: text }).strict(),
|
|
16
19
|
z.object({ type: z.literal('click'), locator: locatorSchema }).strict(),
|
|
17
20
|
z.object({ type: z.literal('fill'), locator: locatorSchema, value: z.string().max(8192) }).strict(),
|
|
@@ -30,6 +33,7 @@ const caseSchema = z.object({
|
|
|
30
33
|
}).strict();
|
|
31
34
|
export const suiteSchema = z.object({
|
|
32
35
|
schemaVersion: z.literal('flow-qa/suite-1'),
|
|
36
|
+
platform: z.enum(['web', 'mobile']).optional(),
|
|
33
37
|
id: idSchema,
|
|
34
38
|
title: text,
|
|
35
39
|
source: z.object({ file: text, sha256: z.string().regex(/^[a-f0-9]{64}$/) }).strict().optional(),
|
|
@@ -49,6 +53,13 @@ export const suiteSchema = z.object({
|
|
|
49
53
|
ctx.addIssue({ code: 'custom', path: ['cases', index, 'requirementIds'], message: 'Unknown requirement ID.' });
|
|
50
54
|
if (!test.questions.length && !test.steps.some((s) => s.type.startsWith('assert')))
|
|
51
55
|
ctx.addIssue({ code: 'custom', path: ['cases', index, 'steps'], message: 'A runnable case needs an explicit assertion.' });
|
|
56
|
+
test.steps.forEach((step, stepIndex) => {
|
|
57
|
+
const invalid = suite.platform === 'mobile'
|
|
58
|
+
? step.type === 'open' || step.type === 'assertUrl'
|
|
59
|
+
: step.type === 'launchApp' || step.type === 'back' || step.type === 'swipe';
|
|
60
|
+
if (invalid)
|
|
61
|
+
ctx.addIssue({ code: 'custom', path: ['cases', index, 'steps', stepIndex, 'type'], message: `The ${step.type} step is not available on ${suite.platform ?? 'web'} suites.` });
|
|
62
|
+
});
|
|
52
63
|
});
|
|
53
64
|
});
|
|
54
65
|
export const modelSchema = z.object({
|
|
@@ -57,10 +68,19 @@ export const modelSchema = z.object({
|
|
|
57
68
|
apiKeyEnv: z.string().regex(/^[A-Z][A-Z0-9_]*$/).optional(),
|
|
58
69
|
}).strict();
|
|
59
70
|
export const plannerSchema = modelSchema;
|
|
60
|
-
|
|
71
|
+
const webTargetSchema = z.object({
|
|
61
72
|
url: httpURL,
|
|
62
73
|
description: z.string().max(2000).default(''),
|
|
74
|
+
kind: z.literal('web').optional(),
|
|
63
75
|
}).strict();
|
|
76
|
+
const mobileTargetSchema = z.object({
|
|
77
|
+
kind: z.literal('mobile'),
|
|
78
|
+
platform: z.enum(['ios', 'android']),
|
|
79
|
+
bundleId: z.string().min(1).max(200).regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/, 'Use a native app ID without spaces or control characters.'),
|
|
80
|
+
device: z.string().trim().min(1).max(200).refine((value) => !/[\p{Cc}\p{Cf}]/u.test(value), 'Device names cannot contain control characters.'),
|
|
81
|
+
description: z.string().max(2000).default(''),
|
|
82
|
+
}).strict();
|
|
83
|
+
export const targetSchema = z.union([webTargetSchema, mobileTargetSchema]);
|
|
64
84
|
export const configSchema = z.object({
|
|
65
85
|
schemaVersion: z.literal('flow-qa/config-1'),
|
|
66
86
|
targets: z.record(idSchema, targetSchema),
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
export function targetKind(target) {
|
|
3
|
+
return target.kind === 'mobile' ? 'mobile' : 'web';
|
|
4
|
+
}
|
|
5
|
+
export function targetLabel(target) {
|
|
6
|
+
return target.kind === 'mobile'
|
|
7
|
+
? `${target.platform} · ${target.bundleId} · ${target.device}`
|
|
8
|
+
: target.url;
|
|
9
|
+
}
|
|
10
|
+
export function targetFingerprint(target) {
|
|
11
|
+
const identity = target.kind === 'mobile'
|
|
12
|
+
? { kind: 'mobile', platform: target.platform, bundleId: target.bundleId, device: target.device }
|
|
13
|
+
: { kind: 'web', url: target.url };
|
|
14
|
+
return createHash('sha256').update(JSON.stringify(identity)).digest('hex');
|
|
15
|
+
}
|
|
16
|
+
export function reportTarget(name, target) {
|
|
17
|
+
return target.kind === 'mobile'
|
|
18
|
+
? { name, kind: 'mobile', platform: target.platform, bundleId: target.bundleId, device: target.device }
|
|
19
|
+
: { name, url: target.url, ...(target.kind === 'web' ? { kind: 'web' } : {}) };
|
|
20
|
+
}
|
|
@@ -55,10 +55,16 @@
|
|
|
55
55
|
$('sidebar').classList.remove('nav-open'); $('nav-toggle').setAttribute('aria-expanded', 'false');
|
|
56
56
|
if (name === 'history') renderHistory();
|
|
57
57
|
}
|
|
58
|
+
function suitePlatform() { return app.draft?.platform || 'web'; }
|
|
59
|
+
function targetPlatform(target) { return target?.kind === 'mobile' ? 'mobile' : 'web'; }
|
|
60
|
+
function selectedTarget() { return app.displayTargets.find((target) => target.name === $('target-select').value); }
|
|
61
|
+
function targetLabel(target) { return target?.kind === 'mobile' ? `${target.platform === 'ios' ? 'iOS' : 'Android'} · ${target.bundleId} · ${target.device}` : target?.url || ''; }
|
|
62
|
+
function targetDisplay(target) { return `${target?.name || t('app.noTarget')} · ${targetLabel(target)}`; }
|
|
63
|
+
function compatibleTarget() { const target = selectedTarget(); return target && targetPlatform(target) === suitePlatform(); }
|
|
58
64
|
function readiness() {
|
|
59
65
|
const active = app.state?.active;
|
|
60
66
|
const ready = !!app.draft && !!app.script && !app.stale && !app.dirty;
|
|
61
|
-
$('run-btn').disabled = !app.connected || !ready || !app.savedFile || !app.savedHash || !$('review-checkbox').checked ||
|
|
67
|
+
$('run-btn').disabled = !app.connected || !ready || !app.savedFile || !app.savedHash || !$('review-checkbox').checked || !compatibleTarget() || !!active || app.busy;
|
|
62
68
|
$('save-btn').disabled = !ready || app.busy;
|
|
63
69
|
$('script-download').disabled = !ready;
|
|
64
70
|
for (const id of ['stop-btn', 'global-stop-btn']) {
|
|
@@ -72,6 +78,8 @@
|
|
|
72
78
|
$('flow-input').disabled = app.busy;
|
|
73
79
|
$('suite-editor').disabled = app.busy;
|
|
74
80
|
$('prompt-btn').disabled = app.busy;
|
|
81
|
+
$('scenario-platform').disabled = app.busy;
|
|
82
|
+
$('mobile-setup-hint').hidden = suitePlatform() !== 'mobile' && $('scenario-platform').value !== 'mobile';
|
|
75
83
|
$('run-readiness').textContent = t(!app.connected ? 'app.connectionRequired'
|
|
76
84
|
: active ? (active.status === 'stopping' ? 'app.stopping' : active.kind === 'plan' ? 'app.planning' : 'app.running')
|
|
77
85
|
: app.stale ? 'app.stale'
|
|
@@ -79,6 +87,7 @@
|
|
|
79
87
|
: !ready ? 'app.needValidated'
|
|
80
88
|
: !app.savedFile || !app.savedHash ? 'app.needSaved'
|
|
81
89
|
: !$('target-select').value ? 'app.needTarget'
|
|
90
|
+
: !compatibleTarget() ? 'app.platformMismatch'
|
|
82
91
|
: !$('review-checkbox').checked ? 'app.needReview' : 'app.ready');
|
|
83
92
|
const cases = app.draft?.cases || [];
|
|
84
93
|
const blocked = cases.filter((item) => item.questions?.length || (item.steps.some((step) => ['act', 'assertAgent'].includes(step.type)) && !app.state?.models?.agent?.keyReady)).length;
|
|
@@ -98,6 +107,9 @@
|
|
|
98
107
|
const target = locatorText(step.locator);
|
|
99
108
|
switch (step.type) {
|
|
100
109
|
case 'open': return t('app.stepOpen', { path: step.path });
|
|
110
|
+
case 'launchApp': return t('app.stepLaunchApp');
|
|
111
|
+
case 'back': return t('app.stepBack');
|
|
112
|
+
case 'swipe': return t('app.stepSwipe', { direction: step.direction });
|
|
101
113
|
case 'click': return t('app.stepClick', { target });
|
|
102
114
|
case 'fill': return t('app.stepFill', { target, value: step.value });
|
|
103
115
|
case 'assertVisible': return t(step.visible === false ? 'app.stepHidden' : 'app.stepVisible', { target });
|
|
@@ -137,23 +149,24 @@
|
|
|
137
149
|
function setDraft(suite, script, origin, sourceFlow = '', savedFile = '', savedHash = '') {
|
|
138
150
|
app.draft = suite; app.script = script; app.origin = origin; app.sourceFlow = sourceFlow; app.boundSource = suite.source ? { ...suite.source } : null; app.savedFile = savedFile; app.savedHash = savedHash; app.stale = false; app.dirty = false;
|
|
139
151
|
$('suite-editor').value = JSON.stringify(suite, null, 2);
|
|
152
|
+
$('scenario-platform').value = suite.platform || 'web';
|
|
140
153
|
$('review-checkbox').checked = false;
|
|
141
154
|
summarize(suite); readiness();
|
|
142
155
|
}
|
|
143
156
|
function targetOptions() {
|
|
144
157
|
const selected = $('target-select').value;
|
|
145
|
-
const
|
|
158
|
+
const previousFingerprint = app.displayTargets.find((item) => item.name === selected)?.targetFingerprint;
|
|
146
159
|
app.displayTargets = (app.state?.targets || []).map((target) => ({ ...target }));
|
|
147
160
|
replace($('target-select'), el('option', '', t('scenario.selectTarget')));
|
|
148
161
|
$('target-select').firstChild.value = '';
|
|
149
|
-
for (const target of app.state?.targets || []) { const option = el('option', '', target
|
|
162
|
+
for (const target of app.state?.targets || []) { const option = el('option', '', targetDisplay(target)); option.value = target.name; $('target-select').append(option); }
|
|
150
163
|
if ((app.state?.targets || []).some((x) => x.name === selected)) $('target-select').value = selected;
|
|
151
|
-
if (
|
|
164
|
+
if (previousFingerprint !== app.displayTargets.find((item) => item.name === selected)?.targetFingerprint) $('review-checkbox').checked = false;
|
|
152
165
|
targetDescription();
|
|
153
166
|
const list = $('target-list'); replace(list);
|
|
154
|
-
for (const target of app.state?.targets || []) { const row = el('div', 'settings-entry'); row.append(el('strong', '', target.name), document.createTextNode(` · ${target
|
|
167
|
+
for (const target of app.state?.targets || []) { const row = el('div', 'settings-entry'); row.append(el('strong', '', target.name), document.createTextNode(` · ${targetLabel(target)}${target.description ? ` · ${target.description}` : ''}`)); list.append(row); }
|
|
155
168
|
}
|
|
156
|
-
function targetDescription() { const target =
|
|
169
|
+
function targetDescription() { const target = selectedTarget(); $('target-description').textContent = target ? `${targetLabel(target)}${target.description ? ` · ${target.description}` : ''}` : t('scenario.targetHint'); readiness(); }
|
|
157
170
|
function modelDescription(role) {
|
|
158
171
|
const model = app.state?.models?.[role] || {};
|
|
159
172
|
const explanation = t(role === 'planner' ? 'app.modelPlannerHelp' : 'app.modelAgentHelp');
|
|
@@ -192,7 +205,7 @@
|
|
|
192
205
|
const runs = app.state?.runs || [];
|
|
193
206
|
if (!runs.length) { host.append(el('p', 'empty', t(app.state?.active ? 'app.historyActiveEmpty' : 'app.historyEmpty'))); return; }
|
|
194
207
|
for (const run of runs) {
|
|
195
|
-
const row = el('div', 'list-row'); const copy = el('div'); copy.append(el('strong', '', `${run.suiteId} · ${i18n.status(run.status)}`), el('small', '', t('app.runMeta', { date: formatDate(run.startedAt), target: run.target
|
|
208
|
+
const row = el('div', 'list-row'); const copy = el('div'); copy.append(el('strong', '', `${run.suiteId} · ${i18n.status(run.status)}`), el('small', '', t('app.runMeta', { date: formatDate(run.startedAt), target: targetDisplay(run.target), passed: run.passedCount ?? 0, total: run.caseCount ?? 0, id: run.runId })));
|
|
196
209
|
const button = el('button', 'button', t('app.viewReport')); button.type = 'button'; button.disabled = run.status === 'running' || run.status === 'stopping'; button.addEventListener('click', () => loadReport(run.runId)); row.append(copy, button); host.append(row);
|
|
197
210
|
}
|
|
198
211
|
}
|
|
@@ -226,16 +239,16 @@
|
|
|
226
239
|
const flow = $('flow-input').value.trim();
|
|
227
240
|
if (!flow) throw localizedError('app.enterFlow');
|
|
228
241
|
if (!app.state?.models?.planner?.configured || app.state.models.planner.kind !== 'openai-compatible' || !app.state.models.planner.keyReady) throw localizedError('app.configurePlanner');
|
|
229
|
-
const request = ++app.planRequest;
|
|
242
|
+
const request = ++app.planRequest; const platform = $('scenario-platform').value;
|
|
230
243
|
app.planning = true; renderActive(); noticeKey('app.planStarted');
|
|
231
244
|
try {
|
|
232
|
-
const result = await api('/api/plan', { flow });
|
|
245
|
+
const result = await api('/api/plan', { flow, platform });
|
|
233
246
|
if (request !== app.planRequest) { noticeKey('app.planCancelledLate'); return; }
|
|
234
247
|
setDraft(result.suite, result.script, 'generated', flow); noticeKey('app.planComplete');
|
|
235
248
|
} catch (error) { if (request !== app.planRequest) noticeKey('app.planCancelledDraft'); else throw error; }
|
|
236
249
|
finally { app.planning = false; await refresh(false).catch(() => {}); }
|
|
237
250
|
}); }
|
|
238
|
-
async function prompt() { await work(async () => { const flow = $('flow-input').value.trim(); if (!flow) throw localizedError('app.enterFlow'); const result = await api('/api/prompt', { flow }); download('flow-qa-prompt.md', result.prompt, 'text/markdown'); noticeKey('app.promptDownloaded'); }); }
|
|
251
|
+
async function prompt() { await work(async () => { const flow = $('flow-input').value.trim(); if (!flow) throw localizedError('app.enterFlow'); const result = await api('/api/prompt', { flow, platform: $('scenario-platform').value }); download('flow-qa-prompt.md', result.prompt, 'text/markdown'); noticeKey('app.promptDownloaded'); }); }
|
|
239
252
|
async function compile() { await work(async () => {
|
|
240
253
|
if (app.stale) throw localizedError('app.compileStale');
|
|
241
254
|
let suite; try { suite = JSON.parse($('suite-editor').value); } catch { throw localizedError('app.invalidJson'); }
|
|
@@ -249,7 +262,7 @@
|
|
|
249
262
|
noticeKey('app.compileComplete');
|
|
250
263
|
}); }
|
|
251
264
|
async function save() { await work(async () => { if (!app.draft || app.dirty || app.stale) throw localizedError('app.needValidated'); const body = { suite: app.draft }; if (!app.boundSource && app.sourceFlow && app.sourceFlow === $('flow-input').value.trim()) body.flow = app.sourceFlow; const result = await api('/api/save', body); const saved = await api(`/api/suite?file=${encodeURIComponent(result.file)}`); $('flow-input').value = saved.flow || ''; setDraft(saved.suite, saved.script, 'library', saved.flow || '', result.file, saved.suiteHash || ''); noticeKey('app.savedSuite', { file: result.file }); await refresh(false); }); }
|
|
252
|
-
async function run() { await work(async () => { if (!app.draft || !app.script || !app.savedFile || !app.savedHash || app.dirty || app.stale || !$('review-checkbox').checked ||
|
|
265
|
+
async function run() { await work(async () => { if (!app.draft || !app.script || !app.savedFile || !app.savedHash || app.dirty || app.stale || !$('review-checkbox').checked || !compatibleTarget()) throw localizedError('app.runNotReady'); const target = selectedTarget(); const result = await api('/api/run', { suite: app.savedFile, suiteHash: app.savedHash, targetName: target.name, targetFingerprint: target.targetFingerprint }); noticeKey('app.runStarted', { id: result.operationId }); await refresh(false); }); }
|
|
253
266
|
async function stop() {
|
|
254
267
|
if (app.stopping || !app.state?.active) return;
|
|
255
268
|
if (app.state.active.kind === 'plan' && app.planning) app.planRequest += 1;
|
|
@@ -271,9 +284,9 @@
|
|
|
271
284
|
}
|
|
272
285
|
function renderReport() {
|
|
273
286
|
const report = app.report; if (!report) return;
|
|
274
|
-
$('report-heading').textContent = `${report.runId} · ${i18n.status(report.status)} · ${report.target
|
|
287
|
+
$('report-heading').textContent = `${report.runId} · ${i18n.status(report.status)} · ${targetDisplay(report.target)}`;
|
|
275
288
|
const host = $('report-content'); replace(host);
|
|
276
|
-
const meta = el('div', 'report-meta'); for (const value of [t('app.reportStatus', { status: i18n.status(report.status) }), t('app.reportTarget', { target: report.target
|
|
289
|
+
const meta = el('div', 'report-meta'); for (const value of [t('app.reportStatus', { status: i18n.status(report.status) }), t('app.reportTarget', { target: targetDisplay(report.target) }), t('app.reportStarted', { date: formatDate(report.startedAt) }), t('app.reportFinished', { date: formatDate(report.finishedAt) })]) meta.append(el('span', '', value)); host.append(meta);
|
|
277
290
|
if (report.usage) host.append(el('p', 'hint', t('app.usage', { tokens: report.usage.modelTokens ?? t('app.unavailable'), cost: typeof report.usage.estimatedCostUsd === 'number' ? `$${report.usage.estimatedCostUsd.toFixed(4)}` : t('app.unavailable') })));
|
|
278
291
|
if (report.errors?.length) host.append(el('p', 'report-errors', report.errors.join('\n')));
|
|
279
292
|
for (const item of report.cases || []) {
|
|
@@ -302,7 +315,8 @@
|
|
|
302
315
|
}
|
|
303
316
|
async function saveModel(event) { event.preventDefault(); const form = event.currentTarget; await work(async () => { const values = new FormData(form); const body = { role: form.dataset.role, baseURL: String(values.get('baseURL') || '').trim(), model: String(values.get('model') || '').trim() }; const apiKeyEnv = String(values.get('apiKeyEnv') || '').trim(); if (apiKeyEnv) body.apiKeyEnv = apiKeyEnv; await api('/api/models', body); await refresh(true); noticeKey('app.modelSaved'); }); }
|
|
304
317
|
async function clearModel(role) { await work(async () => { await api('/api/models', { role, clear: true }); await refresh(true); noticeKey('app.modelCleared'); }); }
|
|
305
|
-
|
|
318
|
+
function targetFields() { const mobile = $('target-kind').value !== 'web'; for (const [id, required] of [['target-url-field', !mobile], ['target-bundle-field', mobile], ['target-device-field', mobile]]) { const field = $(id); field.hidden = !required; field.querySelector('input').required = required; } }
|
|
319
|
+
async function saveTarget(event) { event.preventDefault(); const form = event.currentTarget; await work(async () => { const values = new FormData(form); const kind = String(values.get('kind') || 'web'); const body = { name: String(values.get('name') || '').trim(), description: String(values.get('description') || '').trim() }; if (kind === 'web') body.url = String(values.get('url') || '').trim(); else { body.kind = 'mobile'; body.platform = kind; body.bundleId = String(values.get('bundleId') || '').trim(); body.device = String(values.get('device') || '').trim(); } await api('/api/targets', body); form.reset(); targetFields(); await refresh(true); noticeKey('app.targetSaved'); }); }
|
|
306
320
|
function renderLanguage() {
|
|
307
321
|
const editedModels = [...document.querySelectorAll('.model-form')].map((form) => ({ role: form.dataset.role, values: Object.fromEntries([...form.querySelectorAll('input')].map((input) => [input.name, input.value])) }));
|
|
308
322
|
const focusedModel = document.activeElement?.closest?.('.model-form');
|
|
@@ -310,6 +324,8 @@
|
|
|
310
324
|
$('breadcrumb').textContent = t('app.breadcrumb', { name: t(names[app.view]) });
|
|
311
325
|
const emptyTarget = $('target-select').querySelector('option[value=""]');
|
|
312
326
|
if (emptyTarget) emptyTarget.textContent = t('scenario.selectTarget');
|
|
327
|
+
for (const option of $('target-select').options) { const target = app.displayTargets.find((item) => item.name === option.value); if (target) option.textContent = targetDisplay(target); }
|
|
328
|
+
const targetList = $('target-list'); for (const [row, target] of [...targetList.children].map((node, index) => [node, app.displayTargets[index]])) { if (target) row.replaceChildren(el('strong', '', target.name), document.createTextNode(` · ${targetLabel(target)}${target.description ? ` · ${target.description}` : ''}`)); }
|
|
313
329
|
modelForms();
|
|
314
330
|
for (const edited of editedModels) {
|
|
315
331
|
const form = [...document.querySelectorAll('.model-form')].find((item) => item.dataset.role === edited.role);
|
|
@@ -340,6 +356,7 @@
|
|
|
340
356
|
$('suite-editor').focus();
|
|
341
357
|
});
|
|
342
358
|
$('flow-input').addEventListener('input', () => { if ((app.sourceFlow || app.boundSource) && $('flow-input').value.trim() !== app.sourceFlow.trim()) { app.stale = true; app.savedHash = ''; $('review-checkbox').checked = false; readiness(); } });
|
|
359
|
+
$('scenario-platform').addEventListener('change', () => { app.planRequest += 1; if (app.draft) { app.stale = true; app.savedHash = ''; } $('review-checkbox').checked = false; readiness(); });
|
|
343
360
|
$('suite-editor').addEventListener('input', () => { app.dirty = true; app.script = ''; app.savedFile = ''; app.savedHash = ''; if (app.origin === 'none') app.origin = 'manual'; $('review-checkbox').checked = false; summarize(null); readiness(); });
|
|
344
361
|
$('review-checkbox').addEventListener('change', readiness); $('target-select').addEventListener('change', () => { $('review-checkbox').checked = false; targetDescription(); });
|
|
345
362
|
$('plan-btn').addEventListener('click', generate); $('prompt-btn').addEventListener('click', prompt); $('compile-btn').addEventListener('click', compile); $('save-btn').addEventListener('click', save); $('run-btn').addEventListener('click', run); $('stop-btn').addEventListener('click', stop);
|
|
@@ -349,6 +366,7 @@
|
|
|
349
366
|
$('report-md').addEventListener('click', downloadReportMarkdown);
|
|
350
367
|
$('report-json').addEventListener('click', () => { if (app.report) download(`${app.selectedRun}.json`, JSON.stringify(app.report, null, 2), 'application/json'); });
|
|
351
368
|
$('target-form').addEventListener('submit', saveTarget);
|
|
369
|
+
$('target-kind').addEventListener('change', targetFields); targetFields();
|
|
352
370
|
$('language-select').addEventListener('change', (event) => i18n.setLocale(event.currentTarget.value));
|
|
353
371
|
i18n.onChange(renderLanguage);
|
|
354
372
|
let refreshing = false;
|
|
@@ -23,6 +23,10 @@
|
|
|
23
23
|
'scenario.eyebrow': ['01 / SCENARIO', '01 / 시나리오'],
|
|
24
24
|
'scenario.headingSecond': ['keep the evidence.', '검증을 남깁니다.'],
|
|
25
25
|
'scenario.intro': ['Define the expected behavior, review the cases, then run them against a selected target.', '기대 동작을 정의하고 케이스를 검토한 뒤, 선택한 대상에서 실행하세요.'],
|
|
26
|
+
'scenario.platform': ['Case platform', '케이스 플랫폼'],
|
|
27
|
+
'scenario.web': ['Web', '웹'],
|
|
28
|
+
'scenario.mobile': ['Mobile app', '모바일 앱'],
|
|
29
|
+
'scenario.mobileSetup': ['For iOS, prepare Xcode, a simulator or signed device, and app signing. For Android, prepare the Android SDK and an emulator or authorized device. Configuration alone does not verify device access.', 'iOS는 Xcode, 시뮬레이터 또는 서명된 기기와 앱 서명을 준비하세요. Android는 Android SDK와 에뮬레이터 또는 승인된 기기를 준비하세요. 설정만으로 기기 접근이 검증되지는 않습니다.'],
|
|
26
30
|
'scenario.flowTitle': ['Flow to test', '검증할 흐름'],
|
|
27
31
|
'scenario.flowHelp': ['Describe the action and the result you expect to see.', '어떤 행동에 어떤 결과가 보여야 하는지 적어주세요.'],
|
|
28
32
|
'scenario.flowLabel': ['Scenario and expected behavior', '시나리오와 기대 동작'],
|
|
@@ -98,7 +102,14 @@
|
|
|
98
102
|
'settings.savedHere': ['Your display choices are saved in this browser.', '선택한 화면 설정은 이 브라우저에 저장됩니다.'],
|
|
99
103
|
'settings.reset': ['Reset to defaults', '기본값으로 재설정'],
|
|
100
104
|
'settings.targets': ['Run targets', '실행 대상'],
|
|
101
|
-
'settings.targetsHelp': ['Add
|
|
105
|
+
'settings.targetsHelp': ['Add a web address or a native app and device.', '웹 주소 또는 네이티브 앱과 기기를 등록하세요.'],
|
|
106
|
+
'settings.targetType': ['Target type', '대상 유형'],
|
|
107
|
+
'settings.ios': ['iOS app', 'iOS 앱'],
|
|
108
|
+
'settings.android': ['Android app', 'Android 앱'],
|
|
109
|
+
'settings.targetUrl': ['URL', 'URL'],
|
|
110
|
+
'settings.bundleId': ['App bundle ID / package name', '앱 번들 ID / 패키지 이름'],
|
|
111
|
+
'settings.device': ['Device name or ID', '기기 이름 또는 ID'],
|
|
112
|
+
'settings.targetSafety': ['Saving a target does not connect a device or install an app.', '대상을 저장해도 기기가 연결되거나 앱이 설치되지는 않습니다.'],
|
|
102
113
|
'settings.targetName': ['Target name', '대상 이름'],
|
|
103
114
|
'settings.targetNamePlaceholder': ['e.g. local-demo', '예: local-demo'],
|
|
104
115
|
'settings.description': ['Description', '설명'],
|
|
@@ -126,6 +137,7 @@
|
|
|
126
137
|
'app.needValidated': ['Validated cases required.', '검증된 케이스가 필요합니다.'],
|
|
127
138
|
'app.needSaved': ['Save cases or reload them from the library before running.', '실행 전 케이스를 저장하거나 보관함에서 다시 불러오세요.'],
|
|
128
139
|
'app.needTarget': ['Select a run target.', '실행 대상을 선택하세요.'],
|
|
140
|
+
'app.platformMismatch': ['The case platform and target type must match.', '케이스 플랫폼과 대상 유형이 일치해야 합니다.'],
|
|
129
141
|
'app.needReview': ['Review the contents, then select the checkbox.', '내용 검토 후 확인란을 선택하세요.'],
|
|
130
142
|
'app.ready': ['Ready to run.', '실행할 준비가 됐습니다.'],
|
|
131
143
|
'app.caseReadiness': ['{total} total · {ready} runnable · {blocked} blocked before run{hint}', '전체 {total}개 · 실행 가능 {ready}개 · 사전 차단 {blocked}개{hint}'],
|
|
@@ -144,6 +156,9 @@
|
|
|
144
156
|
'app.locatorTestId': ['test ID', '테스트 ID'],
|
|
145
157
|
'app.locatorName': ['name “{name}”', '이름 “{name}”'],
|
|
146
158
|
'app.stepOpen': ['Open page: {path}', '페이지 열기: {path}'],
|
|
159
|
+
'app.stepLaunchApp': ['Launch app', '앱 실행'],
|
|
160
|
+
'app.stepBack': ['Go back', '뒤로 가기'],
|
|
161
|
+
'app.stepSwipe': ['Swipe {direction}', '{direction} 방향으로 스와이프'],
|
|
147
162
|
'app.stepClick': ['Click: {target}', '클릭: {target}'],
|
|
148
163
|
'app.stepFill': ['Fill: {target} → “{value}”', '입력: {target} → “{value}”'],
|
|
149
164
|
'app.stepVisible': ['Expected: {target} visible', '기대 결과: {target} 표시됨'],
|
|
@@ -196,8 +211,8 @@
|
|
|
196
211
|
'app.connected': ['Local server connected', '로컬 서버 연결됨'],
|
|
197
212
|
'app.refreshSettings': ['Refresh settings', '설정 새로고침'],
|
|
198
213
|
'app.runtimeDisconnected': ['Connect to the server to check the runtime.', '서버 연결 후 실행 환경을 확인할 수 있습니다.'],
|
|
199
|
-
'app.runtimeInstalled': ['
|
|
200
|
-
'app.runtimeMissing': ['
|
|
214
|
+
'app.runtimeInstalled': ['Web browser installed · Fixed-action web cases can run without a model. Mobile device access is checked during a run.', '웹 브라우저 설치 확인됨 · 고정 조작 웹 케이스는 모델 없이 실행할 수 있습니다. 모바일 기기 접근은 실행 중 확인합니다.'],
|
|
215
|
+
'app.runtimeMissing': ['Web browser installation required for web runs. Run {command} in a terminal. Mobile runs check device access at runtime.', '웹 실행에는 브라우저 설치가 필요합니다. 터미널에서 {command}을 실행하세요. 모바일 실행은 실행 중 기기 접근을 확인합니다.'],
|
|
201
216
|
'app.plannerUnconfigured': ['AI planner is not connected. Configure a model in Connections or download the prompt.', 'AI 플래너가 연결되지 않았습니다. 연결 설정에서 모델을 지정하거나 프롬프트를 내려받으세요.'],
|
|
202
217
|
'app.plannerKeyMissing': ['The configured key environment variable is missing on the server. Add it and restart the server.', '설정한 키 환경 변수가 서버에 없습니다. 키를 주입한 뒤 서버를 재시작하세요.'],
|
|
203
218
|
'app.plannerCliOnly': ['This model can be used from the CLI. Connect an OpenAI-compatible model to generate cases in the UI.', '이 모델은 CLI에서 사용할 수 있습니다. UI에서 생성하려면 OpenAI 호환 모델을 연결하세요.'],
|
|
@@ -48,6 +48,8 @@
|
|
|
48
48
|
<section class="work-section flow-section" aria-labelledby="flow-title">
|
|
49
49
|
<div class="section-head"><span class="section-number">01</span><div><h2 id="flow-title" data-i18n="scenario.flowTitle">Flow to test</h2><p data-i18n="scenario.flowHelp">Describe the action and the result you expect to see.</p></div></div>
|
|
50
50
|
<div class="section-body">
|
|
51
|
+
<label for="scenario-platform" class="field-label" data-i18n="scenario.platform">Case platform</label>
|
|
52
|
+
<select id="scenario-platform"><option value="web" data-i18n="scenario.web">Web</option><option value="mobile" data-i18n="scenario.mobile">Mobile app</option></select>
|
|
51
53
|
<label for="flow-input" class="field-label" data-i18n="scenario.flowLabel">Scenario and expected behavior</label>
|
|
52
54
|
<textarea id="flow-input" rows="11" placeholder="For example: When a user enters a title and selects Add, the same title appears in the list. An empty title shows an error." data-i18n-placeholder="scenario.flowPlaceholder"></textarea>
|
|
53
55
|
<p class="hint" data-i18n="scenario.flowHint">Generate cases when an AI planner is ready. You can also download the prompt and write them yourself without a model connection.</p>
|
|
@@ -76,7 +78,7 @@
|
|
|
76
78
|
<section class="execution-section" aria-labelledby="run-title">
|
|
77
79
|
<div class="section-head"><span class="section-number">03</span><div><h2 id="run-title" data-i18n="scenario.runTitle">Review, then run</h2><p data-i18n="scenario.runHelp">Select a target and confirm you have reviewed the cases.</p></div></div>
|
|
78
80
|
<div class="execution-grid">
|
|
79
|
-
<div class="target-field"><label for="target-select" class="field-label" data-i18n="scenario.target">Run target</label><select id="target-select"><option value="" data-i18n="scenario.selectTarget">Select a target</option></select><p id="target-description" class="hint" data-i18n="scenario.targetHint">You can add a target in Connections.</p></div>
|
|
81
|
+
<div class="target-field"><label for="target-select" class="field-label" data-i18n="scenario.target">Run target</label><select id="target-select"><option value="" data-i18n="scenario.selectTarget">Select a target</option></select><p id="target-description" class="hint" data-i18n="scenario.targetHint">You can add a target in Connections.</p><p id="mobile-setup-hint" class="hint" hidden data-i18n="scenario.mobileSetup">For iOS, prepare Xcode, a simulator or signed device, and app signing. For Android, prepare the Android SDK and an emulator or authorized device. Configuration alone does not verify device access.</p></div>
|
|
80
82
|
<div class="execution-confirm"><p id="case-readiness" class="hint"></p><label class="review-line"><input type="checkbox" id="review-checkbox"><span data-i18n="scenario.reviewConfirm">I have reviewed the requirements, case steps, and expected results.</span></label><div class="run-bar"><span id="run-readiness" data-i18n="scenario.runInitial">Validated cases and a run target are required.</span><div class="actions"><button type="button" id="stop-btn" class="button quiet" hidden data-i18n="scenario.stop">Stop run</button><button type="button" id="run-btn" class="button primary" disabled><span data-i18n="scenario.run">Run QA</span><span aria-hidden="true">→</span></button></div></div></div>
|
|
81
83
|
</div>
|
|
82
84
|
</section>
|
|
@@ -106,7 +108,7 @@
|
|
|
106
108
|
<div class="hds-footer"><p id="preferences-status" aria-live="polite">Your display choices are saved in this browser.</p><button id="reset-preferences" type="button" data-i18n="settings.reset">Reset to defaults</button></div>
|
|
107
109
|
</section>
|
|
108
110
|
<div class="settings-grid">
|
|
109
|
-
<section class="settings-section"><div class="section-head"><span class="section-number">01</span><div><h2 data-i18n="settings.targets">Run targets</h2><p data-i18n="settings.targetsHelp">Add
|
|
111
|
+
<section class="settings-section"><div class="section-head"><span class="section-number">01</span><div><h2 data-i18n="settings.targets">Run targets</h2><p data-i18n="settings.targetsHelp">Add a web address or a native app and device.</p></div></div><form class="section-body form-stack" id="target-form"><label><span data-i18n="settings.targetName">Target name</span><input name="name" required placeholder="e.g. local-demo" data-i18n-placeholder="settings.targetNamePlaceholder" pattern="[A-Za-z0-9][A-Za-z0-9_\-]*"></label><label><span data-i18n="settings.targetType">Target type</span><select name="kind" id="target-kind"><option value="web" data-i18n="scenario.web">Web</option><option value="ios" data-i18n="settings.ios">iOS app</option><option value="android" data-i18n="settings.android">Android app</option></select></label><label id="target-url-field"><span data-i18n="settings.targetUrl">URL</span><input name="url" type="url" required placeholder="https://example.com"></label><label id="target-bundle-field" hidden><span data-i18n="settings.bundleId">App bundle ID / package name</span><input name="bundleId" autocomplete="off"></label><label id="target-device-field" hidden><span data-i18n="settings.device">Device name or ID</span><input name="device" autocomplete="off"></label><label><span data-i18n="settings.description">Description</span><textarea name="description" rows="3" placeholder="What this target is for" data-i18n-placeholder="settings.descriptionPlaceholder"></textarea></label><button class="button primary" type="submit" data-i18n="settings.saveTarget">Save target</button><p class="hint" data-i18n="settings.targetSafety">Saving a target does not connect a device or install an app.</p><div id="target-list" class="settings-list"></div></form></section>
|
|
110
112
|
<section class="settings-section"><div class="section-head"><span class="section-number">02</span><div><h2 data-i18n="settings.models">Model connections</h2><p data-i18n="settings.modelsHelp">Manage planner and run agent connections separately.</p></div></div><div class="section-body"><p class="hint" data-i18n="settings.modelsHint">Do not enter or store API key values here. Specify only the name of an environment variable in the server process.</p><div id="model-forms"></div></div></section>
|
|
111
113
|
</div>
|
|
112
114
|
</section>
|
|
@@ -148,7 +148,7 @@ button:focus-visible, input:focus-visible, select:focus-visible, textarea:focus-
|
|
|
148
148
|
.preferences-section .hds-group { padding-block: calc(var(--space-factor) * 22px); }
|
|
149
149
|
.settings-grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); gap: clamp(25px, 3vw, 44px); }
|
|
150
150
|
.form-stack, .model-form { display: grid; gap: calc(var(--space-factor) * 15px); }
|
|
151
|
-
.form-stack label input, .form-stack label textarea, .model-form label input { margin-top: 8px; }
|
|
151
|
+
.form-stack label input, .form-stack label select, .form-stack label textarea, .model-form label input { margin-top: 8px; }
|
|
152
152
|
.form-stack > button { justify-self: start; }
|
|
153
153
|
.settings-list { margin-top: 11px; border-top: 1px solid var(--line); }
|
|
154
154
|
.settings-list:empty { display: none; }
|
package/dist/src/ui/server.js
CHANGED
|
@@ -14,6 +14,7 @@ import { EvidenceError, readEvidence } from "./evidence.js";
|
|
|
14
14
|
import { buildPlanningPrompt, planFlow, PlannerError } from "../planner.js";
|
|
15
15
|
import { renderMarkdown } from "../report.js";
|
|
16
16
|
import { configSchema, idSchema, modelSchema, suiteSchema, targetSchema } from "../schema.js";
|
|
17
|
+
import { targetFingerprint } from "../targets.js";
|
|
17
18
|
import { CONFIG_FILE, sha256 } from "../workspace.js";
|
|
18
19
|
const MAX_BODY = 256 * 1024;
|
|
19
20
|
const MAX_FLOW = 128 * 1024;
|
|
@@ -28,7 +29,10 @@ const RUN_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
|
|
|
28
29
|
const FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,159}$/;
|
|
29
30
|
const reportEnvelope = z.object({
|
|
30
31
|
schemaVersion: z.literal('flow-qa/report-1'), runId: z.string().regex(RUN_ID), suiteId: idSchema,
|
|
31
|
-
suiteHash: z.string(), target: z.
|
|
32
|
+
suiteHash: z.string(), target: z.union([
|
|
33
|
+
z.object({ name: z.string(), url: z.string() }),
|
|
34
|
+
z.object({ name: z.string(), kind: z.literal('mobile'), platform: z.enum(['ios', 'android']), bundleId: z.string(), device: z.string() }),
|
|
35
|
+
]),
|
|
32
36
|
startedAt: z.string(), finishedAt: z.string(), status: z.enum(['passed', 'failed', 'blocked', 'error', 'interrupted']),
|
|
33
37
|
cases: z.array(z.object({ id: z.string(), title: z.string(), status: z.enum(['passed', 'failed', 'flaky', 'blocked', 'skipped', 'error']),
|
|
34
38
|
expected: z.array(z.string()), attempts: z.array(z.unknown()), reason: z.string().optional() }).passthrough()),
|
|
@@ -66,7 +70,8 @@ const SCHEMA_FIELDS = new Set([
|
|
|
66
70
|
'suite', 'schemaVersion', 'id', 'title', 'source', 'file', 'sha256', 'requirements', 'cases',
|
|
67
71
|
'requirementIds', 'preconditions', 'questions', 'steps', 'type', 'path', 'instruction',
|
|
68
72
|
'locator', 'by', 'value', 'name', 'visible', 'expected', 'match', 'flow', 'role', 'url',
|
|
69
|
-
'description', 'baseURL', 'model', 'apiKeyEnv', 'clear', 'targetName', 'targetURL', 'suiteHash',
|
|
73
|
+
'description', 'baseURL', 'model', 'apiKeyEnv', 'clear', 'targetName', 'targetURL', 'targetFingerprint', 'suiteHash',
|
|
74
|
+
'kind', 'platform', 'bundleId', 'device', 'direction',
|
|
70
75
|
]);
|
|
71
76
|
const KNOWN_SCHEMA_MESSAGES = new Map([
|
|
72
77
|
['IDs must be unique.', 'IDs must be unique.'],
|
|
@@ -108,7 +113,8 @@ function plannerFailure(error) {
|
|
|
108
113
|
}
|
|
109
114
|
if (error.code === 'invalid-suite') {
|
|
110
115
|
const allowedMessages = new Set(['A JSON object response is required.', 'A runnable case needs an expected-result assertion.',
|
|
111
|
-
'Check requirement links, duplicate IDs, and assertions.', 'Check required values, types, and limits in the case schema.'
|
|
116
|
+
'Check requirement links, duplicate IDs, and assertions.', 'Check required values, types, and limits in the case schema.',
|
|
117
|
+
'The suite platform must match the selected planning platform.']);
|
|
112
118
|
const allowedPaths = new Set([...SCHEMA_FIELDS, 'response', 'suite', '?']);
|
|
113
119
|
const issues = error.issues?.slice(0, 8).map((issue) => ({
|
|
114
120
|
path: issue.path.split('.').slice(0, 8).every((part) => allowedPaths.has(part) || /^(?:0|[1-9][0-9]?)$/.test(part)) ? issue.path : 'suite',
|
|
@@ -439,7 +445,7 @@ export async function startUIServer(options) {
|
|
|
439
445
|
const settings = await config();
|
|
440
446
|
const history = await runs();
|
|
441
447
|
sendJSON(response, 200, { csrfToken: token,
|
|
442
|
-
targets: Object.entries(settings.targets).map(([name, target]) => ({ name, ...target })),
|
|
448
|
+
targets: Object.entries(settings.targets).map(([name, target]) => ({ name, ...target, targetFingerprint: targetFingerprint(target) })),
|
|
443
449
|
models: { planner: modelState(settings.planner), agent: modelState(settings.agent) },
|
|
444
450
|
runtime: { browserInstalled: await browserInstalled(), browserInstallCommand: BROWSER_INSTALL_COMMAND },
|
|
445
451
|
suites: await suites(), runs: history,
|
|
@@ -522,10 +528,10 @@ export async function startUIServer(options) {
|
|
|
522
528
|
throw new HTTPError(403, 'Invalid request token.');
|
|
523
529
|
const body = await bodyJSON(request);
|
|
524
530
|
if (url.pathname === '/api/prompt') {
|
|
525
|
-
const { flow } = parse(z.object({ flow: z.string().min(1) }).strict(), body);
|
|
531
|
+
const { flow, platform } = parse(z.object({ flow: z.string().min(1), platform: z.enum(['web', 'mobile']).optional() }).strict(), body);
|
|
526
532
|
if (Buffer.byteLength(flow) > MAX_FLOW)
|
|
527
533
|
throw new HTTPError(413, 'The flow is too large.');
|
|
528
|
-
sendJSON(response, 200, { prompt: buildPlanningPrompt(flow) });
|
|
534
|
+
sendJSON(response, 200, { prompt: buildPlanningPrompt(flow, platform) });
|
|
529
535
|
return;
|
|
530
536
|
}
|
|
531
537
|
if (url.pathname === '/api/stop') {
|
|
@@ -539,7 +545,7 @@ export async function startUIServer(options) {
|
|
|
539
545
|
}
|
|
540
546
|
if (url.pathname === '/api/plan') {
|
|
541
547
|
const result = await mutation(async () => {
|
|
542
|
-
const { flow } = parse(z.object({ flow: z.string().min(1) }).strict(), body);
|
|
548
|
+
const { flow, platform } = parse(z.object({ flow: z.string().min(1), platform: z.enum(['web', 'mobile']).optional() }).strict(), body);
|
|
543
549
|
if (Buffer.byteLength(flow) > MAX_FLOW)
|
|
544
550
|
throw new HTTPError(413, 'The flow is too large.');
|
|
545
551
|
const settings = await config();
|
|
@@ -560,7 +566,7 @@ export async function startUIServer(options) {
|
|
|
560
566
|
timeout.unref();
|
|
561
567
|
try {
|
|
562
568
|
const result = await Promise.race([
|
|
563
|
-
Promise.resolve().then(() => plan(flow, settings.planner, { signal: abort.signal })),
|
|
569
|
+
Promise.resolve().then(() => plan(flow, settings.planner, { signal: abort.signal, platform })),
|
|
564
570
|
aborted,
|
|
565
571
|
]);
|
|
566
572
|
if (abort.signal.aborted)
|
|
@@ -569,6 +575,8 @@ export async function startUIServer(options) {
|
|
|
569
575
|
if (!parsed.success)
|
|
570
576
|
throw new HTTPError(502, 'The model returned an invalid suite.', { category: 'invalid-suite' });
|
|
571
577
|
const planned = parsed.data;
|
|
578
|
+
if ((planned.platform ?? 'web') !== (platform ?? 'web'))
|
|
579
|
+
throw new HTTPError(502, 'The model returned cases for a different platform.', { category: 'invalid-suite' });
|
|
572
580
|
const { source: _modelSource, ...withoutSource } = planned;
|
|
573
581
|
const suite = suiteSchema.parse(withoutSource);
|
|
574
582
|
return { suite, script: compileSuite(suite) };
|
|
@@ -629,8 +637,11 @@ export async function startUIServer(options) {
|
|
|
629
637
|
}
|
|
630
638
|
if (url.pathname === '/api/targets') {
|
|
631
639
|
await mutation(async () => {
|
|
632
|
-
const { name,
|
|
633
|
-
|
|
640
|
+
const { name, ...input } = parse(z.union([
|
|
641
|
+
z.object({ name: idSchema, kind: z.literal('mobile'), platform: z.enum(['ios', 'android']), bundleId: z.string(), device: z.string(), description: z.string().optional() }).strict(),
|
|
642
|
+
z.object({ name: idSchema, kind: z.literal('web').optional(), url: z.string(), description: z.string().optional() }).strict(),
|
|
643
|
+
]), body);
|
|
644
|
+
const target = parse(targetSchema, input);
|
|
634
645
|
const settings = await config();
|
|
635
646
|
settings.targets[name] = target;
|
|
636
647
|
await updateConfig(settings);
|
|
@@ -658,18 +669,27 @@ export async function startUIServer(options) {
|
|
|
658
669
|
if (url.pathname === '/api/run') {
|
|
659
670
|
enterMutation();
|
|
660
671
|
try {
|
|
661
|
-
const { suite: file, suiteHash, targetName, targetURL } = parse(z.object({
|
|
662
|
-
suite: z.string(), suiteHash: z.string().regex(/^[a-f0-9]{64}$/), targetName: idSchema,
|
|
672
|
+
const { suite: file, suiteHash, targetName, targetURL, targetFingerprint: reviewedFingerprint } = parse(z.object({
|
|
673
|
+
suite: z.string(), suiteHash: z.string().regex(/^[a-f0-9]{64}$/), targetName: idSchema,
|
|
674
|
+
targetURL: z.string().optional(), targetFingerprint: z.string().regex(/^[a-f0-9]{64}$/).optional(),
|
|
663
675
|
}).strict(), body);
|
|
664
676
|
const selected = await getSuite(file, true);
|
|
665
677
|
if (selected.suiteHash !== suiteHash)
|
|
666
678
|
throw new HTTPError(409, 'The reviewed case file changed. Open it again to review.');
|
|
667
679
|
const { suite } = selected;
|
|
668
680
|
const settings = await config();
|
|
669
|
-
|
|
681
|
+
const target = settings.targets[targetName];
|
|
682
|
+
if (!target)
|
|
670
683
|
invalid('The selected target was not found.');
|
|
671
|
-
if (
|
|
672
|
-
throw new HTTPError(409, 'The selected target
|
|
684
|
+
if ((suite.platform ?? 'web') !== (target.kind === 'mobile' ? 'mobile' : 'web')) {
|
|
685
|
+
throw new HTTPError(409, 'The case platform and selected target do not match. Review them again.');
|
|
686
|
+
}
|
|
687
|
+
if (reviewedFingerprint !== undefined) {
|
|
688
|
+
if (targetFingerprint(target) !== reviewedFingerprint)
|
|
689
|
+
throw new HTTPError(409, 'The selected target changed. Review it again.');
|
|
690
|
+
}
|
|
691
|
+
else if (target.kind === 'mobile' || !targetURL || target.url !== targetURL) {
|
|
692
|
+
throw new HTTPError(409, 'The selected target changed. Review it again.');
|
|
673
693
|
}
|
|
674
694
|
const operationId = randomUUID();
|
|
675
695
|
const abort = new AbortController();
|
package/docs/architecture.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Architecture and verification boundaries
|
|
2
2
|
|
|
3
|
-
The local CLI and loopback web UI share one workspace format, case schema, execution path, and report contract. Flow QA runs browser tests against a target URL
|
|
3
|
+
The local CLI and loopback web UI share one workspace format, case schema, execution path, and report contract. Flow QA runs browser tests against a target URL and native tests against a user-configured iOS or Android app/device. It does not need the target application's source code. Mobile integration is available from version 0.2.0; see the [mobile guide](mobile.md).
|
|
4
4
|
|
|
5
5
|
```mermaid
|
|
6
6
|
flowchart LR
|
|
@@ -8,7 +8,7 @@ flowchart LR
|
|
|
8
8
|
B --> C[JSON case suite]
|
|
9
9
|
C --> D[Schema and source-hash validation]
|
|
10
10
|
D --> E[Compile allowed steps to e2e code]
|
|
11
|
-
E --> F[tester-army/e2e
|
|
11
|
+
E --> F[tester-army/e2e web or mobile run]
|
|
12
12
|
F --> G[QA report with attempts and evidence]
|
|
13
13
|
H[CLI / local UI] --> B
|
|
14
14
|
H --> D
|
|
@@ -22,7 +22,9 @@ flowchart LR
|
|
|
22
22
|
|
|
23
23
|
**Only a defined vocabulary compiles.** The schema checks requirement references, assertions, step types, and inputs. It does not execute arbitrary code or shell commands from model output. Cases with unresolved `questions`, or steps that need an unavailable agent model, are blocked. `preconditions` describe required state; they are not an automatic setup contract.
|
|
24
24
|
|
|
25
|
-
**Source and evidence stay linked.** Planning records the source flow's SHA-256 hash in the suite. `validate`, `generate`, and `run` check it again and stop when the flow changed. Each execution writes a new directory with a suite snapshot, generated tests, raw engine output, and Markdown/JSON reports. Previous evidence is not reused as a current success. `compare` permits a direct comparison only for the same suite hash and target URL; it cannot establish that external data was unchanged.
|
|
25
|
+
**Source and evidence stay linked.** Planning records the source flow's SHA-256 hash in the suite. `validate`, `generate`, and `run` check it again and stop when the flow changed. Each execution writes a new directory with a suite snapshot, generated tests, raw engine output, and Markdown/JSON reports. Previous evidence is not reused as a current success. `compare` permits a direct comparison only for the same suite hash and target identity (web URL or native platform/app/device); it cannot establish that external data was unchanged.
|
|
26
|
+
|
|
27
|
+
**Targets can be configured after authoring.** A suite's optional platform distinguishes web from mobile. Omitted platform preserves legacy web suites and their hashes. Native app IDs and explicit device names belong to target settings, not generated code from the planner. UI execution checks a fingerprint of the reviewed target identity. Mobile runtime readiness is checked before device execution; configuration alone does not prove connectivity.
|
|
26
28
|
|
|
27
29
|
**Blocked and failed mean different things.** An unreachable app or unresolved question is blocked; an assertion mismatch is failed. A pass after retry has its own status. Engine output is validated rather than treating missing results as passes. The report records observations, not a root-cause verdict on the product.
|
|
28
30
|
|
|
@@ -31,8 +33,8 @@ flowchart LR
|
|
|
31
33
|
| Area | Flow QA provides | External implementation or prerequisite |
|
|
32
34
|
| --- | --- | --- |
|
|
33
35
|
| Planning | Prompt, model connection, JSON schema validation | User-selected OpenAI-compatible model |
|
|
34
|
-
| Execution | Step compilation, target connection, orchestration, cancellation | [tester-army/e2e](https://github.com/tester-army/e2e), Chromium |
|
|
36
|
+
| Execution | Step compilation, target connection, orchestration, cancellation | [tester-army/e2e](https://github.com/tester-army/e2e), Chromium for web or `@e2e-dev/mobile`/agent-device for native apps |
|
|
35
37
|
| Results | A fresh report per run, status interpretation, evidence links | Raw runs and evidence from the engine |
|
|
36
38
|
| UI | Local workflow screens and English/Korean language switch | [Hyeonse Design System](https://github.com/hyeonse-swing/hyeonse-design-system) tokens and preference utilities |
|
|
37
39
|
|
|
38
|
-
The UI binds only to `127.0.0.1` and serves approved report evidence, not arbitrary raw logs or HTML/SVG. Local reports may still contain data from the tested app and should be reviewed before sharing.
|
|
40
|
+
The UI binds only to `127.0.0.1` and serves approved report evidence, not arbitrary raw logs or HTML/SVG. Local reports may still contain data from the tested app and should be reviewed before sharing. It does not manage login sessions, cap model spending, schedule CI runs, or provision mobile SDKs, devices, and application builds.
|
package/docs/mobile.md
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# User-configured iOS and Android targets
|
|
2
|
+
|
|
3
|
+
Mobile targets require Flow QA `0.2.0` or later and Node.js 24.8 or later. The web demo remains a browser demo; it is not proof of a mobile run.
|
|
4
|
+
|
|
5
|
+
## Write cases now, connect an app later
|
|
6
|
+
|
|
7
|
+
Choose **Mobile** on the scenario screen, describe the flow, and generate or download a planning prompt. You do not need to register an app or device to author, validate, compile, or save cases. A mobile suite has `"platform": "mobile"`; existing suites without a platform remain web suites.
|
|
8
|
+
|
|
9
|
+
With the installed CLI:
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
npm install -g @ihyeon/flow-qa@0.2.0
|
|
13
|
+
flow-qa init ./my-qa
|
|
14
|
+
# Write your mobile flow in my-qa/flows/my-flow.md first.
|
|
15
|
+
flow-qa plan flows/my-flow.md --platform mobile --prompt-only --workspace ./my-qa
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Use your chosen AI tool to produce JSON from the prompt, then review and save it under `my-qa/cases/`. Alternatively, configure a planner and omit `--prompt-only`. Neither option needs the target app's source code. Product behavior and accessible element names must still be supplied; unknown expectations remain questions.
|
|
19
|
+
|
|
20
|
+
## Register an app when ready
|
|
21
|
+
|
|
22
|
+
In **Connections**, select **iOS** or **Android**, then enter a target name, the installed app's bundle/package ID, and an explicit device name or ID. Register multiple targets to use one mobile suite on both platforms when their accessible labels and expected behavior agree. Use separate suites where the two apps behave differently.
|
|
23
|
+
|
|
24
|
+
CLI examples below contain placeholders; replace them with your own app and devices:
|
|
25
|
+
|
|
26
|
+
```sh
|
|
27
|
+
flow-qa connect my-ios --platform ios --bundle-id com.example.app --device "QA iPhone" --workspace ./my-qa
|
|
28
|
+
flow-qa connect my-android --platform android --bundle-id com.example.app --device emulator-5554 --workspace ./my-qa
|
|
29
|
+
flow-qa run cases/my-flow.json --target my-ios --workspace ./my-qa
|
|
30
|
+
flow-qa run cases/my-flow.json --target my-android --workspace ./my-qa
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Saving a target only writes configuration. It does not contact a model, boot a device, install an app, or start a test. You can update a saved target by saving the same target name again. The UI checks the reviewed app, platform, and device before dispatching a run; changed settings require another review.
|
|
34
|
+
|
|
35
|
+
For an iOS simulator, prepare Xcode, an installed simulator runtime, a booted simulator, and a simulator build of the app installed on it. For Android, prepare the Android SDK and a booted emulator or an authorized connected device with the app installed. Physical iOS devices additionally need pairing, Developer Mode, and signing of the upstream automation runner. Follow the [mobile engine's setup instructions](https://e2e.tester.army/docs/mobile#physical-devices) for device-specific requirements.
|
|
36
|
+
|
|
37
|
+
Flow QA does not download SDKs, create or boot simulators, or install application builds. A missing or unavailable selected device blocks the run; a saved target is not evidence of a working device or app. For physical device IDs that the underlying engine resolves by name, the device name must be unique after case and whitespace normalization; ambiguous selection blocks the run. Chromium is only required for web targets.
|
|
38
|
+
|
|
39
|
+
## Native steps
|
|
40
|
+
|
|
41
|
+
| Step | Behavior |
|
|
42
|
+
| --- | --- |
|
|
43
|
+
| `launchApp` | Launch the app registered in the selected target |
|
|
44
|
+
| `click` | Tap an accessible element |
|
|
45
|
+
| `fill` | Enter text in an accessible input |
|
|
46
|
+
| `back` | Use the engine's app back navigation |
|
|
47
|
+
| `swipe` | Swipe the viewport `up`, `down`, `left`, or `right` |
|
|
48
|
+
| `assertVisible`, `assertText` | Check accessible elements and their text |
|
|
49
|
+
| `act`, `assertAgent` | Use the separately configured agent model for an action or judgment |
|
|
50
|
+
|
|
51
|
+
The compiler launches the configured app at the start of each runnable case unless the first step is already `launchApp`. It does not clear app data or create login state. A launch is not a fresh account/session. The web-only `open` path and `assertUrl` steps are rejected in mobile suites, and native steps are rejected in web suites.
|
|
52
|
+
|
|
53
|
+
Native `testId` locators use iOS accessibility identifiers or Android resource IDs. Accessibility exposure depends on the app framework. Fixed steps need no model; natural-language steps require a compatible tool/image-capable agent. A WebView's exposed native controls may be usable, but this path does not provide a browser DOM session or prove JavaScript bridge behavior.
|
|
54
|
+
|
|
55
|
+
## Results and proof boundaries
|
|
56
|
+
|
|
57
|
+
Mobile reports record platform, app ID, and device rather than a fabricated URL. They use the same case statuses, attempt details, and approved screenshot/text evidence handling as web reports. Comparing different apps, platforms, or configured devices does not produce a direct regression verdict.
|
|
58
|
+
|
|
59
|
+
Configuration, compilation, mocked runtime tests, and browser UI tests establish the integration behavior. Successful iOS and Android execution still requires recorded results on the user's chosen devices and apps. Flow QA does not supply app-specific cases, accounts, build artifacts, or a device farm.
|