@keepsnappy/agent 0.0.0-stage → 0.1.0-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +260 -2
- package/dist/appium-driver.js +1353 -0
- package/dist/keepsnappy.js +11979 -0
- package/examples/farm/flows/perflab/begin-run.yaml +17 -0
- package/examples/farm/flows/perflab/end-run.yaml +16 -0
- package/examples/farm/flows/perflab/next-token.js +7 -0
- package/examples/farm/flows/perflab/run-perflab.yaml +15 -0
- package/examples/farm/flows/perflab/send-markers.js +123 -0
- package/examples/farm/flows/perflab/step-markers.yaml +13 -0
- package/examples/farm/flows/perflab/tokens.js +2 -0
- package/examples/farm/workflows/perf-device-run.fragment.yml +22 -0
- package/examples/farm/workflows/perflab-measure.yml +117 -0
- package/examples/github-actions.yml +77 -0
- package/examples/keepsnappy.yml +36 -0
- package/examples/maestro-steps/e2e-community-and-live-chat.yaml +161 -0
- package/maestro/perflab/send-markers.js +123 -0
- package/maestro/perflab/step.js +38 -0
- package/maestro/perflab/step.yaml +11 -0
- package/package.json +25 -4
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Takes the next run token of JOURNEY and activates the dormant SDK with it. Activation applies from the next app
|
|
2
|
+
# launch, so the journey's own launchApp starts the measured process (plan 36 §1.2).
|
|
3
|
+
appId: com.gigaverse.appdev
|
|
4
|
+
---
|
|
5
|
+
- runScript:
|
|
6
|
+
file: next-token.js
|
|
7
|
+
env:
|
|
8
|
+
JOURNEY: ${JOURNEY}
|
|
9
|
+
- launchApp
|
|
10
|
+
- runScript: ../../elements/home.js
|
|
11
|
+
- extendedWaitUntil:
|
|
12
|
+
visible:
|
|
13
|
+
id: ${output.elements.home.screen}
|
|
14
|
+
timeout: 30000
|
|
15
|
+
# The settings come from tokens.js (`keepsnappy farm open`): the run defaults, never written by hand here (REQ-FARM-28).
|
|
16
|
+
- openLink: gigaversedev://devtools/perf-lab?activate=1&${output.perflabLinkArgs}&runToken=${output.perflabToken}
|
|
17
|
+
- runFlow: ../../subflows/confirm-open-link.yaml
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# End of one measured iteration: the SDK sends everything still unsent (nothing is sent while a run is active), then
|
|
2
|
+
# shows `perflab.flushed` (ok | error | off). `activate=0` makes the next launch dormant again.
|
|
3
|
+
appId: com.gigaverse.appdev
|
|
4
|
+
---
|
|
5
|
+
- runScript: send-markers.js # the iteration's step markers, recorded by step.js, sent now that it has ended
|
|
6
|
+
- openLink: gigaversedev://devtools/perf-lab?flush=1&activate=0
|
|
7
|
+
- runFlow: ../../subflows/confirm-open-link.yaml
|
|
8
|
+
- extendedWaitUntil:
|
|
9
|
+
visible:
|
|
10
|
+
id: perflab.flushed
|
|
11
|
+
timeout: 60000
|
|
12
|
+
optional: true # a dormant iteration (activation failed) shows nothing; farm close marks it invalid
|
|
13
|
+
- assertVisible:
|
|
14
|
+
id: perflab.flushed
|
|
15
|
+
text: ok
|
|
16
|
+
optional: true
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Hands out the run tokens of one journey in order: iteration i of JOURNEY gets output.perflabTokens[JOURNEY][i].
|
|
2
|
+
// ES5 (Maestro runScript). An empty token (more iterations than tokens) leaves the SDK dormant: no data, no harm.
|
|
3
|
+
var list = (output.perflabTokens || {})[JOURNEY] || [];
|
|
4
|
+
output.perflabIdx = output.perflabIdx || {};
|
|
5
|
+
var i = output.perflabIdx[JOURNEY] || 0;
|
|
6
|
+
output.perflabToken = list[i] || '';
|
|
7
|
+
output.perflabIdx[JOURNEY] = i + 1;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# e2e/flows/perflab/run-perflab.yaml (plan 36 D10). One BrowserStack session runs every iteration: the session is the
|
|
2
|
+
# noise block. Untagged, so smoke/regression never select it; the dispatcher names it with --execute.
|
|
3
|
+
# `times` must equal `keepsnappy farm open --repeat`. One begin/journey/end triple per journey listed in --journeys.
|
|
4
|
+
appId: com.gigaverse.appdev
|
|
5
|
+
onFlowComplete:
|
|
6
|
+
- runScript: send-markers.js # a failed iteration never reaches end-run.yaml: its step markers go from here
|
|
7
|
+
---
|
|
8
|
+
- runFlow: ../suite/login.yaml # clearState: the SDK starts dormant
|
|
9
|
+
- runScript: tokens.js # generated by `keepsnappy farm open`, overlaid into the zip (--suite-overlay)
|
|
10
|
+
- repeat:
|
|
11
|
+
times: 5
|
|
12
|
+
commands:
|
|
13
|
+
- runFlow: { file: begin-run.yaml, env: { JOURNEY: e2e-community-and-live-chat } }
|
|
14
|
+
- runFlow: ../chat/e2e-community-and-live-chat.yaml # its own launchApp starts the measured process
|
|
15
|
+
- runFlow: end-run.yaml
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Posts the step markers step.js recorded during the iteration, after it ended (end-run.yaml), so sending them never
|
|
2
|
+
// changes the test's pace. Farm mode only: POST <api>/v1/markers with the marker's own run token as Bearer (local mode
|
|
3
|
+
// posts at once from step.js). ES5 only (older Maestro builds run Rhino). Must not throw.
|
|
4
|
+
// One request per token carries all its markers ({markers: [...]}, at most 500 each), so a long journey stays far
|
|
5
|
+
// under the API's per-run rate limit; a request that fails on the network, 429 or 5xx is tried 3 times (Retry-After
|
|
6
|
+
// honoured, else 1 s then 2 s). An API from before batches answers 400 without per-item results: the markers then go
|
|
7
|
+
// one by one, as they used to.
|
|
8
|
+
// Clock round trips (plan 51 G1.4): each answer carries the API's receive/send times; that trip rides with the next
|
|
9
|
+
// request of the same token, and what is left goes to <api>/v1/clock at the end, as a short chain of CLOCK_PROBES
|
|
10
|
+
// requests whose answers are trips too, so a token still reports several round trips although its markers now go in
|
|
11
|
+
// one request. A trip whose POST failed or came back without a clock (an older API) stays pending, so nothing
|
|
12
|
+
// measured is dropped.
|
|
13
|
+
try {
|
|
14
|
+
var g = typeof globalThis !== 'undefined' ? globalThis : this;
|
|
15
|
+
var pick = function (name, dflt) {
|
|
16
|
+
var names = [name, 'MAESTRO_' + name];
|
|
17
|
+
for (var i = 0; i < names.length; i++) {
|
|
18
|
+
var v = g[names[i]];
|
|
19
|
+
if (typeof v === 'string' && (v !== '' || names[i].indexOf('KEEPSNAPPY_') >= 0)) return v;
|
|
20
|
+
}
|
|
21
|
+
return dflt;
|
|
22
|
+
};
|
|
23
|
+
var MAX_SAMPLES = 16;
|
|
24
|
+
var PER_REQUEST = 500;
|
|
25
|
+
var TRIES = 3;
|
|
26
|
+
var RETRY_CAP_MS = 30000;
|
|
27
|
+
var CLOCK_PROBES = 4;
|
|
28
|
+
var out = typeof output !== 'undefined' && output ? output : {};
|
|
29
|
+
var list = out.perflabMarkers || [];
|
|
30
|
+
out.perflabMarkers = [];
|
|
31
|
+
var api = pick('KEEPSNAPPY_API_URL', typeof out.perflabApi === 'string' ? out.perflabApi : '').replace(/\/+$/, '');
|
|
32
|
+
var wait = function (ms) {
|
|
33
|
+
try {
|
|
34
|
+
java.lang.Thread.sleep(ms);
|
|
35
|
+
return;
|
|
36
|
+
} catch (e0) {
|
|
37
|
+
// no Java bridge: wait on the clock
|
|
38
|
+
}
|
|
39
|
+
var until = Date.now() + ms;
|
|
40
|
+
while (Date.now() < until) {
|
|
41
|
+
// nothing
|
|
42
|
+
}
|
|
43
|
+
};
|
|
44
|
+
var header = function (res, name) {
|
|
45
|
+
var h = res && res.headers;
|
|
46
|
+
if (!h) return null;
|
|
47
|
+
var v = h[name] !== undefined ? h[name] : h[name.toLowerCase()];
|
|
48
|
+
return v === undefined || v === null ? null : String(v);
|
|
49
|
+
};
|
|
50
|
+
// One POST: the answer's status and parsed body, the clock trip when the answer has one; null when it threw.
|
|
51
|
+
var postOnce = function (path, token, body) {
|
|
52
|
+
var t0 = Date.now();
|
|
53
|
+
var res = http.post(api + path, {
|
|
54
|
+
headers: { 'Content-Type': 'application/json', Authorization: 'Bearer ' + token },
|
|
55
|
+
body: JSON.stringify(body),
|
|
56
|
+
});
|
|
57
|
+
var t3 = Date.now();
|
|
58
|
+
var parsed = null;
|
|
59
|
+
try {
|
|
60
|
+
parsed = res && res.body ? JSON.parse(res.body) : null;
|
|
61
|
+
} catch (e1) {
|
|
62
|
+
// not JSON
|
|
63
|
+
}
|
|
64
|
+
var c = parsed ? parsed.clock : null;
|
|
65
|
+
var trip = c && typeof c.recv_ms === 'number' && typeof c.send_ms === 'number' ? { t0: t0, t1: c.recv_ms, t2: c.send_ms, t3: t3 } : null;
|
|
66
|
+
return { status: res && typeof res.status === 'number' ? res.status : null, body: parsed, trip: trip, retryAfter: Number(header(res, 'Retry-After')) };
|
|
67
|
+
};
|
|
68
|
+
var postRetry = function (path, token, body) {
|
|
69
|
+
var a = null;
|
|
70
|
+
for (var attempt = 0; attempt < TRIES; attempt++) {
|
|
71
|
+
if (attempt > 0) wait(Math.min(a && a.retryAfter > 0 ? a.retryAfter * 1000 : 1000 * Math.pow(2, attempt - 1), RETRY_CAP_MS));
|
|
72
|
+
try {
|
|
73
|
+
a = postOnce(path, token, body);
|
|
74
|
+
} catch (e2) {
|
|
75
|
+
a = null;
|
|
76
|
+
}
|
|
77
|
+
if (a && a.status !== 429 && !(a.status >= 500)) return a;
|
|
78
|
+
}
|
|
79
|
+
return a;
|
|
80
|
+
};
|
|
81
|
+
var pending = {};
|
|
82
|
+
var tokens = [];
|
|
83
|
+
var byToken = {};
|
|
84
|
+
for (var k = 0; k < list.length && api !== ''; k++) {
|
|
85
|
+
var m = list[k];
|
|
86
|
+
if (!m || typeof m.token !== 'string' || m.token === '') continue;
|
|
87
|
+
if (!byToken[m.token]) {
|
|
88
|
+
byToken[m.token] = [];
|
|
89
|
+
pending[m.token] = [];
|
|
90
|
+
tokens.push(m.token);
|
|
91
|
+
}
|
|
92
|
+
byToken[m.token].push({ step: m.step, phase: m.phase, t_host: m.t_host });
|
|
93
|
+
}
|
|
94
|
+
var send = function (token, body) {
|
|
95
|
+
if (pending[token].length) body.clock = pending[token].slice(-MAX_SAMPLES);
|
|
96
|
+
var a = postRetry('/v1/markers', token, body);
|
|
97
|
+
if (a && a.trip) pending[token] = [a.trip];
|
|
98
|
+
return a;
|
|
99
|
+
};
|
|
100
|
+
for (var t = 0; t < tokens.length; t++) {
|
|
101
|
+
var mine = byToken[tokens[t]];
|
|
102
|
+
for (var i = 0; i < mine.length; i += PER_REQUEST) {
|
|
103
|
+
var chunk = mine.slice(i, i + PER_REQUEST);
|
|
104
|
+
var a = send(tokens[t], { markers: chunk });
|
|
105
|
+
if (!a || a.status !== 400 || (a.body && a.body.results)) continue;
|
|
106
|
+
for (var j = 0; j < chunk.length; j++) send(tokens[t], chunk[j]);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
for (var p = 0; p < tokens.length; p++) {
|
|
110
|
+
for (var n = 0; n < CLOCK_PROBES && pending[tokens[p]].length; n++) {
|
|
111
|
+
try {
|
|
112
|
+
var probe = postOnce('/v1/clock', tokens[p], { source: 'host', samples: pending[tokens[p]].slice(-MAX_SAMPLES) });
|
|
113
|
+
if (!probe.trip) break;
|
|
114
|
+
pending[tokens[p]] = [probe.trip];
|
|
115
|
+
} catch (e3) {
|
|
116
|
+
// an older API has no /v1/clock
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
} catch (e) {
|
|
122
|
+
// swallowed on purpose
|
|
123
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Declared steps in farm mode (plan 36 M2, G4). Copy apps/agent/maestro/perflab/step.yaml and step.js next to this file
|
|
2
|
+
# (e2e/flows/perflab/); send-markers.js is already here, the same file as maestro/perflab/send-markers.js. In farm mode
|
|
3
|
+
# step.js only records the marker with the iteration's token (`output.perflabToken`, taken by begin-run.yaml); send-markers.js in end-run.yaml posts them to
|
|
4
|
+
# <output.perflabApi>/v1/markers once the iteration ended, so the test never waits on the network between actions and
|
|
5
|
+
# no CLI listens on the farm host. Without send-markers.js in end-run.yaml no farm marker is ever sent.
|
|
6
|
+
# A journey that failed half-way drops out of the statistics because it never posted its set's completion marker and
|
|
7
|
+
# `keepsnappy farm close` marks it invalid. Its step markers still reach KeepSnappy: run-perflab.yaml sends what is left from
|
|
8
|
+
# onFlowComplete, for diagnosis. Inside the journey, around the part to measure:
|
|
9
|
+
appId: com.gigaverse.appdev
|
|
10
|
+
---
|
|
11
|
+
- runFlow: { file: ../perflab/step.yaml, env: { STEP: chat.open, PHASE: start } }
|
|
12
|
+
- tapOn: { id: chat-open }
|
|
13
|
+
- runFlow: { file: ../perflab/step.yaml, env: { STEP: chat.open, PHASE: end } }
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Fragment for .github/workflows/perf-device-run.yml (plan 36 D7): KeepSnappy takes the React profiler's slot, last in the
|
|
2
|
+
# chain, on the same eas:<id> the baseline and gigaload measured. Add `perflab` to the `prepare` options parsing
|
|
3
|
+
# (perflab=on|off, on by default; refuse baseline=off gigaload=off perflab=off) and its output, then this job:
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
# Last, in the slot the React profiler was built for: the same eas:<id> the baseline and gigaload measured, one
|
|
7
|
+
# BrowserStack session, no tunnel, no load fleet. It runs whatever the perf jobs concluded and never fails the night
|
|
8
|
+
# on a KeepSnappy result (exit 1 only when nothing could be judged).
|
|
9
|
+
perflab:
|
|
10
|
+
needs: [prepare, baseline-report, gigaload]
|
|
11
|
+
if: >-
|
|
12
|
+
always() && !cancelled() && needs.prepare.result == 'success'
|
|
13
|
+
&& needs.prepare.outputs.perflab == 'on'
|
|
14
|
+
&& needs.prepare.outputs.drive == 'device'
|
|
15
|
+
&& inputs.platform == 'ios'
|
|
16
|
+
uses: ./.github/workflows/perflab-measure.yml
|
|
17
|
+
secrets: inherit
|
|
18
|
+
with:
|
|
19
|
+
platform: ${{ inputs.platform }}
|
|
20
|
+
device: ${{ needs.prepare.outputs.device }}
|
|
21
|
+
app: ${{ needs.prepare.outputs.app }}
|
|
22
|
+
maestro: ${{ needs.prepare.outputs.maestro }}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# .github/workflows/perflab-measure.yml (plan 36 D8): KeepSnappy on BrowserStack, called last in perf-device-run.yml.
|
|
2
|
+
#
|
|
3
|
+
# The CLI never touches the device. `farm open` makes the run set and one KeepSnappy run per journey per iteration, and
|
|
4
|
+
# writes their run tokens into perflab-tokens.js. BrowserStack caps env values at 100 characters, so the tokens ride
|
|
5
|
+
# inside the uploaded suite: browserstack-run.mjs --suite-overlay writes that file to e2e/flows/perflab/tokens.js for
|
|
6
|
+
# this one upload. The app, activated per iteration with its token, pushes its windows to the KeepSnappy API itself.
|
|
7
|
+
# `farm close` waits until no batch has arrived for --settle-seconds, finishes every run (the one a failed session
|
|
8
|
+
# broke is marked invalid; iterations that never ran show as "no data"), closes the set and prints the verdict.
|
|
9
|
+
#
|
|
10
|
+
# Exit codes of `keepsnappy farm close`: 0 for any result (also "More work"), 1 when nothing could be judged (Blocked: no
|
|
11
|
+
# iteration sent data) or the API failed, 2 for a configuration problem, 3 with --fail-on-worse on "More work".
|
|
12
|
+
# Re-running close is safe: finished runs are left alone and the same result is printed again.
|
|
13
|
+
#
|
|
14
|
+
# Files in $OUT_DIR after the job (uploaded below): perflab-farm.json (state: run set, runs, result; no tokens),
|
|
15
|
+
# perflab-result.json, browserstack.ios.json. perflab-tokens.js holds live run tokens (valid for hours): close puts the
|
|
16
|
+
# empty stub back even when it fails, and the job deletes the file before the upload whatever happened.
|
|
17
|
+
|
|
18
|
+
name: KeepSnappy measure (reusable)
|
|
19
|
+
|
|
20
|
+
on:
|
|
21
|
+
workflow_call:
|
|
22
|
+
inputs:
|
|
23
|
+
platform: { type: string, required: true }
|
|
24
|
+
device: { type: string, default: '' }
|
|
25
|
+
app: { type: string, required: true } # eas:<id> resolved once by perf-device-run's prepare
|
|
26
|
+
maestro: { type: string, default: '2.6.1' }
|
|
27
|
+
|
|
28
|
+
permissions:
|
|
29
|
+
contents: read
|
|
30
|
+
|
|
31
|
+
jobs:
|
|
32
|
+
measure:
|
|
33
|
+
runs-on: ubuntu-latest
|
|
34
|
+
timeout-minutes: 120
|
|
35
|
+
env:
|
|
36
|
+
BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
|
|
37
|
+
BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
|
|
38
|
+
KEEPSNAPPY_KEY: ${{ secrets.KEEPSNAPPY_KEY }} # run scope; never leaves this runner
|
|
39
|
+
KEEPSNAPPY_API: https://perflab-api-864539138788.europe-west1.run.app
|
|
40
|
+
KEEPSNAPPY_CLI: '@keepsnappy/agent@0.1.0-alpha.1' # first version with `farm` (not published yet)
|
|
41
|
+
OUT_DIR: ${{ github.workspace }}/farm-run
|
|
42
|
+
RUN_ID: perflab-${{ inputs.platform }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
43
|
+
JOURNEYS: e2e-community-and-live-chat # = the JOURNEY values in run-perflab.yaml
|
|
44
|
+
REPEAT: '5' # = repeat.times in run-perflab.yaml
|
|
45
|
+
steps:
|
|
46
|
+
- uses: actions/checkout@v5
|
|
47
|
+
- uses: actions/setup-node@v4
|
|
48
|
+
with:
|
|
49
|
+
node-version: '22'
|
|
50
|
+
|
|
51
|
+
- name: Resolve the app
|
|
52
|
+
id: app
|
|
53
|
+
env:
|
|
54
|
+
EXPO_TOKEN: ${{ secrets.EXPO_TOKEN }}
|
|
55
|
+
APP_SOURCE: ${{ inputs.app }}
|
|
56
|
+
PLATFORM: ${{ inputs.platform }}
|
|
57
|
+
run: |
|
|
58
|
+
mkdir -p "$OUT_DIR"
|
|
59
|
+
node scripts/farm/resolve-app.mjs --platform "$PLATFORM" --source "$APP_SOURCE" \
|
|
60
|
+
--profile e2e --out "$OUT_DIR"
|
|
61
|
+
|
|
62
|
+
- name: Wait for a free BrowserStack slot
|
|
63
|
+
timeout-minutes: 65
|
|
64
|
+
run: node scripts/farm/preflight.mjs --sessions 1 --wait-minutes 60
|
|
65
|
+
|
|
66
|
+
# Masks every run token in this job's log (::add-mask::) and sets the outputs run_set_id, dashboard_url, state,
|
|
67
|
+
# tokens_js, runs.
|
|
68
|
+
- name: Open the KeepSnappy run set
|
|
69
|
+
id: open
|
|
70
|
+
env:
|
|
71
|
+
APP_COMMIT: ${{ steps.app.outputs.app_commit }}
|
|
72
|
+
run: |
|
|
73
|
+
[ -n "$APP_COMMIT" ] || { echo '::error::the artifact carries no commit'; exit 1; }
|
|
74
|
+
npx --yes "$KEEPSNAPPY_CLI" farm open \
|
|
75
|
+
--api "$KEEPSNAPPY_API" --branch main --commit "$APP_COMMIT" \
|
|
76
|
+
--device-model 'iPhone 15 Pro' --device-os 17 --device-provider browserstack \
|
|
77
|
+
--journeys "$JOURNEYS" --repeat "$REPEAT" \
|
|
78
|
+
--tokens-js "$OUT_DIR/perflab-tokens.js" --state "$OUT_DIR/perflab-farm.json"
|
|
79
|
+
|
|
80
|
+
- name: Run the KeepSnappy orchestrator on the device
|
|
81
|
+
id: device
|
|
82
|
+
continue-on-error: true
|
|
83
|
+
env:
|
|
84
|
+
APP: ${{ steps.app.outputs.app }}
|
|
85
|
+
APP_CUSTOM_ID: ${{ steps.app.outputs.app_custom_id }}
|
|
86
|
+
DEVICE: ${{ inputs.device || 'iPhone 15 Pro-17' }}
|
|
87
|
+
MAESTRO: ${{ inputs.maestro }}
|
|
88
|
+
PLATFORM: ${{ inputs.platform }}
|
|
89
|
+
run: |
|
|
90
|
+
args=(--platform "$PLATFORM" --device "$DEVICE" --app "$APP" --run-id "$RUN_ID"
|
|
91
|
+
--label perflab --maestro-version "$MAESTRO" --out "$OUT_DIR" --timeout-minutes 80
|
|
92
|
+
--execute flows/perflab/run-perflab.yaml
|
|
93
|
+
--suite-overlay "flows/perflab/tokens.js:$OUT_DIR/perflab-tokens.js")
|
|
94
|
+
[ -n "$APP_CUSTOM_ID" ] && args+=(--custom-id "$APP_CUSTOM_ID")
|
|
95
|
+
node scripts/farm/browserstack-run.mjs "${args[@]}"
|
|
96
|
+
|
|
97
|
+
# Runs whatever the device step did: a failed session still closes the set with what arrived. Writes the verdict
|
|
98
|
+
# and the dashboard link to $GITHUB_STEP_SUMMARY.
|
|
99
|
+
- name: Close the run set and print the verdict
|
|
100
|
+
if: always() && hashFiles('farm-run/perflab-farm.json') != ''
|
|
101
|
+
run: |
|
|
102
|
+
npx --yes "$KEEPSNAPPY_CLI" farm close \
|
|
103
|
+
--state "$OUT_DIR/perflab-farm.json" \
|
|
104
|
+
--bs-build "$OUT_DIR/browserstack.${{ inputs.platform }}.json" --settle-seconds 60 --wait
|
|
105
|
+
|
|
106
|
+
# Run tokens never go into the artifact, also when open or close failed half-way.
|
|
107
|
+
- name: Remove the run tokens
|
|
108
|
+
if: always()
|
|
109
|
+
run: rm -f "$OUT_DIR/perflab-tokens.js"
|
|
110
|
+
|
|
111
|
+
- name: Upload the run
|
|
112
|
+
if: always()
|
|
113
|
+
uses: actions/upload-artifact@v4
|
|
114
|
+
with:
|
|
115
|
+
name: perflab-${{ inputs.platform }}-${{ github.run_id }}
|
|
116
|
+
path: ${{ env.OUT_DIR }}
|
|
117
|
+
retention-days: 30
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# .github/workflows/keepsnappy.yml: measure every pull request against main, and grow main's history on every merge.
|
|
2
|
+
#
|
|
3
|
+
# Needs: a perf build of your app (built with PERFLAB=1, so the KeepSnappy SDK writes its data), Maestro flows, a
|
|
4
|
+
# keepsnappy.yml in the repo root (see examples/keepsnappy.yml), and a project key with `run` scope stored as the repository
|
|
5
|
+
# secret KEEPSNAPPY_KEY. The key never appears in logs; keepsnappy reads it from the environment variable keepsnappy.yml names.
|
|
6
|
+
#
|
|
7
|
+
# What you get: on main, the runs join the history of each journey (one journey per Maestro flow file). On a pull
|
|
8
|
+
# request, each journey is compared with main's recent history; the verdict ("More work", "Less work", "No change",
|
|
9
|
+
# "Not enough data yet") is printed in the log and written to the job summary with a link to the comparison page.
|
|
10
|
+
# The job does not fail on "More work" (no blocking checks for now); add --fail-on-worse to make it exit 3 instead.
|
|
11
|
+
# Exit 2 = configuration problem (keepsnappy.yml, flags), exit 1 = infrastructure (device log, upload).
|
|
12
|
+
|
|
13
|
+
name: perflab
|
|
14
|
+
|
|
15
|
+
on:
|
|
16
|
+
pull_request:
|
|
17
|
+
push:
|
|
18
|
+
branches: [main]
|
|
19
|
+
|
|
20
|
+
concurrency:
|
|
21
|
+
group: perflab-${{ github.ref }}
|
|
22
|
+
cancel-in-progress: true
|
|
23
|
+
|
|
24
|
+
jobs:
|
|
25
|
+
measure:
|
|
26
|
+
runs-on: macos-15
|
|
27
|
+
timeout-minutes: 60
|
|
28
|
+
steps:
|
|
29
|
+
- uses: actions/checkout@v4
|
|
30
|
+
|
|
31
|
+
# Your own build step. It must produce the simulator .app of the perf build.
|
|
32
|
+
- name: Build the perf build
|
|
33
|
+
run: ./scripts/build-perf-simulator.sh # e.g. expo prebuild + xcodebuild with PERFLAB=1 -> build/MyApp.app
|
|
34
|
+
|
|
35
|
+
- name: Boot a simulator and install the app
|
|
36
|
+
run: |
|
|
37
|
+
xcrun simctl boot "iPhone 16" || true
|
|
38
|
+
xcrun simctl bootstatus "iPhone 16" -b
|
|
39
|
+
xcrun simctl install booted build/MyApp.app
|
|
40
|
+
|
|
41
|
+
- name: Install Maestro
|
|
42
|
+
run: |
|
|
43
|
+
curl -fsSL "https://get.maestro.mobile.dev" | bash
|
|
44
|
+
echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"
|
|
45
|
+
|
|
46
|
+
- name: Install perflab
|
|
47
|
+
run: npm install -g @keepsnappy/agent # the KeepSnappy CLI (`perflab`)
|
|
48
|
+
|
|
49
|
+
# perflab picks the branch and commit from the GitHub environment (GITHUB_HEAD_REF on a pull request, else
|
|
50
|
+
# GITHUB_REF_NAME; GITHUB_SHA). On a pull request GITHUB_SHA is the merge commit, so the head commit is passed.
|
|
51
|
+
- name: Measure, upload and wait for the verdict
|
|
52
|
+
env:
|
|
53
|
+
KEEPSNAPPY_KEY: ${{ secrets.KEEPSNAPPY_KEY }}
|
|
54
|
+
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
|
55
|
+
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
|
56
|
+
run: |
|
|
57
|
+
args=(run --upload --wait)
|
|
58
|
+
if [ -n "$PR_HEAD_SHA" ]; then args+=(--commit "$PR_HEAD_SHA" --base-commit "$PR_BASE_SHA"); fi
|
|
59
|
+
perflab "${args[@]}"
|
|
60
|
+
# perflab appends the verdict and the dashboard link to $GITHUB_STEP_SUMMARY itself, and writes
|
|
61
|
+
# perflab-out/perflab-result.json ({run_set_id, dashboard_url, state, summary}) for later steps.
|
|
62
|
+
|
|
63
|
+
# When the measuring step failed before a verdict, still point the summary at the dashboard if a run set exists.
|
|
64
|
+
- name: Link the dashboard
|
|
65
|
+
if: failure()
|
|
66
|
+
run: |
|
|
67
|
+
if [ -f perflab-out/perflab-run.json ]; then
|
|
68
|
+
echo "KeepSnappy did not finish; runs uploaded so far are on the dashboard (run set $(jq -r '.run_set_id // "none"' perflab-out/perflab-run.json))." >> "$GITHUB_STEP_SUMMARY"
|
|
69
|
+
fi
|
|
70
|
+
|
|
71
|
+
- name: Keep the raw results
|
|
72
|
+
if: always()
|
|
73
|
+
uses: actions/upload-artifact@v4
|
|
74
|
+
with:
|
|
75
|
+
name: perflab-out
|
|
76
|
+
path: perflab-out
|
|
77
|
+
retention-days: 14
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# keepsnappy.yml: the only file you maintain for KeepSnappy. Run it with `keepsnappy run`.
|
|
2
|
+
|
|
3
|
+
project:
|
|
4
|
+
key_env: KEEPSNAPPY_KEY # NAME of the env var holding the project key (a CI secret), never the key itself
|
|
5
|
+
ingest_url: https://ingest.example.com # optional; only used with `keepsnappy run --upload`
|
|
6
|
+
|
|
7
|
+
app:
|
|
8
|
+
bundle_id: com.example.app # the app under test; its perf-lab data is read from the device after every run
|
|
9
|
+
|
|
10
|
+
platform: ios-sim # ios-sim | ios-device
|
|
11
|
+
device: booted # simulator UDID or "booted"; required (UDID) for ios-device
|
|
12
|
+
|
|
13
|
+
repetitions: 10 # how many times the whole flow set runs
|
|
14
|
+
baseline_branch: main # the branch your baseline comes from (CI passes --baseline <its out dir>)
|
|
15
|
+
|
|
16
|
+
flows:
|
|
17
|
+
runner: maestro # maestro: build `maestro test ...` from this file | command: run flows.command
|
|
18
|
+
maestro_version: 2.6.1 # note only: printed at the start, so a drifting Maestro is visible in logs
|
|
19
|
+
tags: [perf] # maestro --include-tags=perf
|
|
20
|
+
exclude_tags: [flaky] # maestro --exclude-tags=flaky
|
|
21
|
+
paths: # files, directories or globs (*, ?, **); default "." when only tags are given
|
|
22
|
+
- e2e/flows/**/*.yaml
|
|
23
|
+
# - path: e2e/flows/chat/open.yaml # one flow file with its dashboard title (else its `name:` / opening comment)
|
|
24
|
+
# title: Open the chat
|
|
25
|
+
# description: Community screen -> chat sheet, first message visible
|
|
26
|
+
# command: bun run e2e # runner: command. Anything that exits 0 on success (Appium, Detox, ...)
|
|
27
|
+
|
|
28
|
+
steps:
|
|
29
|
+
rename: # stable names in history; keys are window/step names as they appear in metrics.csv
|
|
30
|
+
"screen:(app)/community/[handle]/(tabs)": community.open
|
|
31
|
+
pin: [community.open] # always listed in the summary, even without a change
|
|
32
|
+
|
|
33
|
+
output:
|
|
34
|
+
dir: perflab-out # runs.ndjson, metrics.csv, <run>/ logs, verdict (compare/verdict.json)
|
|
35
|
+
|
|
36
|
+
timeout_seconds: 600 # per run of the command; optional
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
# Gigaverse chat message lifecycle (send -> edit -> delete), with two declared KeepSnappy steps:
|
|
2
|
+
# chat.open tap the chat button .. composer visible
|
|
3
|
+
# chat.send tap send .. the sent message is visible
|
|
4
|
+
# Copied from gigaverse-mobile-perflab/e2e/flows/chat/e2e-community-and-live-chat.yaml; only the paths to
|
|
5
|
+
# the shared elements/scripts/subflows (sibling checkout) and the four step.yaml calls differ.
|
|
6
|
+
# Run: see README.md in this folder.
|
|
7
|
+
appId: com.gigaverse.appdev
|
|
8
|
+
---
|
|
9
|
+
# Community chat message lifecycle: send → edit → delete, in the community-v2
|
|
10
|
+
# chat sheet. This surface had no V1 equivalent, so nothing was carried over
|
|
11
|
+
# when the screen was rebuilt — it is the largest untested area of the app.
|
|
12
|
+
#
|
|
13
|
+
# Attachments are deliberately out of scope here. Picking an image means the
|
|
14
|
+
# native photo picker, which is its own fragility; the text lifecycle lands
|
|
15
|
+
# first. Worth adding later, because a failed upload resolves rather than
|
|
16
|
+
# rejecting, so the composer drops the image and sends the message without it
|
|
17
|
+
# — silently.
|
|
18
|
+
#
|
|
19
|
+
# Self-cleaning: the message it sends is the message it deletes, so the
|
|
20
|
+
# channel is left as it was found.
|
|
21
|
+
#
|
|
22
|
+
# Prerequisites:
|
|
23
|
+
# - signed-in platform QA account, member of its platform community
|
|
24
|
+
# - QA account may edit and delete its own messages (default for an owner)
|
|
25
|
+
|
|
26
|
+
# Suite gate — abort in ms if the `login` gate failed; undefined (standalone
|
|
27
|
+
# local run) passes.
|
|
28
|
+
- assertTrue:
|
|
29
|
+
condition: ${output.loginSucceeded !== false}
|
|
30
|
+
label: Aborted — the login gate failed, no QA session
|
|
31
|
+
- launchApp
|
|
32
|
+
- runScript:
|
|
33
|
+
file: ../../../../../gigaverse-mobile-perflab/e2e/elements/home.js
|
|
34
|
+
- runScript:
|
|
35
|
+
file: ../../../../../gigaverse-mobile-perflab/e2e/elements/community.js
|
|
36
|
+
- runScript:
|
|
37
|
+
file: ../../../../../gigaverse-mobile-perflab/e2e/elements/community-chat.js
|
|
38
|
+
- runScript:
|
|
39
|
+
file: ../../../../../gigaverse-mobile-perflab/e2e/scripts/generate-test-values.js
|
|
40
|
+
|
|
41
|
+
- runFlow: ../../../../../gigaverse-mobile-perflab/e2e/subflows/open-qa-community.yaml
|
|
42
|
+
|
|
43
|
+
# 1) Open the chat sheet. The chat button sits beside the destination bar,
|
|
44
|
+
# not inside it — it is not one of the nav destinations.
|
|
45
|
+
- runFlow:
|
|
46
|
+
file: ../../maestro/perflab/step.yaml
|
|
47
|
+
env: { STEP: chat.open, PHASE: start }
|
|
48
|
+
- tapOn:
|
|
49
|
+
id: ${output.elements.communityChat.open}
|
|
50
|
+
- extendedWaitUntil:
|
|
51
|
+
visible:
|
|
52
|
+
id: ${output.elements.communityChat.composerInput}
|
|
53
|
+
timeout: 30000
|
|
54
|
+
- runFlow:
|
|
55
|
+
file: ../../maestro/perflab/step.yaml
|
|
56
|
+
env: { STEP: chat.open, PHASE: end }
|
|
57
|
+
|
|
58
|
+
# 2) Send a message. Every assertion below pins it by id + text rather than by
|
|
59
|
+
# index: the channel is live and another member's message can land mid-run.
|
|
60
|
+
- tapOn:
|
|
61
|
+
id: ${output.elements.communityChat.composerInput}
|
|
62
|
+
- waitForAnimationToEnd
|
|
63
|
+
- inputText: ${output.testChatMessage}
|
|
64
|
+
# Assert the text reached the field before sending — Android adb typing races
|
|
65
|
+
# field focus and drops input silently, and an empty send is a no-op that
|
|
66
|
+
# would only surface two steps later as a missing message.
|
|
67
|
+
- extendedWaitUntil:
|
|
68
|
+
visible:
|
|
69
|
+
id: ${output.elements.communityChat.composerInput}
|
|
70
|
+
text: .*${output.testChatMessage}.*
|
|
71
|
+
timeout: 10000
|
|
72
|
+
- runFlow:
|
|
73
|
+
file: ../../maestro/perflab/step.yaml
|
|
74
|
+
env: { STEP: chat.send, PHASE: start }
|
|
75
|
+
- tapOn:
|
|
76
|
+
id: ${output.elements.communityChat.send}
|
|
77
|
+
- extendedWaitUntil:
|
|
78
|
+
visible:
|
|
79
|
+
id: ${output.elements.communityChat.message}
|
|
80
|
+
timeout: 20000
|
|
81
|
+
- runFlow:
|
|
82
|
+
file: ../../maestro/perflab/step.yaml
|
|
83
|
+
env: { STEP: chat.send, PHASE: end }
|
|
84
|
+
- extendedWaitUntil:
|
|
85
|
+
visible:
|
|
86
|
+
text: .*${output.testChatMessage}.*
|
|
87
|
+
timeout: 20000
|
|
88
|
+
label: chat.send.appears
|
|
89
|
+
|
|
90
|
+
# 3) Edit it. Long-press opens the actions sheet; Edit puts the composer into
|
|
91
|
+
# editing mode with the draft pre-filled, which is why the erase is required —
|
|
92
|
+
# a focused input never re-selects on tap, so a bare inputText would append.
|
|
93
|
+
#
|
|
94
|
+
# The plain tap before the long-press is what makes this deterministic, and it
|
|
95
|
+
# is not a retry in disguise. Sending does NOT blur the composer, so the
|
|
96
|
+
# keyboard is still up, and the actions sheet will not present while the
|
|
97
|
+
# keyboard is being torn down — measured: the long-press dismisses the keyboard
|
|
98
|
+
# and no sheet appears. Retrying papers over that; removing the precondition
|
|
99
|
+
# ends it. The message row is a Touchable with no onPress, so a short tap on it
|
|
100
|
+
# blurs the composer and does nothing else, and the long-press that follows
|
|
101
|
+
# lands with no keyboard to dismiss.
|
|
102
|
+
- tapOn:
|
|
103
|
+
text: .*${output.testChatMessage}.*
|
|
104
|
+
- waitForAnimationToEnd:
|
|
105
|
+
timeout: 3000
|
|
106
|
+
- longPressOn:
|
|
107
|
+
text: .*${output.testChatMessage}.*
|
|
108
|
+
- extendedWaitUntil:
|
|
109
|
+
visible:
|
|
110
|
+
id: ${output.elements.communityChat.actions.edit}
|
|
111
|
+
timeout: 15000
|
|
112
|
+
label: chat.actions.long-press-opens
|
|
113
|
+
- tapOn:
|
|
114
|
+
id: ${output.elements.communityChat.actions.edit}
|
|
115
|
+
- extendedWaitUntil:
|
|
116
|
+
visible:
|
|
117
|
+
id: ${output.elements.communityChat.confirmEdit}
|
|
118
|
+
timeout: 15000
|
|
119
|
+
- tapOn:
|
|
120
|
+
id: ${output.elements.communityChat.composerInput}
|
|
121
|
+
- eraseText: 50
|
|
122
|
+
- inputText: ${output.testChatMessageEdited}
|
|
123
|
+
- tapOn:
|
|
124
|
+
id: ${output.elements.communityChat.confirmEdit}
|
|
125
|
+
- extendedWaitUntil:
|
|
126
|
+
visible:
|
|
127
|
+
id: ${output.elements.communityChat.message}
|
|
128
|
+
timeout: 20000
|
|
129
|
+
- extendedWaitUntil:
|
|
130
|
+
visible:
|
|
131
|
+
text: .*${output.testChatMessageEdited}.*
|
|
132
|
+
timeout: 20000
|
|
133
|
+
label: chat.edit.replaces
|
|
134
|
+
|
|
135
|
+
# 4) Delete it, and prove it is gone. extendedWaitUntil rather than a bare
|
|
136
|
+
# assertNotVisible: the row leaves on the next Stream event, not on the tap.
|
|
137
|
+
# Same precondition as above: confirming an edit leaves the composer focused.
|
|
138
|
+
- tapOn:
|
|
139
|
+
text: .*${output.testChatMessageEdited}.*
|
|
140
|
+
- waitForAnimationToEnd:
|
|
141
|
+
timeout: 3000
|
|
142
|
+
- longPressOn:
|
|
143
|
+
text: .*${output.testChatMessageEdited}.*
|
|
144
|
+
- extendedWaitUntil:
|
|
145
|
+
visible:
|
|
146
|
+
id: ${output.elements.communityChat.actions.delete}
|
|
147
|
+
timeout: 15000
|
|
148
|
+
- tapOn:
|
|
149
|
+
id: ${output.elements.communityChat.actions.delete}
|
|
150
|
+
- extendedWaitUntil:
|
|
151
|
+
visible:
|
|
152
|
+
id: ${output.elements.communityChat.deleteConfirm}
|
|
153
|
+
timeout: 15000
|
|
154
|
+
- tapOn:
|
|
155
|
+
id: ${output.elements.communityChat.deleteConfirm}
|
|
156
|
+
- extendedWaitUntil:
|
|
157
|
+
notVisible:
|
|
158
|
+
text: .*${output.testChatMessageEdited}.*
|
|
159
|
+
timeout: 20000
|
|
160
|
+
label: chat.delete.removes
|
|
161
|
+
optional: true
|