@keepsnappy/agent 0.1.0-alpha.11 → 0.1.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -50
- package/dist/appium-driver.js +136 -10
- package/dist/keepsnappy.js +87 -90
- package/examples/farm/flows/{perflab → keepsnappy}/begin-run.yaml +1 -1
- package/examples/farm/flows/{perflab → keepsnappy}/end-run.yaml +4 -4
- package/examples/farm/flows/keepsnappy/next-token.js +7 -0
- package/examples/farm/flows/{perflab/run-perflab.yaml → keepsnappy/run-keepsnappy.yaml} +1 -1
- package/examples/farm/flows/{perflab → keepsnappy}/send-markers.js +4 -5
- package/examples/farm/flows/keepsnappy/step-markers.yaml +13 -0
- package/examples/farm/flows/keepsnappy/tokens.js +2 -0
- package/examples/farm/workflows/{perflab-measure.yml → keepsnappy-measure.yml} +20 -20
- package/examples/farm/workflows/perf-device-run.fragment.yml +5 -5
- package/examples/github-actions.yml +18 -18
- package/examples/keepsnappy.yml +3 -3
- package/examples/maestro-steps/e2e-community-and-live-chat.yaml +10 -10
- package/maestro/{perflab → keepsnappy}/send-markers.js +4 -5
- package/maestro/{perflab → keepsnappy}/step.js +10 -11
- package/maestro/keepsnappy/step.yaml +11 -0
- package/package.json +2 -3
- package/examples/farm/flows/perflab/next-token.js +0 -7
- package/examples/farm/flows/perflab/step-markers.yaml +0 -13
- package/examples/farm/flows/perflab/tokens.js +0 -2
- package/examples/perflab.yml +0 -36
- package/maestro/perflab/step.yaml +0 -11
|
@@ -13,5 +13,5 @@ appId: com.gigaverse.appdev
|
|
|
13
13
|
id: ${output.elements.home.screen}
|
|
14
14
|
timeout: 30000
|
|
15
15
|
# The settings come from tokens.js (`keepsnappy farm open`): the run defaults, never written by hand here (REQ-FARM-28).
|
|
16
|
-
- openLink: gigaversedev://devtools/
|
|
16
|
+
- openLink: gigaversedev://devtools/keepsnappy?activate=1&${output.keepsnappyLinkArgs}&runToken=${output.keepsnappyToken}
|
|
17
17
|
- runFlow: ../../subflows/confirm-open-link.yaml
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
# End of one measured iteration: the SDK sends everything still unsent (nothing is sent while a run is active), then
|
|
2
|
-
# shows `
|
|
2
|
+
# shows `keepsnappy.flushed` (ok | error | off). `activate=0` makes the next launch dormant again.
|
|
3
3
|
appId: com.gigaverse.appdev
|
|
4
4
|
---
|
|
5
5
|
- runScript: send-markers.js # the iteration's step markers, recorded by step.js, sent now that it has ended
|
|
6
|
-
- openLink: gigaversedev://devtools/
|
|
6
|
+
- openLink: gigaversedev://devtools/keepsnappy?flush=1&activate=0
|
|
7
7
|
- runFlow: ../../subflows/confirm-open-link.yaml
|
|
8
8
|
- extendedWaitUntil:
|
|
9
9
|
visible:
|
|
10
|
-
id:
|
|
10
|
+
id: keepsnappy.flushed
|
|
11
11
|
timeout: 60000
|
|
12
12
|
optional: true # a dormant iteration (activation failed) shows nothing; farm close marks it invalid
|
|
13
13
|
- assertVisible:
|
|
14
|
-
id:
|
|
14
|
+
id: keepsnappy.flushed
|
|
15
15
|
text: ok
|
|
16
16
|
optional: true
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Hands out the run tokens of one journey in order: iteration i of JOURNEY gets output.keepsnappyTokens[JOURNEY][i].
|
|
2
|
+
// ES5 (Maestro runScript). An empty token (more iterations than tokens) leaves the SDK dormant: no data, no harm.
|
|
3
|
+
var list = (output.keepsnappyTokens || {})[JOURNEY] || [];
|
|
4
|
+
output.keepsnappyIdx = output.keepsnappyIdx || {};
|
|
5
|
+
var i = output.keepsnappyIdx[JOURNEY] || 0;
|
|
6
|
+
output.keepsnappyToken = list[i] || '';
|
|
7
|
+
output.keepsnappyIdx[JOURNEY] = i + 1;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# e2e/flows/
|
|
1
|
+
# e2e/flows/keepsnappy/run-keepsnappy.yaml (plan 36 D10). One BrowserStack session runs every iteration: the session is the
|
|
2
2
|
# noise block. Untagged, so smoke/regression never select it; the dispatcher names it with --execute.
|
|
3
3
|
# `times` must equal `keepsnappy farm open --repeat`. One begin/journey/end triple per journey listed in --journeys.
|
|
4
4
|
appId: com.gigaverse.appdev
|
|
@@ -13,8 +13,7 @@
|
|
|
13
13
|
try {
|
|
14
14
|
var g = typeof globalThis !== 'undefined' ? globalThis : this;
|
|
15
15
|
var pick = function (name, dflt) {
|
|
16
|
-
var
|
|
17
|
-
var names = modern === name ? [name, 'MAESTRO_' + name] : [modern, 'MAESTRO_' + modern, name, 'MAESTRO_' + name];
|
|
16
|
+
var names = [name, 'MAESTRO_' + name];
|
|
18
17
|
for (var i = 0; i < names.length; i++) {
|
|
19
18
|
var v = g[names[i]];
|
|
20
19
|
if (typeof v === 'string' && (v !== '' || names[i].indexOf('KEEPSNAPPY_') >= 0)) return v;
|
|
@@ -27,9 +26,9 @@ try {
|
|
|
27
26
|
var RETRY_CAP_MS = 30000;
|
|
28
27
|
var CLOCK_PROBES = 4;
|
|
29
28
|
var out = typeof output !== 'undefined' && output ? output : {};
|
|
30
|
-
var list = out.
|
|
31
|
-
out.
|
|
32
|
-
var api = pick('
|
|
29
|
+
var list = out.keepsnappyMarkers || [];
|
|
30
|
+
out.keepsnappyMarkers = [];
|
|
31
|
+
var api = pick('KEEPSNAPPY_API_URL', typeof out.keepsnappyApi === 'string' ? out.keepsnappyApi : '').replace(/\/+$/, '');
|
|
33
32
|
var wait = function (ms) {
|
|
34
33
|
try {
|
|
35
34
|
java.lang.Thread.sleep(ms);
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# Declared steps in farm mode (plan 36 M2, G4). Copy apps/agent/maestro/keepsnappy/step.yaml and step.js next to this file
|
|
2
|
+
# (e2e/flows/keepsnappy/); send-markers.js is already here, the same file as maestro/keepsnappy/send-markers.js. In farm mode
|
|
3
|
+
# step.js only records the marker with the iteration's token (`output.keepsnappyToken`, taken by begin-run.yaml); send-markers.js in end-run.yaml posts them to
|
|
4
|
+
# <output.keepsnappyApi>/v1/markers once the iteration ended, so the test never waits on the network between actions and
|
|
5
|
+
# no CLI listens on the farm host. Without send-markers.js in end-run.yaml no farm marker is ever sent.
|
|
6
|
+
# A journey that failed half-way drops out of the statistics because it never posted its set's completion marker and
|
|
7
|
+
# `keepsnappy farm close` marks it invalid. Its step markers still reach KeepSnappy: run-keepsnappy.yaml sends what is left from
|
|
8
|
+
# onFlowComplete, for diagnosis. Inside the journey, around the part to measure:
|
|
9
|
+
appId: com.gigaverse.appdev
|
|
10
|
+
---
|
|
11
|
+
- runFlow: { file: ../keepsnappy/step.yaml, env: { STEP: chat.open, PHASE: start } }
|
|
12
|
+
- tapOn: { id: chat-open }
|
|
13
|
+
- runFlow: { file: ../keepsnappy/step.yaml, env: { STEP: chat.open, PHASE: end } }
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
# .github/workflows/
|
|
1
|
+
# .github/workflows/keepsnappy-measure.yml (plan 36 D8): KeepSnappy on BrowserStack, called last in perf-device-run.yml.
|
|
2
2
|
#
|
|
3
3
|
# The CLI never touches the device. `farm open` makes the run set and one KeepSnappy run per journey per iteration, and
|
|
4
|
-
# writes their run tokens into
|
|
5
|
-
# inside the uploaded suite: browserstack-run.mjs --suite-overlay writes that file to e2e/flows/
|
|
4
|
+
# writes their run tokens into keepsnappy-tokens.js. BrowserStack caps env values at 100 characters, so the tokens ride
|
|
5
|
+
# inside the uploaded suite: browserstack-run.mjs --suite-overlay writes that file to e2e/flows/keepsnappy/tokens.js for
|
|
6
6
|
# this one upload. The app, activated per iteration with its token, pushes its windows to the KeepSnappy API itself.
|
|
7
7
|
# `farm close` waits until no batch has arrived for --settle-seconds, finishes every run (the one a failed session
|
|
8
8
|
# broke is marked invalid; iterations that never ran show as "no data"), closes the set and prints the verdict.
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
# iteration sent data) or the API failed, 2 for a configuration problem, 3 with --fail-on-worse on "More work".
|
|
12
12
|
# Re-running close is safe: finished runs are left alone and the same result is printed again.
|
|
13
13
|
#
|
|
14
|
-
# Files in $OUT_DIR after the job (uploaded below):
|
|
15
|
-
#
|
|
14
|
+
# Files in $OUT_DIR after the job (uploaded below): keepsnappy-farm.json (state: run set, runs, result; no tokens),
|
|
15
|
+
# keepsnappy-result.json, browserstack.ios.json. keepsnappy-tokens.js holds live run tokens (valid for hours): close puts the
|
|
16
16
|
# empty stub back even when it fails, and the job deletes the file before the upload whatever happened.
|
|
17
17
|
|
|
18
18
|
name: KeepSnappy measure (reusable)
|
|
@@ -36,12 +36,12 @@ jobs:
|
|
|
36
36
|
BROWSERSTACK_USERNAME: ${{ secrets.BROWSERSTACK_USERNAME }}
|
|
37
37
|
BROWSERSTACK_ACCESS_KEY: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
|
|
38
38
|
KEEPSNAPPY_KEY: ${{ secrets.KEEPSNAPPY_KEY }} # run scope; never leaves this runner
|
|
39
|
-
KEEPSNAPPY_API: https://
|
|
40
|
-
|
|
39
|
+
KEEPSNAPPY_API: https://keepsnappy-api-922918702810.europe-west1.run.app
|
|
40
|
+
KEEPSNAPPY_CLI: '@keepsnappy/agent@0.1.0-alpha.1' # first version with `farm` (not published yet)
|
|
41
41
|
OUT_DIR: ${{ github.workspace }}/farm-run
|
|
42
|
-
RUN_ID:
|
|
43
|
-
JOURNEYS: e2e-community-and-live-chat # = the JOURNEY values in run-
|
|
44
|
-
REPEAT: '5' # = repeat.times in run-
|
|
42
|
+
RUN_ID: keepsnappy-${{ inputs.platform }}-${{ github.run_id }}-${{ github.run_attempt }}
|
|
43
|
+
JOURNEYS: e2e-community-and-live-chat # = the JOURNEY values in run-keepsnappy.yaml
|
|
44
|
+
REPEAT: '5' # = repeat.times in run-keepsnappy.yaml
|
|
45
45
|
steps:
|
|
46
46
|
- uses: actions/checkout@v5
|
|
47
47
|
- uses: actions/setup-node@v4
|
|
@@ -71,11 +71,11 @@ jobs:
|
|
|
71
71
|
APP_COMMIT: ${{ steps.app.outputs.app_commit }}
|
|
72
72
|
run: |
|
|
73
73
|
[ -n "$APP_COMMIT" ] || { echo '::error::the artifact carries no commit'; exit 1; }
|
|
74
|
-
npx --yes "$
|
|
74
|
+
npx --yes "$KEEPSNAPPY_CLI" farm open \
|
|
75
75
|
--api "$KEEPSNAPPY_API" --branch main --commit "$APP_COMMIT" \
|
|
76
76
|
--device-model 'iPhone 15 Pro' --device-os 17 --device-provider browserstack \
|
|
77
77
|
--journeys "$JOURNEYS" --repeat "$REPEAT" \
|
|
78
|
-
--tokens-js "$OUT_DIR/
|
|
78
|
+
--tokens-js "$OUT_DIR/keepsnappy-tokens.js" --state "$OUT_DIR/keepsnappy-farm.json"
|
|
79
79
|
|
|
80
80
|
- name: Run the KeepSnappy orchestrator on the device
|
|
81
81
|
id: device
|
|
@@ -88,30 +88,30 @@ jobs:
|
|
|
88
88
|
PLATFORM: ${{ inputs.platform }}
|
|
89
89
|
run: |
|
|
90
90
|
args=(--platform "$PLATFORM" --device "$DEVICE" --app "$APP" --run-id "$RUN_ID"
|
|
91
|
-
--label
|
|
92
|
-
--execute flows/
|
|
93
|
-
--suite-overlay "flows/
|
|
91
|
+
--label keepsnappy --maestro-version "$MAESTRO" --out "$OUT_DIR" --timeout-minutes 80
|
|
92
|
+
--execute flows/keepsnappy/run-keepsnappy.yaml
|
|
93
|
+
--suite-overlay "flows/keepsnappy/tokens.js:$OUT_DIR/keepsnappy-tokens.js")
|
|
94
94
|
[ -n "$APP_CUSTOM_ID" ] && args+=(--custom-id "$APP_CUSTOM_ID")
|
|
95
95
|
node scripts/farm/browserstack-run.mjs "${args[@]}"
|
|
96
96
|
|
|
97
97
|
# Runs whatever the device step did: a failed session still closes the set with what arrived. Writes the verdict
|
|
98
98
|
# and the dashboard link to $GITHUB_STEP_SUMMARY.
|
|
99
99
|
- name: Close the run set and print the verdict
|
|
100
|
-
if: always() && hashFiles('farm-run/
|
|
100
|
+
if: always() && hashFiles('farm-run/keepsnappy-farm.json') != ''
|
|
101
101
|
run: |
|
|
102
|
-
npx --yes "$
|
|
103
|
-
--state "$OUT_DIR/
|
|
102
|
+
npx --yes "$KEEPSNAPPY_CLI" farm close \
|
|
103
|
+
--state "$OUT_DIR/keepsnappy-farm.json" \
|
|
104
104
|
--bs-build "$OUT_DIR/browserstack.${{ inputs.platform }}.json" --settle-seconds 60 --wait
|
|
105
105
|
|
|
106
106
|
# Run tokens never go into the artifact, also when open or close failed half-way.
|
|
107
107
|
- name: Remove the run tokens
|
|
108
108
|
if: always()
|
|
109
|
-
run: rm -f "$OUT_DIR/
|
|
109
|
+
run: rm -f "$OUT_DIR/keepsnappy-tokens.js"
|
|
110
110
|
|
|
111
111
|
- name: Upload the run
|
|
112
112
|
if: always()
|
|
113
113
|
uses: actions/upload-artifact@v4
|
|
114
114
|
with:
|
|
115
|
-
name:
|
|
115
|
+
name: keepsnappy-${{ inputs.platform }}-${{ github.run_id }}
|
|
116
116
|
path: ${{ env.OUT_DIR }}
|
|
117
117
|
retention-days: 30
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
# Fragment for .github/workflows/perf-device-run.yml (plan 36 D7): KeepSnappy takes the React profiler's slot, last in the
|
|
2
|
-
# chain, on the same eas:<id> the baseline and gigaload measured. Add `
|
|
3
|
-
# (
|
|
2
|
+
# chain, on the same eas:<id> the baseline and gigaload measured. Add `keepsnappy` to the `prepare` options parsing
|
|
3
|
+
# (keepsnappy=on|off, on by default; refuse baseline=off gigaload=off keepsnappy=off) and its output, then this job:
|
|
4
4
|
|
|
5
5
|
jobs:
|
|
6
6
|
# Last, in the slot the React profiler was built for: the same eas:<id> the baseline and gigaload measured, one
|
|
7
7
|
# BrowserStack session, no tunnel, no load fleet. It runs whatever the perf jobs concluded and never fails the night
|
|
8
8
|
# on a KeepSnappy result (exit 1 only when nothing could be judged).
|
|
9
|
-
|
|
9
|
+
keepsnappy:
|
|
10
10
|
needs: [prepare, baseline-report, gigaload]
|
|
11
11
|
if: >-
|
|
12
12
|
always() && !cancelled() && needs.prepare.result == 'success'
|
|
13
|
-
&& needs.prepare.outputs.
|
|
13
|
+
&& needs.prepare.outputs.keepsnappy == 'on'
|
|
14
14
|
&& needs.prepare.outputs.drive == 'device'
|
|
15
15
|
&& inputs.platform == 'ios'
|
|
16
|
-
uses: ./.github/workflows/
|
|
16
|
+
uses: ./.github/workflows/keepsnappy-measure.yml
|
|
17
17
|
secrets: inherit
|
|
18
18
|
with:
|
|
19
19
|
platform: ${{ inputs.platform }}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
# .github/workflows/
|
|
1
|
+
# .github/workflows/keepsnappy.yml: measure every pull request against main, and grow main's history on every merge.
|
|
2
2
|
#
|
|
3
|
-
# Needs: a perf build of your app (built with
|
|
4
|
-
#
|
|
5
|
-
# secret KEEPSNAPPY_KEY. The key never appears in logs;
|
|
3
|
+
# Needs: a perf build of your app (built with KEEPSNAPPY=1, so the KeepSnappy SDK writes its data), Maestro flows, a
|
|
4
|
+
# keepsnappy.yml in the repo root (see examples/keepsnappy.yml), and a project key with `run` scope stored as the repository
|
|
5
|
+
# secret KEEPSNAPPY_KEY. The key never appears in logs; keepsnappy reads it from the environment variable keepsnappy.yml names.
|
|
6
6
|
#
|
|
7
7
|
# What you get: on main, the runs join the history of each journey (one journey per Maestro flow file). On a pull
|
|
8
8
|
# request, each journey is compared with main's recent history; the verdict ("More work", "Less work", "No change",
|
|
9
9
|
# "Not enough data yet") is printed in the log and written to the job summary with a link to the comparison page.
|
|
10
10
|
# The job does not fail on "More work" (no blocking checks for now); add --fail-on-worse to make it exit 3 instead.
|
|
11
|
-
# Exit 2 = configuration problem (
|
|
11
|
+
# Exit 2 = configuration problem (keepsnappy.yml, flags), exit 1 = infrastructure (device log, upload).
|
|
12
12
|
|
|
13
|
-
name:
|
|
13
|
+
name: keepsnappy
|
|
14
14
|
|
|
15
15
|
on:
|
|
16
16
|
pull_request:
|
|
@@ -18,7 +18,7 @@ on:
|
|
|
18
18
|
branches: [main]
|
|
19
19
|
|
|
20
20
|
concurrency:
|
|
21
|
-
group:
|
|
21
|
+
group: keepsnappy-${{ github.ref }}
|
|
22
22
|
cancel-in-progress: true
|
|
23
23
|
|
|
24
24
|
jobs:
|
|
@@ -30,7 +30,7 @@ jobs:
|
|
|
30
30
|
|
|
31
31
|
# Your own build step. It must produce the simulator .app of the perf build.
|
|
32
32
|
- name: Build the perf build
|
|
33
|
-
run: ./scripts/build-perf-simulator.sh # e.g. expo prebuild + xcodebuild with
|
|
33
|
+
run: ./scripts/build-perf-simulator.sh # e.g. expo prebuild + xcodebuild with KEEPSNAPPY=1 -> build/MyApp.app
|
|
34
34
|
|
|
35
35
|
- name: Boot a simulator and install the app
|
|
36
36
|
run: |
|
|
@@ -43,10 +43,10 @@ jobs:
|
|
|
43
43
|
curl -fsSL "https://get.maestro.mobile.dev" | bash
|
|
44
44
|
echo "$HOME/.maestro/bin" >> "$GITHUB_PATH"
|
|
45
45
|
|
|
46
|
-
- name: Install
|
|
47
|
-
run: npm install -g @keepsnappy/agent # the KeepSnappy CLI (`
|
|
46
|
+
- name: Install keepsnappy
|
|
47
|
+
run: npm install -g @keepsnappy/agent # the KeepSnappy CLI (`keepsnappy`)
|
|
48
48
|
|
|
49
|
-
#
|
|
49
|
+
# keepsnappy picks the branch and commit from the GitHub environment (GITHUB_HEAD_REF on a pull request, else
|
|
50
50
|
# GITHUB_REF_NAME; GITHUB_SHA). On a pull request GITHUB_SHA is the merge commit, so the head commit is passed.
|
|
51
51
|
- name: Measure, upload and wait for the verdict
|
|
52
52
|
env:
|
|
@@ -56,22 +56,22 @@ jobs:
|
|
|
56
56
|
run: |
|
|
57
57
|
args=(run --upload --wait)
|
|
58
58
|
if [ -n "$PR_HEAD_SHA" ]; then args+=(--commit "$PR_HEAD_SHA" --base-commit "$PR_BASE_SHA"); fi
|
|
59
|
-
|
|
60
|
-
#
|
|
61
|
-
#
|
|
59
|
+
keepsnappy "${args[@]}"
|
|
60
|
+
# keepsnappy appends the verdict and the dashboard link to $GITHUB_STEP_SUMMARY itself, and writes
|
|
61
|
+
# keepsnappy-out/keepsnappy-result.json ({run_set_id, dashboard_url, state, summary}) for later steps.
|
|
62
62
|
|
|
63
63
|
# When the measuring step failed before a verdict, still point the summary at the dashboard if a run set exists.
|
|
64
64
|
- name: Link the dashboard
|
|
65
65
|
if: failure()
|
|
66
66
|
run: |
|
|
67
|
-
if [ -f
|
|
68
|
-
echo "KeepSnappy did not finish; runs uploaded so far are on the dashboard (run set $(jq -r '.run_set_id // "none"'
|
|
67
|
+
if [ -f keepsnappy-out/keepsnappy-run.json ]; then
|
|
68
|
+
echo "KeepSnappy did not finish; runs uploaded so far are on the dashboard (run set $(jq -r '.run_set_id // "none"' keepsnappy-out/keepsnappy-run.json))." >> "$GITHUB_STEP_SUMMARY"
|
|
69
69
|
fi
|
|
70
70
|
|
|
71
71
|
- name: Keep the raw results
|
|
72
72
|
if: always()
|
|
73
73
|
uses: actions/upload-artifact@v4
|
|
74
74
|
with:
|
|
75
|
-
name:
|
|
76
|
-
path:
|
|
75
|
+
name: keepsnappy-out
|
|
76
|
+
path: keepsnappy-out
|
|
77
77
|
retention-days: 14
|
package/examples/keepsnappy.yml
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
#
|
|
1
|
+
# keepsnappy.yml: the only file you maintain for KeepSnappy. Run it with `keepsnappy run`.
|
|
2
2
|
|
|
3
3
|
project:
|
|
4
4
|
key_env: KEEPSNAPPY_KEY # NAME of the env var holding the project key (a CI secret), never the key itself
|
|
5
5
|
ingest_url: https://ingest.example.com # optional; only used with `keepsnappy run --upload`
|
|
6
6
|
|
|
7
7
|
app:
|
|
8
|
-
bundle_id: com.example.app # the app under test; its
|
|
8
|
+
bundle_id: com.example.app # the app under test; its keepsnappy data is read from the device after every run
|
|
9
9
|
|
|
10
10
|
platform: ios-sim # ios-sim | ios-device
|
|
11
11
|
device: booted # simulator UDID or "booted"; required (UDID) for ios-device
|
|
@@ -31,6 +31,6 @@ steps:
|
|
|
31
31
|
pin: [community.open] # always listed in the summary, even without a change
|
|
32
32
|
|
|
33
33
|
output:
|
|
34
|
-
dir:
|
|
34
|
+
dir: keepsnappy-out # runs.ndjson, metrics.csv, <run>/ logs, verdict (compare/verdict.json)
|
|
35
35
|
|
|
36
36
|
timeout_seconds: 600 # per run of the command; optional
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Gigaverse chat message lifecycle (send -> edit -> delete), with two declared KeepSnappy steps:
|
|
2
2
|
# chat.open tap the chat button .. composer visible
|
|
3
3
|
# chat.send tap send .. the sent message is visible
|
|
4
|
-
# Copied from gigaverse-mobile-
|
|
4
|
+
# Copied from gigaverse-mobile-keepsnappy/e2e/flows/chat/e2e-community-and-live-chat.yaml; only the paths to
|
|
5
5
|
# the shared elements/scripts/subflows (sibling checkout) and the four step.yaml calls differ.
|
|
6
6
|
# Run: see README.md in this folder.
|
|
7
7
|
appId: com.gigaverse.appdev
|
|
@@ -30,20 +30,20 @@ appId: com.gigaverse.appdev
|
|
|
30
30
|
label: Aborted — the login gate failed, no QA session
|
|
31
31
|
- launchApp
|
|
32
32
|
- runScript:
|
|
33
|
-
file: ../../../../../gigaverse-mobile-
|
|
33
|
+
file: ../../../../../gigaverse-mobile-keepsnappy/e2e/elements/home.js
|
|
34
34
|
- runScript:
|
|
35
|
-
file: ../../../../../gigaverse-mobile-
|
|
35
|
+
file: ../../../../../gigaverse-mobile-keepsnappy/e2e/elements/community.js
|
|
36
36
|
- runScript:
|
|
37
|
-
file: ../../../../../gigaverse-mobile-
|
|
37
|
+
file: ../../../../../gigaverse-mobile-keepsnappy/e2e/elements/community-chat.js
|
|
38
38
|
- runScript:
|
|
39
|
-
file: ../../../../../gigaverse-mobile-
|
|
39
|
+
file: ../../../../../gigaverse-mobile-keepsnappy/e2e/scripts/generate-test-values.js
|
|
40
40
|
|
|
41
|
-
- runFlow: ../../../../../gigaverse-mobile-
|
|
41
|
+
- runFlow: ../../../../../gigaverse-mobile-keepsnappy/e2e/subflows/open-qa-community.yaml
|
|
42
42
|
|
|
43
43
|
# 1) Open the chat sheet. The chat button sits beside the destination bar,
|
|
44
44
|
# not inside it — it is not one of the nav destinations.
|
|
45
45
|
- runFlow:
|
|
46
|
-
file: ../../maestro/
|
|
46
|
+
file: ../../maestro/keepsnappy/step.yaml
|
|
47
47
|
env: { STEP: chat.open, PHASE: start }
|
|
48
48
|
- tapOn:
|
|
49
49
|
id: ${output.elements.communityChat.open}
|
|
@@ -52,7 +52,7 @@ appId: com.gigaverse.appdev
|
|
|
52
52
|
id: ${output.elements.communityChat.composerInput}
|
|
53
53
|
timeout: 30000
|
|
54
54
|
- runFlow:
|
|
55
|
-
file: ../../maestro/
|
|
55
|
+
file: ../../maestro/keepsnappy/step.yaml
|
|
56
56
|
env: { STEP: chat.open, PHASE: end }
|
|
57
57
|
|
|
58
58
|
# 2) Send a message. Every assertion below pins it by id + text rather than by
|
|
@@ -70,7 +70,7 @@ appId: com.gigaverse.appdev
|
|
|
70
70
|
text: .*${output.testChatMessage}.*
|
|
71
71
|
timeout: 10000
|
|
72
72
|
- runFlow:
|
|
73
|
-
file: ../../maestro/
|
|
73
|
+
file: ../../maestro/keepsnappy/step.yaml
|
|
74
74
|
env: { STEP: chat.send, PHASE: start }
|
|
75
75
|
- tapOn:
|
|
76
76
|
id: ${output.elements.communityChat.send}
|
|
@@ -79,7 +79,7 @@ appId: com.gigaverse.appdev
|
|
|
79
79
|
id: ${output.elements.communityChat.message}
|
|
80
80
|
timeout: 20000
|
|
81
81
|
- runFlow:
|
|
82
|
-
file: ../../maestro/
|
|
82
|
+
file: ../../maestro/keepsnappy/step.yaml
|
|
83
83
|
env: { STEP: chat.send, PHASE: end }
|
|
84
84
|
- extendedWaitUntil:
|
|
85
85
|
visible:
|
|
@@ -13,8 +13,7 @@
|
|
|
13
13
|
try {
|
|
14
14
|
var g = typeof globalThis !== 'undefined' ? globalThis : this;
|
|
15
15
|
var pick = function (name, dflt) {
|
|
16
|
-
var
|
|
17
|
-
var names = modern === name ? [name, 'MAESTRO_' + name] : [modern, 'MAESTRO_' + modern, name, 'MAESTRO_' + name];
|
|
16
|
+
var names = [name, 'MAESTRO_' + name];
|
|
18
17
|
for (var i = 0; i < names.length; i++) {
|
|
19
18
|
var v = g[names[i]];
|
|
20
19
|
if (typeof v === 'string' && (v !== '' || names[i].indexOf('KEEPSNAPPY_') >= 0)) return v;
|
|
@@ -27,9 +26,9 @@ try {
|
|
|
27
26
|
var RETRY_CAP_MS = 30000;
|
|
28
27
|
var CLOCK_PROBES = 4;
|
|
29
28
|
var out = typeof output !== 'undefined' && output ? output : {};
|
|
30
|
-
var list = out.
|
|
31
|
-
out.
|
|
32
|
-
var api = pick('
|
|
29
|
+
var list = out.keepsnappyMarkers || [];
|
|
30
|
+
out.keepsnappyMarkers = [];
|
|
31
|
+
var api = pick('KEEPSNAPPY_API_URL', typeof out.keepsnappyApi === 'string' ? out.keepsnappyApi : '').replace(/\/+$/, '');
|
|
33
32
|
var wait = function (ms) {
|
|
34
33
|
try {
|
|
35
34
|
java.lang.Thread.sleep(ms);
|
|
@@ -1,18 +1,17 @@
|
|
|
1
1
|
// Declared step marker. Runs on the Maestro HOST, so it never touches the device. ES5 only (older Maestro builds run
|
|
2
2
|
// Rhino). Must not throw: a marker is never a test failure.
|
|
3
3
|
// Inputs: STEP, PHASE (from step.yaml). t_host is taken now, when the action happens, whenever the marker is sent.
|
|
4
|
-
// Farm mode (`
|
|
5
|
-
// generated tokens.js) and the API base comes from `output.
|
|
4
|
+
// Farm mode (`keepsnappy farm open`): the flow has set `output.keepsnappyToken` (one run token per iteration, from the
|
|
5
|
+
// generated tokens.js) and the API base comes from `output.keepsnappyApi` or KEEPSNAPPY_API_URL. The marker is only recorded
|
|
6
6
|
// in the flow's memory; send-markers.js (end-run.yaml) posts the iteration's markers to <api>/v1/markers after it
|
|
7
7
|
// ended, so the test never waits on the network between actions.
|
|
8
|
-
// Local mode (`
|
|
8
|
+
// Local mode (`keepsnappy run` / `measure`): KEEPSNAPPY_MARKER_URL, KEEPSNAPPY_RUN_ID point at the CLI's own endpoint on this
|
|
9
9
|
// host, posted at once (local flows have no end-run step). Maestro only forwards host env vars with the MAESTRO_ prefix,
|
|
10
10
|
// unprefixed, so both spellings are read (docs/research/28).
|
|
11
11
|
try {
|
|
12
12
|
var g = typeof globalThis !== 'undefined' ? globalThis : this;
|
|
13
13
|
var pick = function (name, dflt) {
|
|
14
|
-
var
|
|
15
|
-
var names = modern === name ? [name, 'MAESTRO_' + name] : [modern, 'MAESTRO_' + modern, name, 'MAESTRO_' + name];
|
|
14
|
+
var names = [name, 'MAESTRO_' + name];
|
|
16
15
|
for (var i = 0; i < names.length; i++) {
|
|
17
16
|
var v = g[names[i]];
|
|
18
17
|
if (typeof v === 'string' && (v !== '' || names[i].indexOf('KEEPSNAPPY_') >= 0)) return v;
|
|
@@ -22,16 +21,16 @@ try {
|
|
|
22
21
|
var out = typeof output !== 'undefined' && output ? output : {};
|
|
23
22
|
var step = pick('STEP', '');
|
|
24
23
|
var phase = pick('PHASE', '');
|
|
25
|
-
var token = typeof out.
|
|
26
|
-
var api = pick('
|
|
24
|
+
var token = typeof out.keepsnappyToken === 'string' ? out.keepsnappyToken : '';
|
|
25
|
+
var api = pick('KEEPSNAPPY_API_URL', typeof out.keepsnappyApi === 'string' ? out.keepsnappyApi : '');
|
|
27
26
|
if (token !== '' && api !== '') {
|
|
28
|
-
out.
|
|
29
|
-
out.
|
|
27
|
+
out.keepsnappyMarkers = out.keepsnappyMarkers || [];
|
|
28
|
+
out.keepsnappyMarkers.push({ token: token, step: step, phase: phase, t_host: Date.now() });
|
|
30
29
|
} else {
|
|
31
|
-
var base = pick('
|
|
30
|
+
var base = pick('KEEPSNAPPY_MARKER_URL', 'http://127.0.0.1:47321');
|
|
32
31
|
http.post(base + '/marker', {
|
|
33
32
|
headers: { 'Content-Type': 'application/json' },
|
|
34
|
-
body: JSON.stringify({ run: pick('
|
|
33
|
+
body: JSON.stringify({ run: pick('KEEPSNAPPY_RUN_ID', ''), step: step, phase: phase, t_host: Date.now() }),
|
|
35
34
|
});
|
|
36
35
|
}
|
|
37
36
|
} catch (e) {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# KeepSnappy step marker. Call with env STEP (name) and PHASE (start|end):
|
|
2
|
+
# - runFlow: { file: keepsnappy/step.yaml, env: { STEP: chat.open, PHASE: start } }
|
|
3
|
+
# Never fails the surrounding flow, even when no KeepSnappy CLI is listening. On the farm it only records the marker:
|
|
4
|
+
# run send-markers.js at the end of each iteration (examples/farm/flows/keepsnappy/end-run.yaml).
|
|
5
|
+
appId: any
|
|
6
|
+
---
|
|
7
|
+
- runScript:
|
|
8
|
+
file: step.js
|
|
9
|
+
env:
|
|
10
|
+
STEP: ${STEP}
|
|
11
|
+
PHASE: ${PHASE}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@keepsnappy/agent",
|
|
3
|
-
"version": "0.1.0-alpha.
|
|
3
|
+
"version": "0.1.0-alpha.13",
|
|
4
4
|
"description": "KeepSnappy CLI: runs your app journeys, measures from inside the app and reports More work / Less work / No change per step.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -16,8 +16,7 @@
|
|
|
16
16
|
"node": ">=22"
|
|
17
17
|
},
|
|
18
18
|
"bin": {
|
|
19
|
-
"keepsnappy": "dist/keepsnappy.js"
|
|
20
|
-
"perflab": "dist/keepsnappy.js"
|
|
19
|
+
"keepsnappy": "dist/keepsnappy.js"
|
|
21
20
|
},
|
|
22
21
|
"files": [
|
|
23
22
|
"dist",
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
// Hands out the run tokens of one journey in order: iteration i of JOURNEY gets output.perflabTokens[JOURNEY][i].
|
|
2
|
-
// ES5 (Maestro runScript). An empty token (more iterations than tokens) leaves the SDK dormant: no data, no harm.
|
|
3
|
-
var list = (output.perflabTokens || {})[JOURNEY] || [];
|
|
4
|
-
output.perflabIdx = output.perflabIdx || {};
|
|
5
|
-
var i = output.perflabIdx[JOURNEY] || 0;
|
|
6
|
-
output.perflabToken = list[i] || '';
|
|
7
|
-
output.perflabIdx[JOURNEY] = i + 1;
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
# Declared steps in farm mode (plan 36 M2, G4). Copy apps/agent/maestro/perflab/step.yaml and step.js next to this file
|
|
2
|
-
# (e2e/flows/perflab/); send-markers.js is already here, the same file as maestro/perflab/send-markers.js. In farm mode
|
|
3
|
-
# step.js only records the marker with the iteration's token (`output.perflabToken`, taken by begin-run.yaml); send-markers.js in end-run.yaml posts them to
|
|
4
|
-
# <output.perflabApi>/v1/markers once the iteration ended, so the test never waits on the network between actions and
|
|
5
|
-
# no CLI listens on the farm host. Without send-markers.js in end-run.yaml no farm marker is ever sent.
|
|
6
|
-
# A journey that failed half-way drops out of the statistics because it never posted its set's completion marker and
|
|
7
|
-
# `keepsnappy farm close` marks it invalid. Its step markers still reach KeepSnappy: run-perflab.yaml sends what is left from
|
|
8
|
-
# onFlowComplete, for diagnosis. Inside the journey, around the part to measure:
|
|
9
|
-
appId: com.gigaverse.appdev
|
|
10
|
-
---
|
|
11
|
-
- runFlow: { file: ../perflab/step.yaml, env: { STEP: chat.open, PHASE: start } }
|
|
12
|
-
- tapOn: { id: chat-open }
|
|
13
|
-
- runFlow: { file: ../perflab/step.yaml, env: { STEP: chat.open, PHASE: end } }
|
package/examples/perflab.yml
DELETED
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
# perflab.yml: the only file you maintain for KeepSnappy. Run it with `keepsnappy run`.
|
|
2
|
-
|
|
3
|
-
project:
|
|
4
|
-
key_env: KEEPSNAPPY_KEY # NAME of the env var holding the project key (a CI secret), never the key itself
|
|
5
|
-
ingest_url: https://ingest.example.com # optional; only used with `keepsnappy run --upload`
|
|
6
|
-
|
|
7
|
-
app:
|
|
8
|
-
bundle_id: com.example.app # the app under test; its perf-lab data is read from the device after every run
|
|
9
|
-
|
|
10
|
-
platform: ios-sim # ios-sim | ios-device
|
|
11
|
-
device: booted # simulator UDID or "booted"; required (UDID) for ios-device
|
|
12
|
-
|
|
13
|
-
repetitions: 10 # how many times the whole flow set runs
|
|
14
|
-
baseline_branch: main # the branch your baseline comes from (CI passes --baseline <its out dir>)
|
|
15
|
-
|
|
16
|
-
flows:
|
|
17
|
-
runner: maestro # maestro: build `maestro test ...` from this file | command: run flows.command
|
|
18
|
-
maestro_version: 2.6.1 # note only: printed at the start, so a drifting Maestro is visible in logs
|
|
19
|
-
tags: [perf] # maestro --include-tags=perf
|
|
20
|
-
exclude_tags: [flaky] # maestro --exclude-tags=flaky
|
|
21
|
-
paths: # files, directories or globs (*, ?, **); default "." when only tags are given
|
|
22
|
-
- e2e/flows/**/*.yaml
|
|
23
|
-
# - path: e2e/flows/chat/open.yaml # one flow file with its dashboard title (else its `name:` / opening comment)
|
|
24
|
-
# title: Open the chat
|
|
25
|
-
# description: Community screen -> chat sheet, first message visible
|
|
26
|
-
# command: bun run e2e # runner: command. Anything that exits 0 on success (Appium, Detox, ...)
|
|
27
|
-
|
|
28
|
-
steps:
|
|
29
|
-
rename: # stable names in history; keys are window/step names as they appear in metrics.csv
|
|
30
|
-
"screen:(app)/community/[handle]/(tabs)": community.open
|
|
31
|
-
pin: [community.open] # always listed in the summary, even without a change
|
|
32
|
-
|
|
33
|
-
output:
|
|
34
|
-
dir: perflab-out # runs.ndjson, metrics.csv, <run>/ logs, verdict (compare/verdict.json)
|
|
35
|
-
|
|
36
|
-
timeout_seconds: 600 # per run of the command; optional
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
# PerfLab step marker. Call with env STEP (name) and PHASE (start|end):
|
|
2
|
-
# - runFlow: { file: perflab/step.yaml, env: { STEP: chat.open, PHASE: start } }
|
|
3
|
-
# Never fails the surrounding flow, even when no PerfLab CLI is listening. On the farm it only records the marker:
|
|
4
|
-
# run send-markers.js at the end of each iteration (examples/farm/flows/perflab/end-run.yaml).
|
|
5
|
-
appId: any
|
|
6
|
-
---
|
|
7
|
-
- runScript:
|
|
8
|
-
file: step.js
|
|
9
|
-
env:
|
|
10
|
-
STEP: ${STEP}
|
|
11
|
-
PHASE: ${PHASE}
|