@keepsnappy/agent 0.1.0-alpha.11 → 0.1.0-alpha.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -50
- package/dist/appium-driver.js +136 -10
- package/dist/keepsnappy.js +87 -90
- package/examples/farm/flows/{perflab → keepsnappy}/begin-run.yaml +1 -1
- package/examples/farm/flows/{perflab → keepsnappy}/end-run.yaml +4 -4
- package/examples/farm/flows/keepsnappy/next-token.js +7 -0
- package/examples/farm/flows/{perflab/run-perflab.yaml → keepsnappy/run-keepsnappy.yaml} +1 -1
- package/examples/farm/flows/{perflab → keepsnappy}/send-markers.js +4 -5
- package/examples/farm/flows/keepsnappy/step-markers.yaml +13 -0
- package/examples/farm/flows/keepsnappy/tokens.js +2 -0
- package/examples/farm/workflows/{perflab-measure.yml → keepsnappy-measure.yml} +20 -20
- package/examples/farm/workflows/perf-device-run.fragment.yml +5 -5
- package/examples/github-actions.yml +18 -18
- package/examples/keepsnappy.yml +3 -3
- package/examples/maestro-steps/e2e-community-and-live-chat.yaml +10 -10
- package/maestro/{perflab → keepsnappy}/send-markers.js +4 -5
- package/maestro/{perflab → keepsnappy}/step.js +10 -11
- package/maestro/keepsnappy/step.yaml +11 -0
- package/package.json +2 -3
- package/examples/farm/flows/perflab/next-token.js +0 -7
- package/examples/farm/flows/perflab/step-markers.yaml +0 -13
- package/examples/farm/flows/perflab/tokens.js +0 -2
- package/examples/perflab.yml +0 -36
- package/maestro/perflab/step.yaml +0 -11
package/README.md
CHANGED
|
@@ -1,38 +1,38 @@
|
|
|
1
1
|
# @keepsnappy/agent
|
|
2
2
|
|
|
3
|
-
The `keepsnappy` host CLI. It runs the customer's own test command several times, collects the
|
|
3
|
+
The `keepsnappy` host CLI. It runs the customer's own test command several times, collects the keepsnappy data the app wrote during each run, turns it into metrics, and asks `@keepsnappy/stats` for a verdict. v1 is local only and measure-only: no code changes in the customer app, no driver of our own (the test command is Maestro, Appium, XCUITest, anything that exits 0 on success).
|
|
4
4
|
|
|
5
5
|
## Commands
|
|
6
6
|
|
|
7
7
|
```sh
|
|
8
8
|
bun install && bunx tsc --noEmit && bunx vitest run
|
|
9
|
-
bun run build # dist/src/cli.js (bin:
|
|
9
|
+
bun run build # dist/src/cli.js (bin: keepsnappy)
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
keepsnappy measure --repeat 10 \
|
|
12
12
|
--arms A=/builds/base.app,B=/builds/candidate.app \
|
|
13
13
|
--platform ios-sim [--device <udid>] [--bundle-id com.x.app] [--out runs/s1] [--timeout 600] \
|
|
14
14
|
[--ri-capture observations.ndjson] [--mem-window chat.open] \
|
|
15
15
|
-- maestro test flows/open-chat.yaml
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
keepsnappy verdict runs/s1 [--baseline A --candidate B] # table + verdict.json
|
|
17
|
+
keepsnappy extract runs/s1 --ri-capture observations.ndjson # re-derive metrics.csv (adds ri.* / mem.*)
|
|
18
|
+
keepsnappy aa runs/s1 [--reps 300 --boot 400] # noise floor / MDE per metric
|
|
19
|
+
keepsnappy doctor # xcrun, simctl, devicectl, maestro, appium, node, booted sims, devices
|
|
20
20
|
```
|
|
21
21
|
|
|
22
22
|
- `--arms`: `NAME=<.app/.ipa path>` (installed with `simctl install` / `devicectl device install app`, only when the arm changes) or `NAME=<bundle id>` (already installed). Without `--arms` there is one arm `A`: plain repetition, usable for `keepsnappy aa`.
|
|
23
|
-
- `--arm-env 'A:KEY=VAL'` (repeatable): extra env for the test command on that arm's runs, also exported as `MAESTRO_KEY` (Maestro forwards only `MAESTRO_*` to flows). Used for same-binary arms that differ only in knobs, e.g. `--arms A=dev.
|
|
23
|
+
- `--arm-env 'A:KEY=VAL'` (repeatable): extra env for the test command on that arm's runs, also exported as `MAESTRO_KEY` (Maestro forwards only `MAESTRO_*` to flows). Used for same-binary arms that differ only in knobs, e.g. `--arms A=dev.keepsnappy.reference,B=dev.keepsnappy.reference --arm-env 'A:REF_KNOBS=…' --arm-env 'B:REF_KNOBS=…'`. The arms with their env are written to `<out>/arms.json`. Order stays ABBA.
|
|
24
24
|
- `--platform ios-device` needs `--device <udid>`. The simulator default target is `booted`.
|
|
25
|
-
- The test command runs as argv (no shell) with `
|
|
26
|
-
- Declared steps (Maestro): call `maestro/
|
|
25
|
+
- The test command runs as argv (no shell) with `KEEPSNAPPY_RUN_ID`, `KEEPSNAPPY_ARM`, `KEEPSNAPPY_RUN_DIR`, `KEEPSNAPPY_MARKER_URL` and `MAESTRO_KEEPSNAPPY_MARKER_URL` / `MAESTRO_KEEPSNAPPY_RUN_ID` in its environment.
|
|
26
|
+
- Declared steps (Maestro): call `maestro/keepsnappy/step.yaml` with `STEP`/`PHASE` (start|end) from your flow; `measure` adds `step:<name>.*` rows (source=declared). `--no-markers` turns the endpoint off, `--clock-offset-ms` overrides the clock estimate. See docs/research/28-step-markers-prototype.md.
|
|
27
27
|
|
|
28
28
|
## `keepsnappy run` and `keepsnappy.yml` (customer entry point)
|
|
29
29
|
|
|
30
|
-
`keepsnappy run` is the wrapper for the e2e command you already have (mode A in docs/research/23-execution-modes-and-data-flow.md). It runs the command N times, drops runs where it failed, pulls the app's
|
|
30
|
+
`keepsnappy run` is the wrapper for the e2e command you already have (mode A in docs/research/23-execution-modes-and-data-flow.md). It runs the command N times, drops runs where it failed, pulls the app's keepsnappy data after each run, derives metrics and writes the out dir. With `--baseline <earlier out dir>` it also says what changed. Nothing is built or installed by it: your CI supplies the build (with `KEEPSNAPPY=1`) and the device.
|
|
31
31
|
|
|
32
32
|
```sh
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
33
|
+
keepsnappy run # command built from keepsnappy.yml
|
|
34
|
+
keepsnappy run -- maestro test --include-tags perf e2e/flows/ # wrap your own command (config still gives bundle id, device, repetitions)
|
|
35
|
+
keepsnappy run --baseline main-out --upload # compare with the baseline run, then send results to the ingest
|
|
36
36
|
```
|
|
37
37
|
|
|
38
38
|
Flags: `--config` (default `keepsnappy.yml`), `--repeat N`, `--out DIR`, `--baseline DIR`, `--upload [--ingest-url URL]`, `--wait`, `--fail-on-worse`, `--commit SHA` / `--branch NAME` (default `GITHUB_SHA` / `GITHUB_HEAD_REF` on a pull request, else `GITHUB_REF_NAME`), `--base-commit SHA`, `--device-model` / `--device-os` (default: read from the booted simulator), `--scenario NAME` (flow name for a command given after `--`), `--replay <out dir>[:ARM]` (repeatable: take recorded runs instead of running anything), `--no-markers`, `--clock-offset-ms`.
|
|
@@ -55,18 +55,18 @@ Annotated example: `examples/keepsnappy.yml`. Every setting:
|
|
|
55
55
|
|---|---|
|
|
56
56
|
| `project.key_env` | Name of the env var with the project key (default `KEEPSNAPPY_KEY`). Only read with `--upload`. |
|
|
57
57
|
| `project.ingest_url` | KeepSnappy API base URL for `--upload` (or `--ingest-url`, or env `KEEPSNAPPY_API`). Defaults to the hosted KeepSnappy API. |
|
|
58
|
-
| `app.bundle_id` | Required. App whose
|
|
58
|
+
| `app.bundle_id` | Required. App whose keepsnappy log is read after each run. |
|
|
59
59
|
| `platform`, `device` | `ios-sim` (default; `device` = UDID or `booted`) or `ios-device` (UDID required). One device per invocation in v1. |
|
|
60
60
|
| `repetitions` | Runs of the whole flow set (default 5). |
|
|
61
61
|
| `baseline_branch` | Branch the baseline comes from (default `main`). Recorded for CI; the CLI compares with the directory you pass to `--baseline`. |
|
|
62
62
|
| `flows.runner` | `maestro` (build `maestro test ...`) or `command` (run `flows.command`). |
|
|
63
63
|
| `flows.tags`, `flows.exclude_tags` | `--include-tags=` / `--exclude-tags=`. |
|
|
64
|
-
| `flows.paths` | Files, directories or globs (`*`, `?`, `**`); expanded by
|
|
64
|
+
| `flows.paths` | Files, directories or globs (`*`, `?`, `**`); expanded by keepsnappy (no shell). With only tags the path defaults to `.`. An entry can also be `{path, title, description}` (one flow file): how the dashboard names the journey. |
|
|
65
65
|
| `flows.command` | Any command that exits 0 on success (Appium, Detox, ...); text or list, run without a shell. |
|
|
66
66
|
| `flows.maestro_version` | Note only: printed at the start of the run. |
|
|
67
67
|
| `steps.rename` | `old: new` names applied to metrics (`old.dur_ms` becomes `new.dur_ms`, `step:old.*` too), so history keeps stable names. |
|
|
68
68
|
| `steps.pin` | Names always listed in the summary. |
|
|
69
|
-
| `output.dir` | Out dir (default `keepsnappy-out`): `runs.ndjson`, `metrics.csv`, `<run>/test.log`, `<run>/
|
|
69
|
+
| `output.dir` | Out dir (default `keepsnappy-out`): `runs.ndjson`, `metrics.csv`, `<run>/test.log`, `<run>/keepsnappy.log`, `<run>/markers.ndjson`, and with a baseline `compare/metrics.csv` + `compare/verdict.json`. |
|
|
70
70
|
| `timeout_seconds` | Per run of the command. |
|
|
71
71
|
|
|
72
72
|
Unknown settings and wrong types are errors, all listed together.
|
|
@@ -81,12 +81,12 @@ Top changes:
|
|
|
81
81
|
More work: 1 Less work: 0 No change: 38 Not enough data yet: 3
|
|
82
82
|
```
|
|
83
83
|
|
|
84
|
-
`--upload` (off by default) needs a project key with `run` scope in the env var named by `project.key_env` (default `KEEPSNAPPY_KEY`); the API URL defaults to the hosted KeepSnappy API. With a branch it opens a run set (`POST /v1/run-sets`) before measuring, so a wrong key fails before device time is spent; then sends each valid run: `POST /v1/runs` (header `x-
|
|
84
|
+
`--upload` (off by default) needs a project key with `run` scope in the env var named by `project.key_env` (default `KEEPSNAPPY_KEY`); the API URL defaults to the hosted KeepSnappy API. With a branch it opens a run set (`POST /v1/run-sets`) before measuring, so a wrong key fails before device time is spent; then sends each valid run: `POST /v1/runs` (header `x-keepsnappy-key`, `meta.scenario`, `meta.run_set_id`), `POST /v1/windows` with the run's own `KEEPSNAPPY window` log lines (run token, 200 per batch, deterministic batch ids so a retry is safe), `POST /v1/markers` with its declared step markers, `POST /v1/runs/:id/finish`. A 429 is retried after `retry-after`. Then it closes the run set:
|
|
85
85
|
|
|
86
86
|
- with `--wait`: `POST /v1/run-sets/:id/complete` computes the verdicts inline and the CLI prints them in plain words (More work / Less work / No change / Not enough data yet per journey and device, the top changed checks, "Few runs — result may be noisy" when the server flags it) plus a dashboard link that opens the comparison;
|
|
87
87
|
- without `--wait`: the set is closed with `verdicts: on-read` and the CLI exits 0 at once, printing the dashboard link; the verdicts are computed when someone looks.
|
|
88
88
|
|
|
89
|
-
On the baseline branch (`main` by default, set per app on the server) the runs grow each journey's history; on any other branch they are compared with it. Without a branch the runs are uploaded as before, without a run set and without a verdict. The CLI writes `<out>/
|
|
89
|
+
On the baseline branch (`main` by default, set per app on the server) the runs grow each journey's history; on any other branch they are compared with it. Without a branch the runs are uploaded as before, without a run set and without a verdict. The CLI writes `<out>/keepsnappy-result.json` (`run_set_id`, `dashboard_url`, `state`, summary) and `<out>/keepsnappy-run.json` (device, branch), and on GitHub Actions appends the result to `$GITHUB_STEP_SUMMARY`. Ready workflow: `examples/github-actions.yml`.
|
|
90
90
|
|
|
91
91
|
`keepsnappy verdict <out> --baseline remote [--branch main] [--limit N]` compares each flow's local runs with the server baseline of the same flow and device (`GET /v1/baselines`: the history of the baseline branch, inside the same window the dashboard uses), for the metrics the server judges.
|
|
92
92
|
|
|
@@ -97,12 +97,12 @@ app (SDK activated per iteration with a run token) pushes its windows to the API
|
|
|
97
97
|
markers there directly. Ready workflow and Maestro flows: `examples/farm/`.
|
|
98
98
|
|
|
99
99
|
```sh
|
|
100
|
-
|
|
100
|
+
keepsnappy farm open --api https://keepsnappy-api-….run.app --branch main --commit "$APP_COMMIT" \
|
|
101
101
|
--device-model 'iPhone 15 Pro' --device-os 17 --device-provider browserstack \
|
|
102
102
|
--journeys e2e-community-and-live-chat --repeat 5 \
|
|
103
|
-
--tokens-js farm-run/
|
|
104
|
-
# ... the farm session runs (browserstack-run.mjs --suite-overlay "flows/
|
|
105
|
-
|
|
103
|
+
--tokens-js farm-run/keepsnappy-tokens.js --state farm-run/keepsnappy-farm.json
|
|
104
|
+
# ... the farm session runs (browserstack-run.mjs --suite-overlay "flows/keepsnappy/tokens.js:farm-run/keepsnappy-tokens.js")
|
|
105
|
+
keepsnappy farm close --state farm-run/keepsnappy-farm.json --bs-build farm-run/browserstack.ios.json --settle-seconds 60 --wait
|
|
106
106
|
```
|
|
107
107
|
|
|
108
108
|
`keepsnappy farm run` does the whole night in one command (docs/plans/54a): resolve the app (`--app eas:<id>|file|bs://`), `farm open`, an optional BrowserStack Local tunnel (`--tunnel-for localhost:<port>`, repeatable), the device run (`--driver maestro` with `--suite-dir`/`--execute`, or `--driver appium --journey-module FILE` for the built-in Appium driver, `@keepsnappy/appium-driver`, docs/plans/54b; `--driver-script FILE` still runs your own driver), `farm close`, and cleanup (tunnel stopped, tokens file reset) whatever happened. `--env KEY=VALUE` values may use `{tunnel:<port>}`, `{run_id}`, `{out}`, `{tokens_js}`, `{run_set_id}`. Exit codes as `farm close`; a failed device run is 1. `--dry-run` prints the plan with no request. Needs `KEEPSNAPPY_KEY`, `BROWSERSTACK_USERNAME`, `BROWSERSTACK_ACCESS_KEY` (and `EXPO_TOKEN` for `eas:`).
|
|
@@ -117,14 +117,14 @@ it, not by what the phone reports) and one run per journey per iteration, ahead
|
|
|
117
117
|
runner. It writes:
|
|
118
118
|
|
|
119
119
|
- `--tokens-js`: the only file holding run tokens. BrowserStack caps env values at 100 characters, so the tokens travel
|
|
120
|
-
inside the uploaded suite; the dispatcher overlays this file at `e2e/flows/
|
|
120
|
+
inside the uploaded suite; the dispatcher overlays this file at `e2e/flows/keepsnappy/tokens.js` for one upload. ES5,
|
|
121
121
|
for Maestro's `runScript`:
|
|
122
122
|
```js
|
|
123
|
-
output.
|
|
124
|
-
output.
|
|
125
|
-
output.
|
|
123
|
+
output.keepsnappyApi = "https://keepsnappy-api-….run.app";
|
|
124
|
+
output.keepsnappyRunSet = "<run set id>";
|
|
125
|
+
output.keepsnappyTokens = {"<journey>": ["pl_rt_…", …one per iteration]};
|
|
126
126
|
```
|
|
127
|
-
`next-token.js` (examples) hands out `keepsnappyTokens[JOURNEY][i]` as `output.
|
|
127
|
+
`next-token.js` (examples) hands out `keepsnappyTokens[JOURNEY][i]` as `output.keepsnappyToken` for iteration i.
|
|
128
128
|
- `--state`: JSON for `farm close`, no token and no key (safe to upload as an artifact): `version`, `api`, `run_set_id`,
|
|
129
129
|
`dashboard_url`, `branch`, `commit`, `base_commit`, `device`, `journeys`, `repeat`, `tokens_js`, `opened_at`,
|
|
130
130
|
`expires_at`, `runs[]` (`order`, `journey`, `iteration`, `run_id`, `external_id`), then `closed_at` and `result`.
|
|
@@ -138,7 +138,7 @@ runner. It writes:
|
|
|
138
138
|
is never blamed; a Maestro failure stops the session, so later iterations have no data;
|
|
139
139
|
3. finishes every run not yet finished (a run without a single batch is reported as "no data", never judged), closes the
|
|
140
140
|
set (`--wait`: verdicts inline; else on read) and prints the iterations and the same plain-words verdict as
|
|
141
|
-
`keepsnappy run --upload --wait`, plus `$GITHUB_STEP_SUMMARY` and `<out>/
|
|
141
|
+
`keepsnappy run --upload --wait`, plus `$GITHUB_STEP_SUMMARY` and `<out>/keepsnappy-result.json` (`--out`, default the
|
|
142
142
|
state's directory);
|
|
143
143
|
4. puts the empty stub back into the tokens file, also when a step above failed. The example workflow also deletes the
|
|
144
144
|
file before uploading `$OUT_DIR`.
|
|
@@ -148,23 +148,23 @@ that arrives after a run was finished is added by the server and the run re-proc
|
|
|
148
148
|
result, `3` with `--fail-on-worse` on "More work", `2` for configuration, `1` when the API fails or nothing could be
|
|
149
149
|
judged ("Blocked": no iteration sent any data).
|
|
150
150
|
|
|
151
|
-
Slack: set `
|
|
151
|
+
Slack: set `KEEPSNAPPY_SLACK_WEBHOOK` (an incoming-webhook URL kept as a CI secret) and `farm close --wait` posts the
|
|
152
152
|
result: a pull request only when something got worse (a table of the 3 biggest changes, main vs PR), main only when
|
|
153
|
-
something changed (`
|
|
153
|
+
something changed (`KEEPSNAPPY_SLACK_ALWAYS=1` also posts "no change"), and a measurement that sent no data always. The
|
|
154
154
|
commit's title and author come from `git log` in the checkout. A Slack failure is printed and never changes the exit code.
|
|
155
155
|
|
|
156
|
-
Declared steps on the farm: `maestro/
|
|
156
|
+
Declared steps on the farm: `maestro/keepsnappy/step.yaml` / `step.js` / `send-markers.js` (copy them into the suite).
|
|
157
157
|
With a token set, `step.js` only records the marker; `send-markers.js`, run in `end-run.yaml` after each iteration,
|
|
158
|
-
posts them to `<output.
|
|
158
|
+
posts them to `<output.keepsnappyApi or KEEPSNAPPY_API_URL>/v1/markers` with `Authorization: Bearer <run token>`, so the test
|
|
159
159
|
never waits on the network between actions (and sends the host's clock round trips, plan 51 G1.4). A failed iteration
|
|
160
|
-
never reaches `end-run.yaml`, so the example `run-
|
|
160
|
+
never reaches `end-run.yaml`, so the example `run-keepsnappy.yaml` also runs `send-markers.js` from `onFlowComplete`; such a
|
|
161
161
|
run is invalid through its missing completion marker, and its markers stay for diagnosis. The example folder
|
|
162
|
-
(`examples/farm/flows/
|
|
162
|
+
(`examples/farm/flows/keepsnappy/`) ships its own copy of `send-markers.js`. Without a token, `step.js` posts to the local
|
|
163
163
|
CLI endpoint at once; there a step started but never ended invalidates its run (`unclosed-steps`).
|
|
164
164
|
|
|
165
|
-
Detox (Jest): `packages/detox` (`@keepsnappy/detox`) gives `step(name, fn)` / `stepStart` / `stepEnd`, posting the same markers: to the local endpoint from `
|
|
165
|
+
Detox (Jest): `packages/detox` (`@keepsnappy/detox`) gives `step(name, fn)` / `stepStart` / `stepEnd`, posting the same markers: to the local endpoint from `KEEPSNAPPY_MARKER_URL` + `KEEPSNAPPY_RUN_ID` (set by `keepsnappy run`/`measure`), or on the farm to `KEEPSNAPPY_API_URL/v1/markers` with `KEEPSNAPPY_RUN_TOKEN` as the bearer token. Optional per-test step names via `@keepsnappy/detox/auto-steps`. See its README.
|
|
166
166
|
|
|
167
|
-
Farm e2e against a local API: `KEEPSNAPPY_KEY=…
|
|
167
|
+
Farm e2e against a local API: `KEEPSNAPPY_KEY=… KEEPSNAPPY_INGEST_KEY=… e2e/farm/run.sh <new out dir>` (fake session using
|
|
168
168
|
the SDK push client, one late batch, one empty iteration).
|
|
169
169
|
|
|
170
170
|
## `keepsnappy findings` and friends (read-only questions)
|
|
@@ -177,16 +177,16 @@ same words (`packages/mcp/README.md`).
|
|
|
177
177
|
|
|
178
178
|
```sh
|
|
179
179
|
export KEEPSNAPPY_KEY=pl_rk_… # a project key with read scope (a CI key with run scope works too)
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
180
|
+
keepsnappy findings --branch feat/keepsnappy-integration # what got worse or better on a PR, worst first
|
|
181
|
+
keepsnappy findings --main --journey live-ui --phone "iPhone 15"
|
|
182
|
+
keepsnappy findings --state all --json # also the resolved and not judged ones, as the API's JSON
|
|
183
|
+
keepsnappy findings --state all --offset 200 # the next page (at most 200 per answer)
|
|
184
|
+
keepsnappy finding f_0123456789abcdef # one finding with its evidence
|
|
185
|
+
keepsnappy finding f_0123456789abcdef --prompt | pbcopy # only the hand-off for a coding agent, ending with its id and page
|
|
186
|
+
keepsnappy compare <runA> <runB> # two runs (ids or external ids), window by window, no statistics
|
|
187
|
+
keepsnappy history live-ui "Tap: live chat send" # one window on main, run by run (id or name)
|
|
188
|
+
keepsnappy commits a1b2c3d c3d4e5f [--branch main] # the measured commits between two points
|
|
189
|
+
keepsnappy layers # which layer answers which question, and its certificate
|
|
190
190
|
```
|
|
191
191
|
|
|
192
192
|
Flags: `--api URL` (or `KEEPSNAPPY_API`, default the hosted API), `--key-env NAME` (default `KEEPSNAPPY_KEY`), `--json`.
|
|
@@ -205,7 +205,7 @@ branch was measured differently from main.
|
|
|
205
205
|
<out>/arms.json arms of a `measure` (name, target, per-arm env)
|
|
206
206
|
<out>/runs.ndjson one record per run: run_id, arm, block, t0, t1, exit_code, valid, invalid_reason
|
|
207
207
|
<out>/<run_id>/test.log stdout/stderr of the test command
|
|
208
|
-
<out>/<run_id>/
|
|
208
|
+
<out>/<run_id>/keepsnappy.log Documents/keepsnappy/log.txt pulled from the app
|
|
209
209
|
<out>/metrics.csv long format: run_id,arm,block,metric,value,valid,invalid_reason
|
|
210
210
|
<out>/verdict.json written by verdict / aa
|
|
211
211
|
```
|
|
@@ -226,7 +226,7 @@ Pure core (`schedule`, `window-log`, `metrics`, table formatting) plus thin adap
|
|
|
226
226
|
|
|
227
227
|
## How measure-only works
|
|
228
228
|
|
|
229
|
-
1. The customer's build already contains the
|
|
229
|
+
1. The customer's build already contains the keepsnappy SDK, which appends `<wall ms> KEEPSNAPPY window {json}` lines to `Documents/keepsnappy/log.txt` for every window it observes (screen opens, interactions).
|
|
230
230
|
2. After each run the log is pulled and the windows whose start (`wall - durMs`) lies in `[t0, t1]` of that run are kept. Anything older is leftover from earlier runs.
|
|
231
231
|
3. Per window name, the run's value is the median over its cycles of: `dur_ms`, `commits`, `renders`, `mounts`, `longtask_ms`, `gc_count`, `gc_ms`, `alloc_mb`, `touch_max_ms`; plus `n`, `timeouts` (windows ending in `timeout` are excluded from medians) and `retained_mb_per_cycle` (Theil-Sen slope of heap after GC across cycles, needs 3+ cycles). Metric names are `<window>.<field>`, kept as-is (e.g. `screen:(app)/community/[handle]/(tabs).dur_ms`) and quoted in the CSV when needed.
|
|
232
232
|
4. A failed test, an install or log-pull failure, or a run with no windows is written as invalid with a reason. Invalid runs stay in `metrics.csv` (a `run.status` row when they have no metrics) and the stats engine drops them whole.
|
|
@@ -258,4 +258,4 @@ Memory slopes (Theil-Sen over the cycle index, 3+ cycles of `--mem-window`, defa
|
|
|
258
258
|
`bun run typecheck` and `bun run test` first build the workspace packages the CLI imports (`bun run deps`: stats, derive,
|
|
259
259
|
verdicts, profile-analysis, mcp), so both work from a clean checkout.
|
|
260
260
|
|
|
261
|
-
|
|
261
|
+
Only `keepsnappy` and `keepsnappy.yml` / `keepsnappy.yaml` are supported. Configure `KEEPSNAPPY_*` variables. Explicit `--key-env`, `project.key_env`, and `--config` remain arbitrary user-supplied names and paths.
|
package/dist/appium-driver.js
CHANGED
|
@@ -17,7 +17,7 @@ import { basename } from "node:path";
|
|
|
17
17
|
var ALERT_POLL_MS = 250;
|
|
18
18
|
var RUN_DEFAULTS = { ttl: 3600, profile: 1, diag: 1 };
|
|
19
19
|
var END_GC_DEFAULT = true;
|
|
20
|
-
var route = (scheme) => `${scheme}devtools/
|
|
20
|
+
var route = (scheme) => `${scheme}devtools/keepsnappy`;
|
|
21
21
|
function parseSdkArgs(args) {
|
|
22
22
|
return args.split("&").filter(Boolean).map((kv) => {
|
|
23
23
|
const i = kv.indexOf("=");
|
|
@@ -448,9 +448,12 @@ class Driver {
|
|
|
448
448
|
}
|
|
449
449
|
async tapElement(el) {
|
|
450
450
|
const r = this.rects.get(el) ?? await this.cmd("GET", `/element/${el}/rect`);
|
|
451
|
+
await this.tapPoint({ x: r.x + r.width / 2, y: r.y + r.height / 2 });
|
|
452
|
+
}
|
|
453
|
+
async tapPoint(p) {
|
|
451
454
|
const w = await this.screen();
|
|
452
|
-
const x = Math.round(Math.min(Math.max(
|
|
453
|
-
const y = Math.round(Math.min(Math.max(
|
|
455
|
+
const x = Math.round(Math.min(Math.max(p.x, 1), w.width - 1));
|
|
456
|
+
const y = Math.round(Math.min(Math.max(p.y, 1), w.height - 1));
|
|
454
457
|
await this.cmd("POST", "/actions", {
|
|
455
458
|
actions: [
|
|
456
459
|
{
|
|
@@ -503,8 +506,8 @@ class Driver {
|
|
|
503
506
|
// packages/appium-driver/src/platform.ts
|
|
504
507
|
var modelOf = (device) => device.replace(/-[^-]*$/, "");
|
|
505
508
|
var versionOf = (device) => device.replace(/^.*-/, "");
|
|
506
|
-
var FLUSH_ID = "
|
|
507
|
-
var ACK_ID = "
|
|
509
|
+
var FLUSH_ID = "keepsnappy.flushed";
|
|
510
|
+
var ACK_ID = "keepsnappy.ack";
|
|
508
511
|
var GC_ANSWER_WAIT_MS = 1e4;
|
|
509
512
|
var GC_POLL_MS = 100;
|
|
510
513
|
function parseBroadcastAnswer(out, n) {
|
|
@@ -605,7 +608,7 @@ var ANDROID = {
|
|
|
605
608
|
uploadFields: {},
|
|
606
609
|
async gcCommand(driver, n) {
|
|
607
610
|
const out = await driver.execute("mobile: broadcast", {
|
|
608
|
-
action: `${driver.bundleId}.
|
|
611
|
+
action: `${driver.bundleId}.keepsnappy.COMMAND`,
|
|
609
612
|
package: driver.bundleId,
|
|
610
613
|
extras: [["s", "cmd", "gc"], ["i", "n", String(n)]],
|
|
611
614
|
flags: "0x10000000"
|
|
@@ -653,7 +656,7 @@ function sessionCaps(i) {
|
|
|
653
656
|
...(i.platform ?? IOS).caps(i.device),
|
|
654
657
|
"appium:app": i.appUrl,
|
|
655
658
|
"bstack:options": {
|
|
656
|
-
projectName: "
|
|
659
|
+
projectName: "keepsnappy",
|
|
657
660
|
buildName: i.buildName,
|
|
658
661
|
sessionName: i.sessionName,
|
|
659
662
|
...tunnel,
|
|
@@ -674,6 +677,13 @@ var MARKERS_PER_REQUEST = 500;
|
|
|
674
677
|
var MARKER_TRIES = 3;
|
|
675
678
|
var HOST_CLOCK_TRIPS = 6;
|
|
676
679
|
var RETRY_BASE_MS = 1000;
|
|
680
|
+
var FLING_MS = 80;
|
|
681
|
+
var FLING_FROM = 0.8;
|
|
682
|
+
var FLING_TO = 0.3;
|
|
683
|
+
var FLING_TIMEOUT_MS = 30000;
|
|
684
|
+
var STALE_REFINDS = 3;
|
|
685
|
+
var FRAME_POLL_MS = 50;
|
|
686
|
+
var sameRect = (a, b) => a.x === b.x && a.y === b.y && a.width === b.width && a.height === b.height;
|
|
677
687
|
var RETRY_AFTER_CAP_MS = 30000;
|
|
678
688
|
async function post(fetchImpl, url, token, body, now) {
|
|
679
689
|
const t0 = now();
|
|
@@ -780,6 +790,10 @@ class Session {
|
|
|
780
790
|
endGc;
|
|
781
791
|
gcN = 0;
|
|
782
792
|
stepDepth = 0;
|
|
793
|
+
refinds = 0;
|
|
794
|
+
get staleRefinds() {
|
|
795
|
+
return this.refinds;
|
|
796
|
+
}
|
|
783
797
|
constructor(o) {
|
|
784
798
|
this.driver = o.driver;
|
|
785
799
|
this.scheme = o.scheme;
|
|
@@ -992,6 +1006,118 @@ class Session {
|
|
|
992
1006
|
await this.swipe([50, 50], direction === "down" ? [50, 10] : [50, 90], durationMs);
|
|
993
1007
|
}
|
|
994
1008
|
}
|
|
1009
|
+
async frameOf(sel, el) {
|
|
1010
|
+
for (let refinds = 0;; refinds++) {
|
|
1011
|
+
try {
|
|
1012
|
+
return { el, rect: await this.driver.cmd("GET", `/element/${el}/rect`) };
|
|
1013
|
+
} catch (e) {
|
|
1014
|
+
if (!(e instanceof WdError && e.wdError === "stale element reference"))
|
|
1015
|
+
throw e;
|
|
1016
|
+
const what = describeSelector(toSelector(sel));
|
|
1017
|
+
if (refinds >= STALE_REFINDS)
|
|
1018
|
+
throw new Error(`${what} went stale ${refinds + 1} times in a row (${refinds} re-finds): ${e.message}`);
|
|
1019
|
+
this.refinds++;
|
|
1020
|
+
try {
|
|
1021
|
+
el = await this.waitVisible(sel);
|
|
1022
|
+
} catch (lost) {
|
|
1023
|
+
throw new Error(`${what} went stale and was not found again: ${lost instanceof Error ? lost.message : String(lost)}`);
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
async settledFrame(sel, timeoutMs) {
|
|
1029
|
+
const start = this.now();
|
|
1030
|
+
let at = await this.frameOf(sel, await this.waitVisible(sel));
|
|
1031
|
+
for (;; ) {
|
|
1032
|
+
await this.driver.sleep(FRAME_POLL_MS);
|
|
1033
|
+
const now = await this.frameOf(sel, at.el);
|
|
1034
|
+
if (sameRect(at.rect, now.rect))
|
|
1035
|
+
return now;
|
|
1036
|
+
if (this.now() - start > timeoutMs)
|
|
1037
|
+
throw new Error(`${describeSelector(toSelector(sel))} did not stop moving within ${timeoutMs} ms`);
|
|
1038
|
+
at = now;
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
async flingIn(r, toward, durationMs) {
|
|
1042
|
+
const [from, to] = toward === "end" ? [FLING_FROM, FLING_TO] : [FLING_TO, FLING_FROM];
|
|
1043
|
+
const x = Math.round(r.x + r.width / 2);
|
|
1044
|
+
await this.driver.drag({ x, y: Math.round(r.y + r.height * from) }, { x, y: Math.round(r.y + r.height * to) }, durationMs);
|
|
1045
|
+
}
|
|
1046
|
+
async flingTimes(list, times, toward, { durationMs = FLING_MS, timeoutMs = FLING_TIMEOUT_MS } = {}) {
|
|
1047
|
+
if (!Number.isInteger(times) || times < 1)
|
|
1048
|
+
throw new Error(`flingTimes needs a whole number of flings >= 1, got ${times}`);
|
|
1049
|
+
await this.ready();
|
|
1050
|
+
const start = this.now();
|
|
1051
|
+
const refindsBefore = this.refinds;
|
|
1052
|
+
let flings = 0;
|
|
1053
|
+
let status = 1;
|
|
1054
|
+
try {
|
|
1055
|
+
const { rect } = await this.settledFrame(list, timeoutMs);
|
|
1056
|
+
for (;flings < times; flings++)
|
|
1057
|
+
await this.flingIn(rect, toward, durationMs);
|
|
1058
|
+
status = 0;
|
|
1059
|
+
} finally {
|
|
1060
|
+
this.driver.log.push({ ...this.driver.context, cmd: `fling-${toward}-times`, t: this.now(), ms: this.now() - start, status, flings, refinds: this.refinds - refindsBefore });
|
|
1061
|
+
}
|
|
1062
|
+
}
|
|
1063
|
+
async flingUntil(list, target, toward, { blind = 0, durationMs = FLING_MS, timeoutMs = FLING_TIMEOUT_MS } = {}) {
|
|
1064
|
+
await this.ready();
|
|
1065
|
+
const start = this.now();
|
|
1066
|
+
const refindsBefore = this.refinds;
|
|
1067
|
+
let flings = 0;
|
|
1068
|
+
let checks = 0;
|
|
1069
|
+
let status = 1;
|
|
1070
|
+
try {
|
|
1071
|
+
let at = await this.settledFrame(list, timeoutMs);
|
|
1072
|
+
const fling = async () => {
|
|
1073
|
+
if (flings > 0)
|
|
1074
|
+
at = await this.frameOf(list, at.el);
|
|
1075
|
+
await this.flingIn(at.rect, toward, durationMs);
|
|
1076
|
+
flings++;
|
|
1077
|
+
};
|
|
1078
|
+
while (flings < blind)
|
|
1079
|
+
await fling();
|
|
1080
|
+
const sel = toSelector(target);
|
|
1081
|
+
while (checks++, !await this.driver.findShown(sel, true)) {
|
|
1082
|
+
if (this.now() - start > timeoutMs)
|
|
1083
|
+
throw new Error(`the ${toward} of the list did not show within ${timeoutMs} ms (${flings} flings)`);
|
|
1084
|
+
await fling();
|
|
1085
|
+
}
|
|
1086
|
+
status = 0;
|
|
1087
|
+
return flings;
|
|
1088
|
+
} finally {
|
|
1089
|
+
this.driver.log.push({ ...this.driver.context, cmd: `fling-to-${toward}`, t: this.now(), ms: this.now() - start, status, flings, checks, blind, refinds: this.refinds - refindsBefore });
|
|
1090
|
+
}
|
|
1091
|
+
}
|
|
1092
|
+
async stableCentres(sels, timeoutMs = FLING_TIMEOUT_MS) {
|
|
1093
|
+
await this.ready();
|
|
1094
|
+
const start = this.now();
|
|
1095
|
+
const els = [];
|
|
1096
|
+
for (const sel of sels)
|
|
1097
|
+
els.push(await this.waitVisible(sel));
|
|
1098
|
+
const frames = async () => {
|
|
1099
|
+
const out = [];
|
|
1100
|
+
for (let i = 0;i < sels.length; i++) {
|
|
1101
|
+
const f = await this.frameOf(sels[i], els[i]);
|
|
1102
|
+
els[i] = f.el;
|
|
1103
|
+
out.push(f.rect);
|
|
1104
|
+
}
|
|
1105
|
+
return out;
|
|
1106
|
+
};
|
|
1107
|
+
for (let before = await frames();; ) {
|
|
1108
|
+
await this.driver.sleep(FRAME_POLL_MS);
|
|
1109
|
+
const now = await frames();
|
|
1110
|
+
if (now.every((r, i) => sameRect(r, before[i])))
|
|
1111
|
+
return now.map((r) => ({ x: Math.round(r.x + r.width / 2), y: Math.round(r.y + r.height / 2) }));
|
|
1112
|
+
if (this.now() - start > timeoutMs)
|
|
1113
|
+
throw new Error(`the elements did not stop moving within ${timeoutMs} ms`);
|
|
1114
|
+
before = now;
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
1117
|
+
async tapAt(point) {
|
|
1118
|
+
await this.ready();
|
|
1119
|
+
await this.driver.tapPoint(point);
|
|
1120
|
+
}
|
|
995
1121
|
async type(sel, value) {
|
|
996
1122
|
await this.ready();
|
|
997
1123
|
const el = await this.waitVisible(sel);
|
|
@@ -1052,7 +1178,7 @@ function parseTokensJs(text) {
|
|
|
1052
1178
|
const m = text.match(new RegExp(`output\\.${name}\\s*=\\s*(.+);\\s*$`, "m"));
|
|
1053
1179
|
return m ? JSON.parse(m[1]) : undefined;
|
|
1054
1180
|
};
|
|
1055
|
-
return { api: pick("
|
|
1181
|
+
return { api: pick("keepsnappyApi") ?? "", tokens: pick("keepsnappyTokens") ?? {} };
|
|
1056
1182
|
}
|
|
1057
1183
|
function plan(journeys, repeat, tokens) {
|
|
1058
1184
|
const out = [];
|
|
@@ -1081,7 +1207,7 @@ var DRIVER_OPTIONS = {
|
|
|
1081
1207
|
repeat: { type: "string", default: "1" },
|
|
1082
1208
|
device: { type: "string" },
|
|
1083
1209
|
"local-identifier": { type: "string" },
|
|
1084
|
-
"build-name": { type: "string", default: "
|
|
1210
|
+
"build-name": { type: "string", default: "keepsnappy-appium" },
|
|
1085
1211
|
"appium-version": { type: "string", default: "2.19.0" },
|
|
1086
1212
|
out: { type: "string", default: "farm-run" },
|
|
1087
1213
|
platform: { type: "string", default: "ios" },
|
|
@@ -1125,7 +1251,7 @@ async function runDriver(argv, module, deps = realRunDeps()) {
|
|
|
1125
1251
|
const log = [];
|
|
1126
1252
|
const driver = new Driver({ hub: HUB, log, headers: { Authorization: auth }, bundleId: module.app.bundleId, platform, fetch: deps.fetch, now: deps.now, sleep: deps.sleep });
|
|
1127
1253
|
const t = new Session({ driver, scheme: module.app.scheme, fetch: deps.fetch, api, actionGc: a["action-gc"] === "on", endGc: a["end-gc"] === "on" });
|
|
1128
|
-
const sessionName = module.sessionName ?? "
|
|
1254
|
+
const sessionName = module.sessionName ?? "keepsnappy (appium)";
|
|
1129
1255
|
const device = String(a.device);
|
|
1130
1256
|
const ctx = { fetch: deps.fetch, tokens, journeys, repeat, args: a };
|
|
1131
1257
|
const caps = sessionCaps({ appUrl, device, buildName: String(a["build-name"]), sessionName, appiumVersion: String(a["appium-version"]), platform, localIdentifier: a["local-identifier"] });
|