@keepsnappy/agent 0.1.0-alpha.13 → 0.1.0-alpha.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -16
- package/dist/appium-driver.js +328 -78
- package/dist/keepsnappy.js +386 -861
- package/examples/farm/flows/keepsnappy/end-run.yaml +4 -3
- package/examples/keepsnappy.yml +1 -3
- package/package.json +1 -1
package/dist/keepsnappy.js
CHANGED
|
@@ -1,32 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
// apps/agent/src/compat.ts
|
|
4
|
-
import { resolve } from "node:path";
|
|
5
|
-
function envValue(env, name) {
|
|
6
|
-
return env[`KEEPSNAPPY_${name}`];
|
|
7
|
-
}
|
|
8
|
-
function projectKey(env, name = "KEEPSNAPPY_KEY") {
|
|
9
|
-
return name === "KEEPSNAPPY_KEY" ? envValue(env, "KEY") : env[name];
|
|
10
|
-
}
|
|
11
|
-
var CONFIG_NAMES = ["keepsnappy.yml", "keepsnappy.yaml"];
|
|
12
|
-
async function readProjectConfig(read, explicit) {
|
|
13
|
-
if (explicit !== undefined)
|
|
14
|
-
return read(resolve(explicit));
|
|
15
|
-
for (const name of CONFIG_NAMES) {
|
|
16
|
-
try {
|
|
17
|
-
return await read(resolve(name));
|
|
18
|
-
} catch (error) {
|
|
19
|
-
if (error.code !== "ENOENT")
|
|
20
|
-
throw error;
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
return read(resolve(CONFIG_NAMES[0]));
|
|
24
|
-
}
|
|
25
|
-
|
|
26
3
|
// apps/agent/src/cli.ts
|
|
27
4
|
import { resolve as resolve7 } from "node:path";
|
|
28
5
|
import { parseArgs as parseArgs4 } from "node:util";
|
|
29
|
-
import { readFile as readFile5 } from "node:fs/promises";
|
|
30
6
|
|
|
31
7
|
// apps/agent/src/config.ts
|
|
32
8
|
import { readdir } from "node:fs/promises";
|
|
@@ -238,7 +214,7 @@ class ConfigError extends Error {
|
|
|
238
214
|
}
|
|
239
215
|
var TITLE_MAX = 200;
|
|
240
216
|
var DESCRIPTION_MAX = 2000;
|
|
241
|
-
var TOP = ["project", "app", "platform", "device", "repetitions", "
|
|
217
|
+
var TOP = ["project", "app", "platform", "device", "repetitions", "flows", "steps", "output", "timeout_seconds"];
|
|
242
218
|
var isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
243
219
|
function splitCommand(s) {
|
|
244
220
|
const out = [];
|
|
@@ -341,7 +317,6 @@ function validateConfig(raw, opts = {}) {
|
|
|
341
317
|
else
|
|
342
318
|
repetitions = raw.repetitions;
|
|
343
319
|
}
|
|
344
|
-
const baselineBranch = str("baseline_branch", raw.baseline_branch, false, "main");
|
|
345
320
|
const flows = section("flows", ["runner", "maestro_version", "tags", "exclude_tags", "paths", "command"], !given);
|
|
346
321
|
const runner = str("flows.runner", flows.runner, false, "maestro");
|
|
347
322
|
if (runner !== "maestro" && runner !== "command")
|
|
@@ -398,7 +373,7 @@ function validateConfig(raw, opts = {}) {
|
|
|
398
373
|
if (runner === "command" && (tags.length || excludeTags.length || paths.length))
|
|
399
374
|
p.push("flows.tags/exclude_tags/paths only apply to runner: maestro");
|
|
400
375
|
const maestroVersion = flows.maestro_version === undefined ? undefined : String(flows.maestro_version);
|
|
401
|
-
const steps = section("steps", ["rename"
|
|
376
|
+
const steps = section("steps", ["rename"], false);
|
|
402
377
|
const rename = {};
|
|
403
378
|
if (steps.rename !== undefined && steps.rename !== null) {
|
|
404
379
|
if (!isObj(steps.rename))
|
|
@@ -411,7 +386,6 @@ function validateConfig(raw, opts = {}) {
|
|
|
411
386
|
rename[k] = v;
|
|
412
387
|
}
|
|
413
388
|
}
|
|
414
|
-
const pin = strList("steps.pin", steps.pin);
|
|
415
389
|
const output = section("output", ["dir"], false);
|
|
416
390
|
const outDir = str("output.dir", output.dir, false, "keepsnappy-out");
|
|
417
391
|
let timeoutSeconds;
|
|
@@ -429,9 +403,8 @@ function validateConfig(raw, opts = {}) {
|
|
|
429
403
|
platform,
|
|
430
404
|
device,
|
|
431
405
|
repetitions,
|
|
432
|
-
baselineBranch,
|
|
433
406
|
flows: { runner, ...maestroVersion ? { maestroVersion } : {}, tags, excludeTags, paths, info, command },
|
|
434
|
-
steps: { rename
|
|
407
|
+
steps: { rename },
|
|
435
408
|
outDir,
|
|
436
409
|
...timeoutSeconds ? { timeoutSeconds } : {}
|
|
437
410
|
};
|
|
@@ -513,9 +486,32 @@ async function expandGlob(pattern, listDir) {
|
|
|
513
486
|
return [...new Set(out)].sort();
|
|
514
487
|
}
|
|
515
488
|
|
|
489
|
+
// apps/agent/src/compat.ts
|
|
490
|
+
import { resolve } from "node:path";
|
|
491
|
+
function envValue(env, name) {
|
|
492
|
+
return env[`KEEPSNAPPY_${name}`];
|
|
493
|
+
}
|
|
494
|
+
function projectKey(env, name = "KEEPSNAPPY_KEY") {
|
|
495
|
+
return name === "KEEPSNAPPY_KEY" ? envValue(env, "KEY") : env[name];
|
|
496
|
+
}
|
|
497
|
+
var CONFIG_NAMES = ["keepsnappy.yml", "keepsnappy.yaml"];
|
|
498
|
+
async function readProjectConfig(read, explicit) {
|
|
499
|
+
if (explicit !== undefined)
|
|
500
|
+
return read(resolve(explicit));
|
|
501
|
+
for (const name of CONFIG_NAMES) {
|
|
502
|
+
try {
|
|
503
|
+
return await read(resolve(name));
|
|
504
|
+
} catch (error) {
|
|
505
|
+
if (error.code !== "ENOENT")
|
|
506
|
+
throw error;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return read(resolve(CONFIG_NAMES[0]));
|
|
510
|
+
}
|
|
511
|
+
|
|
516
512
|
// apps/agent/src/run-command.ts
|
|
517
513
|
import { readFile } from "node:fs/promises";
|
|
518
|
-
import { join as
|
|
514
|
+
import { join as join9, resolve as resolve3 } from "node:path";
|
|
519
515
|
|
|
520
516
|
// apps/agent/src/data-source.ts
|
|
521
517
|
import { join as join2 } from "node:path";
|
|
@@ -2053,6 +2049,13 @@ var NOT_FACTS = {
|
|
|
2053
2049
|
decimals: 0,
|
|
2054
2050
|
what: "Whether the test cleaned up the app’s memory once at the end of the journey, outside any measured window, before the end reading"
|
|
2055
2051
|
},
|
|
2052
|
+
"sdk.end_read": {
|
|
2053
|
+
unit: "flag",
|
|
2054
|
+
label: "End memory read taken on request after the clean-up (1 = yes)",
|
|
2055
|
+
short: "end memory read on request",
|
|
2056
|
+
decimals: 0,
|
|
2057
|
+
what: "Whether the end memory reading is the one the app took when the test asked for it right after the clean-up (1), or the clean-up’s own reading (0, an app or run without the request)"
|
|
2058
|
+
},
|
|
2056
2059
|
"sdk.appstate_changes": {
|
|
2057
2060
|
unit: "count",
|
|
2058
2061
|
label: "App went to the background or came back",
|
|
@@ -2092,7 +2095,23 @@ var NOT_FACTS = {
|
|
|
2092
2095
|
};
|
|
2093
2096
|
var isNotFactId = (id) => Object.hasOwn(NOT_FACTS, id);
|
|
2094
2097
|
var fieldDef = (id) => isMeasureId(id) ? MEASURES[id] : isNotFactId(id) ? NOT_FACTS[id] : undefined;
|
|
2098
|
+
var DIMENSIONS = {
|
|
2099
|
+
component: { label: "Component", rankBy: "self_ms" },
|
|
2100
|
+
updater: { label: "Update scheduled by", rankBy: "share_of_commits" },
|
|
2101
|
+
slow_commit_component: { label: "Component in the slowest commit", rankBy: "self_ms" },
|
|
2102
|
+
slow_commit_updater: { label: "Update of the slowest commit scheduled by", rankBy: "scheduled" }
|
|
2103
|
+
};
|
|
2104
|
+
var BREAKDOWN_MEASURES = {
|
|
2105
|
+
self_ms: { unit: "ms", worse: "higher", label: "render time", decimals: 1 },
|
|
2106
|
+
total_ms: { unit: "ms", worse: "higher", label: "total render time", decimals: 1 },
|
|
2107
|
+
renders: { unit: "count", worse: "higher", label: "renders", decimals: 0 },
|
|
2108
|
+
mounts: { unit: "count", worse: "higher", label: "mounts", decimals: 0 },
|
|
2109
|
+
share_of_commits: { unit: "share", worse: "none", label: "share of commits", decimals: 0 },
|
|
2110
|
+
scheduled: { unit: "count", worse: "none", label: "updates scheduled", decimals: 0 }
|
|
2111
|
+
};
|
|
2095
2112
|
var measureDef = (id) => isMeasureId(id) ? MEASURES[id] : undefined;
|
|
2113
|
+
var HEAT_METRIC = "sampler.thermal.heats_up";
|
|
2114
|
+
var HEAT_LABEL = MEASURES[HEAT_METRIC].label;
|
|
2096
2115
|
// packages/facts/dist/src/places.js
|
|
2097
2116
|
var AFTER_3S = "+3s";
|
|
2098
2117
|
var kindOf = (stem) => {
|
|
@@ -2432,32 +2451,6 @@ function mannWhitney(a, b, exactLimit = MWU_EXACT_LIMIT) {
|
|
|
2432
2451
|
const z = Math.max((Math.abs(u - mu) - 0.5) / Math.sqrt(v), 0);
|
|
2433
2452
|
return { u, p: Math.min(1, 2 * (1 - normCdf(z))), cles, exact: false };
|
|
2434
2453
|
}
|
|
2435
|
-
function hodgesLehmann(a, b) {
|
|
2436
|
-
const d = [];
|
|
2437
|
-
for (const bj of b)
|
|
2438
|
-
for (const ai of a)
|
|
2439
|
-
d.push(bj - ai);
|
|
2440
|
-
return median(d);
|
|
2441
|
-
}
|
|
2442
|
-
// packages/stats/dist/src/inference/wilcoxon.js
|
|
2443
|
-
function wilcoxonSignedRank(d) {
|
|
2444
|
-
const nz = d.filter((v) => v !== 0);
|
|
2445
|
-
const n = nz.length;
|
|
2446
|
-
if (n === 0)
|
|
2447
|
-
return 1;
|
|
2448
|
-
const { ranks, tieTerm } = averageRanks(nz.map(Math.abs));
|
|
2449
|
-
let w = 0;
|
|
2450
|
-
nz.forEach((v, i) => {
|
|
2451
|
-
if (v > 0)
|
|
2452
|
-
w += ranks[i];
|
|
2453
|
-
});
|
|
2454
|
-
const mu = n * (n + 1) / 4;
|
|
2455
|
-
const v = n * (n + 1) * (2 * n + 1) / 24 - tieTerm / 48;
|
|
2456
|
-
if (v <= 0)
|
|
2457
|
-
return 1;
|
|
2458
|
-
const z = Math.max((Math.abs(w - mu) - 0.5) / Math.sqrt(v), 0);
|
|
2459
|
-
return Math.min(1, 2 * (1 - normCdf(z)));
|
|
2460
|
-
}
|
|
2461
2454
|
// packages/stats/dist/src/inference/bootstrap.js
|
|
2462
2455
|
var NAN_INTERVAL = { lo: Number.NaN, hi: Number.NaN };
|
|
2463
2456
|
function locationInPlace(scratch, kind) {
|
|
@@ -2472,88 +2465,6 @@ function locationInPlace(scratch, kind) {
|
|
|
2472
2465
|
function location(x, kind) {
|
|
2473
2466
|
return kind === "count" ? mean(x) : locationInPlace(Float64Array.from(x), kind);
|
|
2474
2467
|
}
|
|
2475
|
-
function resampleLocation(x, kind, rng, buf) {
|
|
2476
|
-
for (let i = 0;i < buf.length; i++)
|
|
2477
|
-
buf[i] = x[rng.int(x.length)];
|
|
2478
|
-
return locationInPlace(buf, kind);
|
|
2479
|
-
}
|
|
2480
|
-
function bootLogRatio(a, b, kind, nBoot, rng) {
|
|
2481
|
-
const logRatio = new Float64Array(nBoot);
|
|
2482
|
-
const diff = new Float64Array(nBoot);
|
|
2483
|
-
const ba = new Float64Array(a.length);
|
|
2484
|
-
const bb = new Float64Array(b.length);
|
|
2485
|
-
for (let r = 0;r < nBoot; r++) {
|
|
2486
|
-
const la = resampleLocation(a, kind, rng, ba);
|
|
2487
|
-
const lb = resampleLocation(b, kind, rng, bb);
|
|
2488
|
-
logRatio[r] = Math.log(lb) - Math.log(la);
|
|
2489
|
-
diff[r] = lb - la;
|
|
2490
|
-
}
|
|
2491
|
-
return { logRatio, diff };
|
|
2492
|
-
}
|
|
2493
|
-
function bootPairedLogRatio(blockLogRatios, nBoot, rng) {
|
|
2494
|
-
const out = new Float64Array(nBoot);
|
|
2495
|
-
const buf = new Float64Array(blockLogRatios.length);
|
|
2496
|
-
for (let r = 0;r < nBoot; r++)
|
|
2497
|
-
out[r] = resampleLocation(blockLogRatios, "continuous", rng, buf);
|
|
2498
|
-
return out;
|
|
2499
|
-
}
|
|
2500
|
-
function percentileCi(draws, level) {
|
|
2501
|
-
const lo = (1 - level) / 2;
|
|
2502
|
-
const [a, b] = finiteQuantiles(draws, [lo, 1 - lo]);
|
|
2503
|
-
return { lo: a ?? Number.NaN, hi: b ?? Number.NaN };
|
|
2504
|
-
}
|
|
2505
|
-
function bcaCi(draws, thetaHat, jackknife, level) {
|
|
2506
|
-
const d = [];
|
|
2507
|
-
for (let i = 0;i < draws.length; i++) {
|
|
2508
|
-
const v = draws[i];
|
|
2509
|
-
if (Number.isFinite(v))
|
|
2510
|
-
d.push(v);
|
|
2511
|
-
}
|
|
2512
|
-
if (d.length === 0 || !Number.isFinite(thetaHat))
|
|
2513
|
-
return NAN_INTERVAL;
|
|
2514
|
-
let below = 0;
|
|
2515
|
-
let equal = 0;
|
|
2516
|
-
for (const v of d) {
|
|
2517
|
-
if (v < thetaHat)
|
|
2518
|
-
below++;
|
|
2519
|
-
else if (v === thetaHat)
|
|
2520
|
-
equal++;
|
|
2521
|
-
}
|
|
2522
|
-
let prop = (below + 0.5 * equal) / d.length;
|
|
2523
|
-
prop = Math.min(Math.max(prop, 1 / (d.length + 1)), 1 - 1 / (d.length + 1));
|
|
2524
|
-
const z0 = normPpf(prop);
|
|
2525
|
-
const jack = jackknife.filter(Number.isFinite);
|
|
2526
|
-
const jm = mean(jack);
|
|
2527
|
-
let num = 0;
|
|
2528
|
-
let sq = 0;
|
|
2529
|
-
for (const v of jack) {
|
|
2530
|
-
num += (jm - v) ** 3;
|
|
2531
|
-
sq += (jm - v) ** 2;
|
|
2532
|
-
}
|
|
2533
|
-
const den = 6 * sq ** 1.5;
|
|
2534
|
-
const acc = den > 0 ? num / den : 0;
|
|
2535
|
-
const qs = [(1 - level) / 2, 1 - (1 - level) / 2].map((q) => {
|
|
2536
|
-
const zq = normPpf(q);
|
|
2537
|
-
const adj = normCdf(z0 + (z0 + zq) / (1 - acc * (z0 + zq)));
|
|
2538
|
-
return Math.min(Math.max(adj, 0), 1);
|
|
2539
|
-
});
|
|
2540
|
-
const [lo, hi] = finiteQuantiles(d, qs);
|
|
2541
|
-
return { lo: lo ?? Number.NaN, hi: hi ?? Number.NaN };
|
|
2542
|
-
}
|
|
2543
|
-
function jackknifeLogRatio(a, b, kind) {
|
|
2544
|
-
const out = [];
|
|
2545
|
-
const la = location(a, kind);
|
|
2546
|
-
const lb = location(b, kind);
|
|
2547
|
-
for (let i = 0;i < a.length; i++) {
|
|
2548
|
-
const l = location(a.filter((_, j) => j !== i), kind);
|
|
2549
|
-
out.push(l > 0 && lb > 0 ? Math.log(lb / l) : Number.NaN);
|
|
2550
|
-
}
|
|
2551
|
-
for (let i = 0;i < b.length; i++) {
|
|
2552
|
-
const l = location(b.filter((_, j) => j !== i), kind);
|
|
2553
|
-
out.push(la > 0 && l > 0 ? Math.log(l / la) : Number.NaN);
|
|
2554
|
-
}
|
|
2555
|
-
return out;
|
|
2556
|
-
}
|
|
2557
2468
|
// packages/stats/dist/src/inference/equivalence.js
|
|
2558
2469
|
var relativeMargin = (value) => ({ kind: "relative", value });
|
|
2559
2470
|
var absoluteMargin = (value) => ({ kind: "absolute", value });
|
|
@@ -2563,78 +2474,6 @@ function equivalenceBounds(margin) {
|
|
|
2563
2474
|
function isEquivalent(ci, bounds) {
|
|
2564
2475
|
return Number.isFinite(ci.lo) && Number.isFinite(ci.hi) && ci.lo > bounds.lo && ci.hi < bounds.hi;
|
|
2565
2476
|
}
|
|
2566
|
-
function excludesZero(ci) {
|
|
2567
|
-
return ci.lo > 0 || ci.hi < 0;
|
|
2568
|
-
}
|
|
2569
|
-
// packages/stats/dist/src/inference/fisher.js
|
|
2570
|
-
function logFactorials(n) {
|
|
2571
|
-
const out = new Float64Array(n + 1);
|
|
2572
|
-
for (let i = 2;i <= n; i++)
|
|
2573
|
-
out[i] = out[i - 1] + Math.log(i);
|
|
2574
|
-
return out;
|
|
2575
|
-
}
|
|
2576
|
-
function fisherExact(t) {
|
|
2577
|
-
const r1 = t.a + t.b;
|
|
2578
|
-
const r2 = t.c + t.d;
|
|
2579
|
-
const c1 = t.a + t.c;
|
|
2580
|
-
const n = r1 + r2;
|
|
2581
|
-
if (r1 === 0 || r2 === 0 || c1 === 0 || c1 === n)
|
|
2582
|
-
return 1;
|
|
2583
|
-
const lf = logFactorials(n);
|
|
2584
|
-
const f = (k) => lf[k];
|
|
2585
|
-
const fixed = f(r1) + f(r2) + f(c1) + f(n - c1) - f(n);
|
|
2586
|
-
const logP = (x) => fixed - f(x) - f(r1 - x) - f(c1 - x) - f(r2 - c1 + x);
|
|
2587
|
-
const observed = logP(t.a);
|
|
2588
|
-
let p = 0;
|
|
2589
|
-
for (let x = Math.max(0, c1 - r2);x <= Math.min(r1, c1); x++) {
|
|
2590
|
-
const lp = logP(x);
|
|
2591
|
-
if (lp <= observed + 0.0000001)
|
|
2592
|
-
p += Math.exp(lp);
|
|
2593
|
-
}
|
|
2594
|
-
return Math.min(1, p);
|
|
2595
|
-
}
|
|
2596
|
-
// packages/stats/dist/src/inference/multiplicity.js
|
|
2597
|
-
function order(p) {
|
|
2598
|
-
return p.map((_, i) => i).sort((i, j) => p[i] - p[j] || i - j);
|
|
2599
|
-
}
|
|
2600
|
-
function bonferroni(p) {
|
|
2601
|
-
return p.map((v) => Math.min(1, v * p.length));
|
|
2602
|
-
}
|
|
2603
|
-
function holm(p) {
|
|
2604
|
-
const m = p.length;
|
|
2605
|
-
const out = new Array(m).fill(1);
|
|
2606
|
-
let running = 0;
|
|
2607
|
-
order(p).forEach((idx, rank) => {
|
|
2608
|
-
running = Math.max(running, Math.min(1, (m - rank) * p[idx]));
|
|
2609
|
-
out[idx] = running;
|
|
2610
|
-
});
|
|
2611
|
-
return out;
|
|
2612
|
-
}
|
|
2613
|
-
function benjaminiHochberg(p) {
|
|
2614
|
-
const m = p.length;
|
|
2615
|
-
const out = new Array(m).fill(1);
|
|
2616
|
-
const ord = order(p);
|
|
2617
|
-
let running = 1;
|
|
2618
|
-
for (let rank = m - 1;rank >= 0; rank--) {
|
|
2619
|
-
const idx = ord[rank];
|
|
2620
|
-
running = Math.min(running, Math.min(1, p[idx] * m / (rank + 1)));
|
|
2621
|
-
out[idx] = running;
|
|
2622
|
-
}
|
|
2623
|
-
return out;
|
|
2624
|
-
}
|
|
2625
|
-
function adjustPValues(p, method) {
|
|
2626
|
-
const finite = p.map((v) => Number.isFinite(v) ? v : 1);
|
|
2627
|
-
switch (method) {
|
|
2628
|
-
case "none":
|
|
2629
|
-
return [...p];
|
|
2630
|
-
case "bonferroni":
|
|
2631
|
-
return bonferroni(finite);
|
|
2632
|
-
case "holm":
|
|
2633
|
-
return holm(finite);
|
|
2634
|
-
case "bh":
|
|
2635
|
-
return benjaminiHochberg(finite);
|
|
2636
|
-
}
|
|
2637
|
-
}
|
|
2638
2477
|
// packages/stats/dist/src/modes/config.js
|
|
2639
2478
|
var DEFAULT_MODE_CONFIG = {
|
|
2640
2479
|
keys: ["mounts", "renders", "commits"],
|
|
@@ -2666,9 +2505,6 @@ var DEFAULT_CHANGE_CONFIG = {
|
|
|
2666
2505
|
seed: 12345,
|
|
2667
2506
|
modes: DEFAULT_MODE_CONFIG
|
|
2668
2507
|
};
|
|
2669
|
-
function withChangeDefaults(partial = {}) {
|
|
2670
|
-
return { ...DEFAULT_CHANGE_CONFIG, ...partial };
|
|
2671
|
-
}
|
|
2672
2508
|
// packages/stats/dist/src/change/decide.js
|
|
2673
2509
|
function decide(ci95, ci90, p, alpha, bounds, higherIsBetter) {
|
|
2674
2510
|
if (isEquivalent(ci90, bounds))
|
|
@@ -2680,51 +2516,6 @@ function decide(ci95, ci90, p, alpha, bounds, higherIsBetter) {
|
|
|
2680
2516
|
return higherIsBetter ? "worse" : "better";
|
|
2681
2517
|
return "inconclusive";
|
|
2682
2518
|
}
|
|
2683
|
-
// packages/stats/dist/src/change/paired.js
|
|
2684
|
-
function groupByBlock(values, blocks) {
|
|
2685
|
-
const out = new Map;
|
|
2686
|
-
values.forEach((v, i) => {
|
|
2687
|
-
const k = blocks[i];
|
|
2688
|
-
const g = out.get(k);
|
|
2689
|
-
if (g)
|
|
2690
|
-
g.push(v);
|
|
2691
|
-
else
|
|
2692
|
-
out.set(k, [v]);
|
|
2693
|
-
});
|
|
2694
|
-
return out;
|
|
2695
|
-
}
|
|
2696
|
-
function pairedAnalysis(a, aBlocks, b, bBlocks, kind, alpha, nBoot, minBlocks, rng) {
|
|
2697
|
-
const ga = groupByBlock(a, aBlocks);
|
|
2698
|
-
const gb = groupByBlock(b, bBlocks);
|
|
2699
|
-
const common = [...ga.keys()].filter((k) => gb.has(k)).sort();
|
|
2700
|
-
if (common.length < minBlocks)
|
|
2701
|
-
return null;
|
|
2702
|
-
const lr = [];
|
|
2703
|
-
const df = [];
|
|
2704
|
-
for (const k of common) {
|
|
2705
|
-
const la = location(ga.get(k), kind);
|
|
2706
|
-
const lb = location(gb.get(k), kind);
|
|
2707
|
-
if (la > 0 && lb > 0) {
|
|
2708
|
-
lr.push(Math.log(lb / la));
|
|
2709
|
-
df.push(lb - la);
|
|
2710
|
-
}
|
|
2711
|
-
}
|
|
2712
|
-
if (lr.length < minBlocks)
|
|
2713
|
-
return null;
|
|
2714
|
-
const lrDraws = bootPairedLogRatio(lr, nBoot, rng);
|
|
2715
|
-
const dfDraws = bootPairedLogRatio(df, nBoot, rng);
|
|
2716
|
-
return {
|
|
2717
|
-
blocks: lr.length,
|
|
2718
|
-
logRatio: median(lr),
|
|
2719
|
-
logRatioCi95: percentileCi(lrDraws, 1 - alpha),
|
|
2720
|
-
logRatioCi90: percentileCi(lrDraws, 1 - 2 * alpha),
|
|
2721
|
-
diff: median(df),
|
|
2722
|
-
diffCi95: percentileCi(dfDraws, 1 - alpha),
|
|
2723
|
-
diffCi90: percentileCi(dfDraws, 1 - 2 * alpha),
|
|
2724
|
-
p: wilcoxonSignedRank(lr)
|
|
2725
|
-
};
|
|
2726
|
-
}
|
|
2727
|
-
|
|
2728
2519
|
// packages/stats/dist/src/change/compare-metric.js
|
|
2729
2520
|
function effectiveMargin(a, o) {
|
|
2730
2521
|
const floor = o.countFloor;
|
|
@@ -2733,89 +2524,6 @@ function effectiveMargin(a, o) {
|
|
|
2733
2524
|
const units = o.margin.value * Math.abs(location(a, "count"));
|
|
2734
2525
|
return units < floor ? absoluteMargin(floor) : o.margin;
|
|
2735
2526
|
}
|
|
2736
|
-
var expInterval = (ci) => ({
|
|
2737
|
-
lo: Number.isFinite(ci.lo) ? Math.exp(ci.lo) : Number.NaN,
|
|
2738
|
-
hi: Number.isFinite(ci.hi) ? Math.exp(ci.hi) : Number.NaN
|
|
2739
|
-
});
|
|
2740
|
-
function summarize(x, kind) {
|
|
2741
|
-
const loc = location(x, kind);
|
|
2742
|
-
const base = { n: x.length, location: loc, mad: mad(x), robustCv: robustCv(x) };
|
|
2743
|
-
if (kind !== "count")
|
|
2744
|
-
return base;
|
|
2745
|
-
return { ...base, dispersion: x.length > 1 && loc > 0 ? variance(x, 1) / loc : Number.NaN };
|
|
2746
|
-
}
|
|
2747
|
-
function unpairedEstimates(a, b, o, pMwu, rng) {
|
|
2748
|
-
const la = location(a, o.kind);
|
|
2749
|
-
const lb = location(b, o.kind);
|
|
2750
|
-
const draws = bootLogRatio(a, b, o.kind, o.nBoot, rng);
|
|
2751
|
-
const logRatio = la > 0 && lb > 0 ? Math.log(lb / la) : Number.NaN;
|
|
2752
|
-
let lr95;
|
|
2753
|
-
let lr90;
|
|
2754
|
-
if (o.ci === "bca") {
|
|
2755
|
-
const jack = jackknifeLogRatio(a, b, o.kind);
|
|
2756
|
-
lr95 = bcaCi(draws.logRatio, logRatio, jack, 1 - o.alpha);
|
|
2757
|
-
lr90 = bcaCi(draws.logRatio, logRatio, jack, 1 - 2 * o.alpha);
|
|
2758
|
-
} else {
|
|
2759
|
-
lr95 = percentileCi(draws.logRatio, 1 - o.alpha);
|
|
2760
|
-
lr90 = percentileCi(draws.logRatio, 1 - 2 * o.alpha);
|
|
2761
|
-
}
|
|
2762
|
-
return {
|
|
2763
|
-
logRatio,
|
|
2764
|
-
lr95,
|
|
2765
|
-
lr90,
|
|
2766
|
-
diff: lb - la,
|
|
2767
|
-
diff95: percentileCi(draws.diff, 1 - o.alpha),
|
|
2768
|
-
diff90: percentileCi(draws.diff, 1 - 2 * o.alpha),
|
|
2769
|
-
p: pMwu,
|
|
2770
|
-
paired: null
|
|
2771
|
-
};
|
|
2772
|
-
}
|
|
2773
|
-
function compareMetric(a, b, o, rng, blocks) {
|
|
2774
|
-
const margin = effectiveMargin(a, o);
|
|
2775
|
-
const mw = mannWhitney(a, b);
|
|
2776
|
-
const pairedRes = o.paired && blocks ? pairedAnalysis(a, blocks.a, b, blocks.b, o.kind, o.alpha, o.nBoot, o.minPairedBlocks, rng) : null;
|
|
2777
|
-
const est = pairedRes ? {
|
|
2778
|
-
logRatio: pairedRes.logRatio,
|
|
2779
|
-
lr95: pairedRes.logRatioCi95,
|
|
2780
|
-
lr90: pairedRes.logRatioCi90,
|
|
2781
|
-
diff: pairedRes.diff,
|
|
2782
|
-
diff95: pairedRes.diffCi95,
|
|
2783
|
-
diff90: pairedRes.diffCi90,
|
|
2784
|
-
p: pairedRes.p,
|
|
2785
|
-
paired: { blocks: pairedRes.blocks, p: pairedRes.p }
|
|
2786
|
-
} : unpairedEstimates(a, b, o, mw.p, rng);
|
|
2787
|
-
const absolute = margin.kind === "absolute";
|
|
2788
|
-
const ci95 = absolute ? est.diff95 : est.lr95;
|
|
2789
|
-
const ci90 = absolute ? est.diff90 : est.lr90;
|
|
2790
|
-
let verdict = decide(ci95, ci90, est.p, o.alpha, equivalenceBounds(margin), o.higherIsBetter);
|
|
2791
|
-
let note;
|
|
2792
|
-
if (verdict === "no-change" && est.p < o.alpha && excludesZero(ci95)) {
|
|
2793
|
-
note = `detectable shift, but inside the ${absolute ? `+/-${margin.value}` : `+/-${Math.round(margin.value * 100)}%`} margin`;
|
|
2794
|
-
}
|
|
2795
|
-
if (Math.min(a.length, b.length) < o.minN) {
|
|
2796
|
-
verdict = "inconclusive";
|
|
2797
|
-
note = `n < ${o.minN}`;
|
|
2798
|
-
}
|
|
2799
|
-
return {
|
|
2800
|
-
kind: o.kind,
|
|
2801
|
-
baseline: summarize(a, o.kind),
|
|
2802
|
-
candidate: summarize(b, o.kind),
|
|
2803
|
-
mannWhitney: { u: mw.u, p: mw.p, cles: mw.cles },
|
|
2804
|
-
hodgesLehmann: hodgesLehmann(a, b),
|
|
2805
|
-
paired: est.paired !== null,
|
|
2806
|
-
...est.paired ? { blocks: est.paired.blocks, pPaired: est.paired.p } : {},
|
|
2807
|
-
p: est.p,
|
|
2808
|
-
ratio: Number.isFinite(est.logRatio) ? Math.exp(est.logRatio) : Number.NaN,
|
|
2809
|
-
ratioCi95: expInterval(est.lr95),
|
|
2810
|
-
ratioCi90: expInterval(est.lr90),
|
|
2811
|
-
diff: est.diff,
|
|
2812
|
-
diffCi95: est.diff95,
|
|
2813
|
-
diffCi90: est.diff90,
|
|
2814
|
-
margin,
|
|
2815
|
-
verdict,
|
|
2816
|
-
...note ? { note } : {}
|
|
2817
|
-
};
|
|
2818
|
-
}
|
|
2819
2527
|
// packages/stats/dist/src/modes/detect.js
|
|
2820
2528
|
function group(x) {
|
|
2821
2529
|
return { n: x.length, median: median(x), min: x[0], max: x[x.length - 1], spread: 1.4826 * mad(x) };
|
|
@@ -2860,22 +2568,19 @@ function splitTwoModes(values, cfg) {
|
|
|
2860
2568
|
return null;
|
|
2861
2569
|
return { threshold: (low.max + high.min) / 2, low, high, gap, spread, separation: gap / spread };
|
|
2862
2570
|
}
|
|
2863
|
-
|
|
2571
|
+
// packages/stats/dist/src/modes/bimodal.js
|
|
2572
|
+
var BIMODAL_CONFIG = withModeDefaults({
|
|
2573
|
+
minModeRuns: 3,
|
|
2574
|
+
minModeShare: 0.15,
|
|
2575
|
+
gapFactor: 3,
|
|
2576
|
+
minRelativeGap: 0.25
|
|
2577
|
+
});
|
|
2864
2578
|
// packages/stats/dist/src/modes/windows.js
|
|
2865
2579
|
var MODE_LABELS = ["low", "high"];
|
|
2866
2580
|
function splitMetricName(metric) {
|
|
2867
2581
|
const i = metric.lastIndexOf(".");
|
|
2868
2582
|
return i < 0 ? { window: metric, measure: "" } : { window: metric.slice(0, i), measure: metric.slice(i + 1) };
|
|
2869
2583
|
}
|
|
2870
|
-
function countByArm(obs, labels) {
|
|
2871
|
-
const out = { low: 0, high: 0 };
|
|
2872
|
-
for (const o of obs ?? []) {
|
|
2873
|
-
const l = labels[o.runId];
|
|
2874
|
-
if (l)
|
|
2875
|
-
out[l] += 1;
|
|
2876
|
-
}
|
|
2877
|
-
return out;
|
|
2878
|
-
}
|
|
2879
2584
|
function splitWindows(dataset, arms, cfg) {
|
|
2880
2585
|
const keyed = new Map;
|
|
2881
2586
|
for (const metric of dataset.metrics.keys()) {
|
|
@@ -2905,126 +2610,6 @@ function splitWindows(dataset, arms, cfg) {
|
|
|
2905
2610
|
}
|
|
2906
2611
|
return out;
|
|
2907
2612
|
}
|
|
2908
|
-
function detectWindowModes(dataset, baseline, candidate, cfg, alpha) {
|
|
2909
|
-
const out = new Map;
|
|
2910
|
-
for (const [window, ws] of splitWindows(dataset, [baseline, candidate], cfg)) {
|
|
2911
|
-
const arms = dataset.metrics.get(`${window}.${ws.key}`);
|
|
2912
|
-
const a = countByArm(arms?.get(baseline), ws.labels);
|
|
2913
|
-
const b = countByArm(arms?.get(candidate), ws.labels);
|
|
2914
|
-
const p = fisherExact({ a: a.low, b: a.high, c: b.low, d: b.high });
|
|
2915
|
-
const mixed = MODE_LABELS.some((l) => a[l] > 0 && b[l] > 0);
|
|
2916
|
-
out.set(window, { ...ws, mix: { baseline: a, candidate: b, p, shifted: p < alpha }, confounded: !mixed });
|
|
2917
|
-
}
|
|
2918
|
-
return out;
|
|
2919
|
-
}
|
|
2920
|
-
|
|
2921
|
-
// packages/stats/dist/src/change/compare-modes.js
|
|
2922
|
-
function combine(modes, confounded, pooled) {
|
|
2923
|
-
if (confounded)
|
|
2924
|
-
return pooled;
|
|
2925
|
-
const tested = modes.filter((m) => m.result);
|
|
2926
|
-
if (tested.some((m) => m.verdict === "worse"))
|
|
2927
|
-
return "worse";
|
|
2928
|
-
if (tested.some((m) => m.verdict === "better"))
|
|
2929
|
-
return "better";
|
|
2930
|
-
if (tested.length === modes.length && tested.every((m) => m.verdict === "no-change"))
|
|
2931
|
-
return "no-change";
|
|
2932
|
-
return "inconclusive";
|
|
2933
|
-
}
|
|
2934
|
-
function explain(m, verdict) {
|
|
2935
|
-
const counts = (r) => `${r.low}/${r.high}`;
|
|
2936
|
-
const mix = `mix low/high ${counts(m.mix.baseline)} vs ${counts(m.mix.candidate)}, Fisher p=${m.mix.p.toPrecision(3)}${m.mix.shifted ? " (mix shifted)" : ""}`;
|
|
2937
|
-
if (m.confounded) {
|
|
2938
|
-
return `modes on ${m.key} coincide with the arms (${mix}); pooled verdict ${verdict.toUpperCase()} kept, but a state difference between arms looks the same`;
|
|
2939
|
-
}
|
|
2940
|
-
const parts = m.modes.map((x) => `${x.label}: ${x.verdict}${x.reason ? ` (${x.reason})` : ""}`);
|
|
2941
|
-
return `state modes on ${m.key}: ${parts.join(", ")}; ${mix}; pooled ${m.pooledVerdict}`;
|
|
2942
|
-
}
|
|
2943
|
-
function compareByMode(metric, A, B, pooled, wm, options, cfg, seed, modeMargins) {
|
|
2944
|
-
const rng = createRng(labelSeed(seed, metric));
|
|
2945
|
-
const pick = (obs, l) => obs.filter((o) => wm.labels[o.runId] === l);
|
|
2946
|
-
const modes = MODE_LABELS.map((label) => {
|
|
2947
|
-
const a = pick(A, label);
|
|
2948
|
-
const b = pick(B, label);
|
|
2949
|
-
const base = { label, keyMedian: wm.split[label].median, nBaseline: a.length, nCandidate: b.length };
|
|
2950
|
-
if (Math.min(a.length, b.length) < cfg.minRunsPerArm) {
|
|
2951
|
-
return { ...base, verdict: "inconclusive", reason: `n ${a.length} vs ${b.length}, needs ${cfg.minRunsPerArm} per arm` };
|
|
2952
|
-
}
|
|
2953
|
-
const own = modeMargins?.[label];
|
|
2954
|
-
const result = compareMetric(a.map((o) => o.value), b.map((o) => o.value), {
|
|
2955
|
-
...options,
|
|
2956
|
-
...own?.margin ? { margin: own.margin } : {},
|
|
2957
|
-
countFloor: own?.countFloor ?? (options.countFloor || DEFAULT_COUNT_FLOOR),
|
|
2958
|
-
paired: false,
|
|
2959
|
-
minN: cfg.minRunsPerArm
|
|
2960
|
-
}, rng);
|
|
2961
|
-
return { ...base, result, verdict: result.verdict, ...result.note ? { reason: result.note } : {} };
|
|
2962
|
-
});
|
|
2963
|
-
const unlabeled = [...A, ...B].filter((o) => !(o.runId in wm.labels)).length;
|
|
2964
|
-
const partial = { window: wm.window, key: wm.key, modes, mix: wm.mix, confounded: wm.confounded, unlabeled, pooledVerdict: pooled.verdict };
|
|
2965
|
-
const verdict = combine(modes, wm.confounded, pooled.verdict);
|
|
2966
|
-
return { ...partial, verdict, why: explain(partial, verdict) };
|
|
2967
|
-
}
|
|
2968
|
-
|
|
2969
|
-
// packages/stats/dist/src/change/compare-runs.js
|
|
2970
|
-
function resolveKind(spec, values, autoCount) {
|
|
2971
|
-
if (spec?.kind)
|
|
2972
|
-
return spec.kind;
|
|
2973
|
-
return autoCount && isCountMetric(values) ? "count" : "continuous";
|
|
2974
|
-
}
|
|
2975
|
-
function compareRuns(input) {
|
|
2976
|
-
const config = withChangeDefaults(input.config);
|
|
2977
|
-
const rng = createRng(config.seed);
|
|
2978
|
-
const windowModes = config.modes ? detectWindowModes(input.dataset, input.baseline, input.candidate, config.modes, config.alpha) : new Map;
|
|
2979
|
-
const names = [...input.dataset.metrics.keys()].sort();
|
|
2980
|
-
const raw = names.map((metric) => {
|
|
2981
|
-
const arms = input.dataset.metrics.get(metric);
|
|
2982
|
-
const spec = input.metrics?.[metric];
|
|
2983
|
-
const higherIsBetter = spec?.higherIsBetter ?? false;
|
|
2984
|
-
const A = arms?.get(input.baseline);
|
|
2985
|
-
const B = arms?.get(input.candidate);
|
|
2986
|
-
if (!A || !B) {
|
|
2987
|
-
return { metric, higherIsBetter, skipped: `arms present: ${[...arms?.keys() ?? []].sort().join(",")}` };
|
|
2988
|
-
}
|
|
2989
|
-
const a = A.map((o) => o.value);
|
|
2990
|
-
const b = B.map((o) => o.value);
|
|
2991
|
-
const options = {
|
|
2992
|
-
kind: resolveKind(spec, [...a, ...b], config.autoCount),
|
|
2993
|
-
higherIsBetter,
|
|
2994
|
-
margin: spec?.margin ?? config.margin,
|
|
2995
|
-
alpha: config.alpha,
|
|
2996
|
-
nBoot: config.nBoot,
|
|
2997
|
-
ci: config.ci,
|
|
2998
|
-
paired: config.paired,
|
|
2999
|
-
minPairedBlocks: config.minPairedBlocks,
|
|
3000
|
-
minN: config.minN,
|
|
3001
|
-
countFloor: spec?.countFloor ?? config.countFloor
|
|
3002
|
-
};
|
|
3003
|
-
const result = compareMetric(a, b, options, rng, { a: A.map((o) => o.block), b: B.map((o) => o.block) });
|
|
3004
|
-
const wm = windowModes.get(splitMetricName(metric).window);
|
|
3005
|
-
if (!wm || !config.modes)
|
|
3006
|
-
return { metric, higherIsBetter, result };
|
|
3007
|
-
const modes = compareByMode(metric, A, B, result, wm, options, config.modes, config.seed, spec?.modeMargins);
|
|
3008
|
-
return { metric, higherIsBetter, result: { ...result, verdict: modes.verdict, note: modes.why }, modes };
|
|
3009
|
-
});
|
|
3010
|
-
const tested = raw.filter((c) => c.result);
|
|
3011
|
-
const adj = adjustPValues(tested.map((c) => c.result?.p ?? Number.NaN), config.multiplicity);
|
|
3012
|
-
const adjBy = new Map(tested.map((c, i) => [c.metric, adj[i]]));
|
|
3013
|
-
return {
|
|
3014
|
-
config,
|
|
3015
|
-
nRuns: input.dataset.nRuns,
|
|
3016
|
-
nInvalidRuns: input.dataset.invalidRuns.length,
|
|
3017
|
-
comparisons: raw.map((c) => adjBy.has(c.metric) ? { ...c, pAdjusted: adjBy.get(c.metric) } : c),
|
|
3018
|
-
windowModes: [...windowModes.values()].sort((x, y) => x.window.localeCompare(y.window))
|
|
3019
|
-
};
|
|
3020
|
-
}
|
|
3021
|
-
// packages/stats/dist/src/modes/bimodal.js
|
|
3022
|
-
var BIMODAL_CONFIG = withModeDefaults({
|
|
3023
|
-
minModeRuns: 3,
|
|
3024
|
-
minModeShare: 0.15,
|
|
3025
|
-
gapFactor: 3,
|
|
3026
|
-
minRelativeGap: 0.25
|
|
3027
|
-
});
|
|
3028
2613
|
// packages/stats/dist/src/calibration/gate.js
|
|
3029
2614
|
class MwuTables {
|
|
3030
2615
|
tables = new Map;
|
|
@@ -3169,6 +2754,11 @@ function suggestMargin(pool, kind, policy = DEFAULT_MARGIN_POLICY) {
|
|
|
3169
2754
|
}
|
|
3170
2755
|
|
|
3171
2756
|
// packages/stats/dist/src/calibration/aa-runs.js
|
|
2757
|
+
function resolveKind(spec, values, autoCount) {
|
|
2758
|
+
if (spec?.kind)
|
|
2759
|
+
return spec.kind;
|
|
2760
|
+
return autoCount && isCountMetric(values) ? "count" : "continuous";
|
|
2761
|
+
}
|
|
3172
2762
|
function calibrateModes(metric, obs, kind, ws, config, policy) {
|
|
3173
2763
|
const modes = MODE_LABELS.map((label) => {
|
|
3174
2764
|
const pool = obs.filter((o) => ws.labels[o.runId] === label).map((o) => o.value);
|
|
@@ -3397,6 +2987,8 @@ var readRunDataset = (csvText) => buildRunDataset(parseRunRows(csvText));
|
|
|
3397
2987
|
var obj2 = (v) => typeof v === "object" && v !== null ? v : {};
|
|
3398
2988
|
var num = (v) => typeof v === "number" && Number.isFinite(v) ? v : undefined;
|
|
3399
2989
|
function inputWall(w) {
|
|
2990
|
+
if (w.touchWall !== undefined)
|
|
2991
|
+
return w.touchWall;
|
|
3400
2992
|
const touch = num(obj2(w.raw.interaction).touchStartT);
|
|
3401
2993
|
const t1 = num(w.raw.t1);
|
|
3402
2994
|
return touch === undefined || t1 === undefined ? undefined : w.endWall - (t1 - touch);
|
|
@@ -3494,17 +3086,21 @@ function clusters(windows) {
|
|
|
3494
3086
|
}
|
|
3495
3087
|
return out;
|
|
3496
3088
|
}
|
|
3497
|
-
function onAppClock(windows) {
|
|
3089
|
+
function onAppClock(windows, version) {
|
|
3498
3090
|
const order = windows.map((w, i) => ({ w, i })).sort((a, b) => a.w.endWall - b.w.endWall || a.i - b.i);
|
|
3499
3091
|
const segment = new Map;
|
|
3500
3092
|
let seg = 0;
|
|
3501
3093
|
let prevId = -Infinity;
|
|
3094
|
+
let seen = new Set;
|
|
3502
3095
|
for (const { w } of order) {
|
|
3503
3096
|
const id = num(obj2(w.raw).id);
|
|
3504
3097
|
if (id === undefined)
|
|
3505
3098
|
continue;
|
|
3506
|
-
if (id <= prevId)
|
|
3099
|
+
if (version >= 23 ? seen.has(id) : id <= prevId) {
|
|
3507
3100
|
seg += 1;
|
|
3101
|
+
seen = new Set;
|
|
3102
|
+
}
|
|
3103
|
+
seen.add(id);
|
|
3508
3104
|
prevId = id;
|
|
3509
3105
|
segment.set(w, seg);
|
|
3510
3106
|
}
|
|
@@ -3524,7 +3120,8 @@ function onAppClock(windows) {
|
|
|
3524
3120
|
const t1 = num(w.raw.t1);
|
|
3525
3121
|
if (o === undefined || t0 === undefined || t1 === undefined)
|
|
3526
3122
|
return w;
|
|
3527
|
-
|
|
3123
|
+
const touch = version >= 23 ? num(obj2(w.raw.interaction).touchStartT) : undefined;
|
|
3124
|
+
return { ...w, startWall: t0 + o, endWall: t1 + o, ...touch === undefined ? {} : { touchWall: touch + o } };
|
|
3528
3125
|
});
|
|
3529
3126
|
}
|
|
3530
3127
|
|
|
@@ -3906,7 +3503,9 @@ function screenWindowModelMetrics(windows) {
|
|
|
3906
3503
|
var t0Of = (w) => num3(w.raw.t0);
|
|
3907
3504
|
var t1Of = (w) => num3(w.raw.t1);
|
|
3908
3505
|
var NEXT_ACTION_ENDS = ["next-press", "touch-lost", "gc-command"];
|
|
3909
|
-
function actionsOf(windows, steps = []) {
|
|
3506
|
+
function actionsOf(windows, steps = [], version) {
|
|
3507
|
+
const onClock = version !== undefined && version >= 23 ? onAppClock(windows, version) : undefined;
|
|
3508
|
+
const placed = onClock && new Map(windows.map((w, i) => [w, onClock[i]]));
|
|
3910
3509
|
const inputs = windows.filter((w) => isInputWindow(w) && t0Of(w) !== undefined).sort((a, b) => t0Of(a) - t0Of(b));
|
|
3911
3510
|
const journeyEnd = Math.max(Number.NEGATIVE_INFINITY, ...windows.map((w) => t1Of(w) ?? Number.NEGATIVE_INFINITY));
|
|
3912
3511
|
const twins = new Map;
|
|
@@ -3928,9 +3527,10 @@ function actionsOf(windows, steps = []) {
|
|
|
3928
3527
|
let endReason = next ? ACTION_END.nextAction : ACTION_END.journeyEnd;
|
|
3929
3528
|
const own1 = t1Of(w);
|
|
3930
3529
|
const ws = windowStart(w);
|
|
3931
|
-
const
|
|
3530
|
+
const holder = placed ? steps.find((s) => s.windows.some((x) => x.raw === w.raw)) : undefined;
|
|
3531
|
+
const step = placed ? holder?.endReason === "end-marker" ? holder : undefined : steps.find((s) => s.endReason === "end-marker" && ws >= s.startWall && ws <= s.endWall);
|
|
3932
3532
|
if (step && own1 !== undefined) {
|
|
3933
|
-
const markerT = own1 - (w.endWall - step.endWall);
|
|
3533
|
+
const markerT = own1 - ((placed?.get(w) ?? w).endWall - step.endWall);
|
|
3934
3534
|
if (markerT > t0 && markerT < endT) {
|
|
3935
3535
|
endT = markerT;
|
|
3936
3536
|
endReason = ACTION_END.stepEnd;
|
|
@@ -3959,9 +3559,9 @@ function actionsOf(windows, steps = []) {
|
|
|
3959
3559
|
return out;
|
|
3960
3560
|
}
|
|
3961
3561
|
var KEPT_FROM_INPUT = ["id", "interaction", "heapAfterGc"];
|
|
3962
|
-
function withActionWindows(windows, steps = []) {
|
|
3562
|
+
function withActionWindows(windows, steps = [], version) {
|
|
3963
3563
|
const byWindow = new Map;
|
|
3964
|
-
for (const a of actionsOf(windows, steps))
|
|
3564
|
+
for (const a of actionsOf(windows, steps, version))
|
|
3965
3565
|
byWindow.set(a.window, a);
|
|
3966
3566
|
return windows.map((w) => {
|
|
3967
3567
|
const a = byWindow.get(w);
|
|
@@ -3980,9 +3580,9 @@ function withActionWindows(windows, steps = []) {
|
|
|
3980
3580
|
return { name: w.name, startWall: w.startWall, endWall: w.endWall, raw };
|
|
3981
3581
|
});
|
|
3982
3582
|
}
|
|
3983
|
-
function actionMetrics(windows, steps = []) {
|
|
3583
|
+
function actionMetrics(windows, steps = [], version) {
|
|
3984
3584
|
const out = new Map;
|
|
3985
|
-
const actions = actionsOf(windows, steps);
|
|
3585
|
+
const actions = actionsOf(windows, steps, version);
|
|
3986
3586
|
const namesOf = new Map;
|
|
3987
3587
|
const updatersOf = (a) => {
|
|
3988
3588
|
const p = windowProfile(a.covered.raw);
|
|
@@ -4660,9 +4260,9 @@ var samplesAround = (samples, a, b) => {
|
|
|
4660
4260
|
const after = samples.find((s) => s.wall > b);
|
|
4661
4261
|
return after ? [after] : [];
|
|
4662
4262
|
};
|
|
4663
|
-
function v6Metrics(windows, samples, t0, t1) {
|
|
4263
|
+
function v6Metrics(windows, samples, t0, t1, version) {
|
|
4664
4264
|
const out = new Map;
|
|
4665
|
-
const placed = onAppClock(windows);
|
|
4265
|
+
const placed = onAppClock(windows, version);
|
|
4666
4266
|
const inputs = placed.filter(isInput).map((w) => w.startWall).sort((a, b) => a - b);
|
|
4667
4267
|
const inRun = samplesInRun(samples, t0, t1);
|
|
4668
4268
|
const gaps = new Map;
|
|
@@ -4728,7 +4328,7 @@ function windowEndFootprints(placed, records) {
|
|
|
4728
4328
|
}
|
|
4729
4329
|
return out;
|
|
4730
4330
|
}
|
|
4731
|
-
function v7Metrics(windows, steps, records) {
|
|
4331
|
+
function v7Metrics(windows, steps, records, version) {
|
|
4732
4332
|
const out = new Map;
|
|
4733
4333
|
const added = new Map;
|
|
4734
4334
|
for (const s of steps) {
|
|
@@ -4738,7 +4338,7 @@ function v7Metrics(windows, steps, records) {
|
|
|
4738
4338
|
}
|
|
4739
4339
|
for (const [name, v] of added)
|
|
4740
4340
|
out.set(`step:${name}.sampler_footprint_delta_mb`, median(v));
|
|
4741
|
-
const placed = onAppClock(windows).filter((w) => w.raw.endReason !== "timeout");
|
|
4341
|
+
const placed = onAppClock(windows, version).filter((w) => w.raw.endReason !== "timeout");
|
|
4742
4342
|
const ends = windowEndFootprints(placed, records);
|
|
4743
4343
|
const withEnd = [...ends.keys()];
|
|
4744
4344
|
const launchWindow = [...withEnd].sort((a, b) => a.startWall - b.startWall || a.endWall - b.endWall)[0];
|
|
@@ -4943,6 +4543,12 @@ function gcMetrics(markers, events, samples, windows, opts = {}) {
|
|
|
4943
4543
|
out.set(`step:${step}.${field}`, median3(vs));
|
|
4944
4544
|
return out;
|
|
4945
4545
|
}
|
|
4546
|
+
var END_READ_METRIC = "sdk.end_read";
|
|
4547
|
+
var manualReadFootprint = (samples, n) => {
|
|
4548
|
+
const reason = `endgc-read:${n}`;
|
|
4549
|
+
const found = samples.filter((s) => s.raw.reason === reason).sort((a, b) => a.wall - b.wall).at(-1);
|
|
4550
|
+
return found === undefined ? undefined : num9(obj9(found.raw.mem).footprintMb);
|
|
4551
|
+
};
|
|
4946
4552
|
var footprintsBySeq = (samples) => {
|
|
4947
4553
|
const bySeq = new Map;
|
|
4948
4554
|
for (const s of samples) {
|
|
@@ -4993,7 +4599,7 @@ function endsHome(windows, endCommandWall) {
|
|
|
4993
4599
|
const last = screens.filter((w) => w.startWall <= end).at(-1) ?? screens[screens.length - 1];
|
|
4994
4600
|
return screenChain(home.name) === screenChain(last.name) ? 1 : 0;
|
|
4995
4601
|
}
|
|
4996
|
-
function endMetrics(markers, events, samples, windows, launchMb) {
|
|
4602
|
+
function endMetrics(markers, events, samples, windows, launchMb, opts = {}) {
|
|
4997
4603
|
const out = new Map;
|
|
4998
4604
|
const n = markers.some((m) => m.phase === "start") ? runEndCommand(markers, events, windows) : null;
|
|
4999
4605
|
const command = n === null ? undefined : events.gc.find((g) => g.n === n && g.status === "ok");
|
|
@@ -5006,8 +4612,12 @@ function endMetrics(markers, events, samples, windows, launchMb) {
|
|
|
5006
4612
|
const g = n === null ? undefined : cleanOf(events, windows).get(n);
|
|
5007
4613
|
if (!g)
|
|
5008
4614
|
return out;
|
|
5009
|
-
const
|
|
4615
|
+
const post = g.post === undefined ? undefined : footprintsBySeq(samples).get(g.post);
|
|
4616
|
+
const manual = opts.manualRead ? manualReadFootprint(samples, g.n) : undefined;
|
|
4617
|
+
const footprint = manual ?? post;
|
|
5010
4618
|
if (footprint !== undefined) {
|
|
4619
|
+
if (opts.manualRead)
|
|
4620
|
+
out.set(END_READ_METRIC, manual === undefined ? 0 : 1);
|
|
5011
4621
|
out.set("mem.end_after_gc_mb", footprint);
|
|
5012
4622
|
if (launchMb !== undefined)
|
|
5013
4623
|
out.set("mem.retained_growth_mb", footprint - launchMb);
|
|
@@ -5068,12 +4678,15 @@ function keyFirstOpens(windows) {
|
|
|
5068
4678
|
}
|
|
5069
4679
|
|
|
5070
4680
|
// packages/derive/dist/src/derive.js
|
|
5071
|
-
var DERIVATION_VERSION =
|
|
5072
|
-
var DERIVATION_VERSIONS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22];
|
|
4681
|
+
var DERIVATION_VERSION = 23;
|
|
4682
|
+
var DERIVATION_VERSIONS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23];
|
|
5073
4683
|
var windowsForVersion = (windows, version = DERIVATION_VERSION) => version >= 22 ? keyFirstOpens(windows) : windows;
|
|
4684
|
+
function stepWindowsFor(windows, version = DERIVATION_VERSION) {
|
|
4685
|
+
return windowsForVersion(version >= 16 ? withLaunchNames(windows).filter((w) => !isBackgroundWindow(w)) : windows, version);
|
|
4686
|
+
}
|
|
5074
4687
|
function alignRunSteps(markers, windows, opts = {}, version = DERIVATION_VERSION) {
|
|
5075
4688
|
const order = version >= 17 ? { byOrder: true } : {};
|
|
5076
|
-
return version >= 3 ? alignSteps(markers, onAppClock(windows), { startToleranceMs: APP_CLOCK_TOLERANCE_MS, ...order, ...opts }) : alignSteps(markers, windows, opts);
|
|
4689
|
+
return version >= 3 ? alignSteps(markers, onAppClock(windows, version), { startToleranceMs: APP_CLOCK_TOLERANCE_MS, ...order, ...opts }) : alignSteps(markers, windows, opts);
|
|
5077
4690
|
}
|
|
5078
4691
|
function samplerFrameBudgetMs(samples) {
|
|
5079
4692
|
let hz = 0;
|
|
@@ -5094,10 +4707,10 @@ function deriveRun(input) {
|
|
|
5094
4707
|
const renamed = new Map(input.windows.flatMap((w, i) => named[i].name !== w.name ? [[w.name, named[i].name]] : []));
|
|
5095
4708
|
const cpu = input.cpu && renamed.size ? new Map([...input.cpu].map(([k, v]) => [renamed.get(k) ?? k, v])) : input.cpu;
|
|
5096
4709
|
const background = version >= 16 ? named.filter(isBackgroundWindow) : [];
|
|
5097
|
-
const sdkWindows =
|
|
4710
|
+
const sdkWindows = stepWindowsFor(input.windows, version);
|
|
5098
4711
|
const off = clockOffsetMs === undefined ? {} : { clockOffsetMs };
|
|
5099
4712
|
const preSteps = version >= 11 && markers.length ? alignRunSteps(markers, sdkWindows, off, version) : [];
|
|
5100
|
-
const windows = version >= 11 ? withActionWindows(sdkWindows, preSteps) : sdkWindows;
|
|
4713
|
+
const windows = version >= 11 ? withActionWindows(sdkWindows, preSteps, version) : sdkWindows;
|
|
5101
4714
|
const steps = version >= 11 ? preSteps : alignRunSteps(markers, windows, clockOffsetMs === undefined ? {} : { clockOffsetMs }, version);
|
|
5102
4715
|
const valid = run.valid && windows.length > 0;
|
|
5103
4716
|
const final = run.valid && !valid ? { ...run, valid, invalid_reason: "no-windows" } : run;
|
|
@@ -5128,16 +4741,16 @@ function deriveRun(input) {
|
|
|
5128
4741
|
...version >= 3 ? v3Metrics(windows, steps, samples, final.t0, final.t1, ri?.samples) : [],
|
|
5129
4742
|
...version >= 4 ? v4Metrics(windows) : [],
|
|
5130
4743
|
...version >= 5 ? v5Metrics(windows) : [],
|
|
5131
|
-
...version >= 6 ? v6Metrics(windows, samples, final.t0, final.t1) : [],
|
|
4744
|
+
...version >= 6 ? v6Metrics(windows, samples, final.t0, final.t1, version) : [],
|
|
5132
4745
|
...version >= 10 ? unsettledMetrics(sdkWindows) : [],
|
|
5133
|
-
...version >= 11 ? actionMetrics(sdkWindows, preSteps) : [],
|
|
4746
|
+
...version >= 11 ? actionMetrics(sdkWindows, preSteps, version) : [],
|
|
5134
4747
|
...version >= 11 ? windowModelMetrics(sdkWindows) : [],
|
|
5135
4748
|
...version >= 13 ? screenWindowModelMetrics(sdkWindows) : [],
|
|
5136
4749
|
...version >= 11 ? routeChangeMetrics(sdkWindows, preSteps) : [],
|
|
5137
4750
|
...version >= 14 ? v14Metrics(steps, samplesInRun(samples, final.t0, final.t1)) : [],
|
|
5138
4751
|
...version >= 15 ? v15Metrics(windows, steps, samplesInRun(samples, final.t0, final.t1)) : [],
|
|
5139
4752
|
...version >= 21 ? hookMetrics(sdkWindows, samplesInRun(samples, final.t0, final.t1).some((x) => typeof x.raw.reason === "string" && /^window-(start|end):/.test(x.raw.reason))) : [],
|
|
5140
|
-
...version >= 7 ? v7Metrics(windows, steps, version >= 8 ? samplesInRun(samples, final.t0, final.t1) : undefined) : [],
|
|
4753
|
+
...version >= 7 ? v7Metrics(windows, steps, version >= 8 ? samplesInRun(samples, final.t0, final.t1) : undefined, version) : [],
|
|
5141
4754
|
...version >= 16 ? externalMetrics(windows) : [],
|
|
5142
4755
|
...version >= 16 ? causeSplitMetrics(sdkWindows) : [],
|
|
5143
4756
|
...version >= 16 ? bg.metrics : [],
|
|
@@ -5146,7 +4759,7 @@ function deriveRun(input) {
|
|
|
5146
4759
|
if (version >= 19) {
|
|
5147
4760
|
const events = input.events ?? { gc: [], appState: [] };
|
|
5148
4761
|
const launch = extra.get("mem.launch_mb");
|
|
5149
|
-
for (const [k, v] of [...endMetrics(markers, events, samplesInRun(samples, final.t0, final.t1), sdkWindows, launch), ...jsHeapMetrics(events, sdkWindows)])
|
|
4762
|
+
for (const [k, v] of [...endMetrics(markers, events, samplesInRun(samples, final.t0, final.t1), sdkWindows, launch, { manualRead: version >= 23 }), ...jsHeapMetrics(events, sdkWindows)])
|
|
5150
4763
|
extra.set(k, v);
|
|
5151
4764
|
}
|
|
5152
4765
|
const base = { run_id: final.run_id, arm: final.arm, block: final.block, valid: final.valid, invalid_reason: final.invalid_reason };
|
|
@@ -5211,7 +4824,11 @@ function parseWindows(logText) {
|
|
|
5211
4824
|
}
|
|
5212
4825
|
return out;
|
|
5213
4826
|
}
|
|
5214
|
-
function windowsInRun(windows, t0, t1) {
|
|
4827
|
+
function windowsInRun(windows, t0, t1, version) {
|
|
4828
|
+
if (version !== undefined && version >= 23) {
|
|
4829
|
+
const placed = onAppClock(windows, version);
|
|
4830
|
+
return windows.filter((_, i) => placed[i].startWall <= t1);
|
|
4831
|
+
}
|
|
5215
4832
|
return windows.filter((w) => w.startWall >= t0 && w.startWall <= t1);
|
|
5216
4833
|
}
|
|
5217
4834
|
// apps/agent/src/markers.ts
|
|
@@ -5345,74 +4962,6 @@ async function uploadRuns(deps, outDir, runs, opts) {
|
|
|
5345
4962
|
return { uploaded, skipped };
|
|
5346
4963
|
}
|
|
5347
4964
|
|
|
5348
|
-
// packages/verdicts/dist/src/headline.js
|
|
5349
|
-
var HEAD_WORD = {
|
|
5350
|
-
worse: "Worse",
|
|
5351
|
-
warn: "Warning",
|
|
5352
|
-
better: "Better",
|
|
5353
|
-
"no-change": "No change",
|
|
5354
|
-
inconclusive: "Not enough data yet",
|
|
5355
|
-
"cant-compare": "Can’t compare"
|
|
5356
|
-
};
|
|
5357
|
-
var NOT_BLOCKING = "not blocking";
|
|
5358
|
-
var NOT_JUDGED = "Not judged";
|
|
5359
|
-
|
|
5360
|
-
// apps/agent/src/summary.ts
|
|
5361
|
-
var fieldWord = (field) => measureDef(field)?.short ?? field;
|
|
5362
|
-
function plainMetric(metric) {
|
|
5363
|
-
const i = metric.lastIndexOf(".");
|
|
5364
|
-
if (i <= 0)
|
|
5365
|
-
return metric;
|
|
5366
|
-
const word = measureDef(metric.slice(i + 1))?.short;
|
|
5367
|
-
return word ? `${metric.slice(0, i)}: ${word}` : metric;
|
|
5368
|
-
}
|
|
5369
|
-
var g = (x) => Number(x.toPrecision(3)).toString();
|
|
5370
|
-
var pct = (ratio) => `${ratio >= 1 ? "+" : "-"}${Math.round(Math.abs(ratio - 1) * 100)}%`;
|
|
5371
|
-
var countsWords = (c) => `${HEAD_WORD.worse}: ${c.worse} ${c.warnings === undefined ? "" : `${HEAD_WORD.warn}: ${c.warnings} `}${HEAD_WORD.better}: ${c.better} ${HEAD_WORD["no-change"]}: ${c.noChange} ${HEAD_WORD.inconclusive}: ${c.inconclusive}`;
|
|
5372
|
-
var isPinned = (metric, pins) => pins.some((p) => metric === p || metric.startsWith(`${p}.`) || metric.startsWith(`step:${p}.`));
|
|
5373
|
-
function summarize2(input) {
|
|
5374
|
-
const valid = input.runs.filter((r) => r.valid).length;
|
|
5375
|
-
const lines = [`${valid}/${input.runs.length} runs valid`];
|
|
5376
|
-
const dropped = input.runs.filter((r) => !r.valid);
|
|
5377
|
-
if (dropped.length) {
|
|
5378
|
-
const reasons = new Map;
|
|
5379
|
-
for (const r of dropped)
|
|
5380
|
-
reasons.set(r.invalid_reason, (reasons.get(r.invalid_reason) ?? 0) + 1);
|
|
5381
|
-
lines[0] += ` (dropped: ${[...reasons].map(([k, n]) => `${n} ${k}`).join(", ")})`;
|
|
5382
|
-
}
|
|
5383
|
-
if (valid === 0)
|
|
5384
|
-
lines.push("No valid runs: nothing to compare. See <out>/<run>/test.log for the first failure.");
|
|
5385
|
-
const cmp = input.comparison;
|
|
5386
|
-
if (!cmp) {
|
|
5387
|
-
lines.push(`No baseline given; pass --baseline <earlier out dir> to see what changed. Results: ${input.outDir}`);
|
|
5388
|
-
return lines.join(`
|
|
5389
|
-
`);
|
|
5390
|
-
}
|
|
5391
|
-
const rows = cmp.comparisons.filter((c) => c.metric !== "run.status");
|
|
5392
|
-
const count = { worse: 0, better: 0, "no-change": 0, inconclusive: 0 };
|
|
5393
|
-
for (const c of rows)
|
|
5394
|
-
count[c.result ? c.result.verdict : "inconclusive"]++;
|
|
5395
|
-
lines.push(`Compared with ${input.baselineDir ?? "baseline"}: ${rows.length} metrics`);
|
|
5396
|
-
const changes = rows.filter((c) => c.result && (c.result.verdict === "worse" || c.result.verdict === "better") && Number.isFinite(c.result.ratio)).sort((a, b) => Math.abs(Math.log(b.result?.ratio || 1)) - Math.abs(Math.log(a.result?.ratio || 1))).slice(0, input.top ?? 5);
|
|
5397
|
-
if (changes.length) {
|
|
5398
|
-
lines.push("Top changes:");
|
|
5399
|
-
for (const c of changes) {
|
|
5400
|
-
const r = c.result;
|
|
5401
|
-
lines.push(` ${HEAD_WORD[r.verdict].padEnd(9)} ${plainMetric(c.metric)} ${pct(r.ratio)} (${g(r.baseline.location)} -> ${g(r.candidate.location)})`);
|
|
5402
|
-
}
|
|
5403
|
-
} else
|
|
5404
|
-
lines.push("No metric changed beyond its noise.");
|
|
5405
|
-
const pinned = rows.filter((c) => input.pins && isPinned(c.metric, input.pins) && !changes.includes(c));
|
|
5406
|
-
if (pinned.length) {
|
|
5407
|
-
lines.push("Pinned:");
|
|
5408
|
-
for (const c of pinned)
|
|
5409
|
-
lines.push(` ${HEAD_WORD[c.result ? c.result.verdict : "inconclusive"].padEnd(19)} ${plainMetric(c.metric)}`);
|
|
5410
|
-
}
|
|
5411
|
-
lines.push(countsWords({ worse: count.worse, better: count.better, noChange: count["no-change"], inconclusive: count.inconclusive }));
|
|
5412
|
-
return lines.join(`
|
|
5413
|
-
`);
|
|
5414
|
-
}
|
|
5415
|
-
|
|
5416
4965
|
// packages/present/dist/src/places.js
|
|
5417
4966
|
var PLACE_KIND_WORDS = {
|
|
5418
4967
|
screen: "Screen",
|
|
@@ -5491,11 +5040,30 @@ function placeKindWord(id) {
|
|
|
5491
5040
|
const { kind } = placeRef(id);
|
|
5492
5041
|
return kind === "other" ? "Window" : KIND_WORD[kind];
|
|
5493
5042
|
}
|
|
5494
|
-
|
|
5495
|
-
|
|
5043
|
+
var ELEMENT_SEPARATORS = /(?<!\d)\.|\.(?!\d)|[_-]+/g;
|
|
5044
|
+
var elementWords = (testId) => testId.replace(ELEMENT_SEPARATORS, " ").replace(/\s+/g, " ").trim();
|
|
5045
|
+
function placeLabel(id, screen) {
|
|
5046
|
+
if (id === "flow")
|
|
5047
|
+
return PLACE_KIND_WORDS.journey;
|
|
5048
|
+
const ref = placeRef(id);
|
|
5049
|
+
const after = ref.segment === "after3s" ? ` · ${PLACE_KIND_WORDS.after3s}` : "";
|
|
5050
|
+
if (ref.kind === "press" || ref.kind === "drag" || ref.kind === "step") {
|
|
5051
|
+
const element = elementWords(ref.stem.slice(ref.stem.indexOf(":") + 1).replace(OPEN_MARK, ""));
|
|
5052
|
+
return `${KIND_WORD[ref.kind]} · “${element}”${screen ? ` · ${screen}` : ""}${after}`;
|
|
5053
|
+
}
|
|
5054
|
+
const name = windowName(ref.stem);
|
|
5496
5055
|
const colon = name.indexOf(": ");
|
|
5497
|
-
return colon > 0 &&
|
|
5056
|
+
return `${colon > 0 && ref.kind !== "other" ? `${name.slice(0, colon)} · ${name.slice(colon + 2)}` : name}${after}`;
|
|
5498
5057
|
}
|
|
5058
|
+
// packages/present/dist/src/states.js
|
|
5059
|
+
var STATE_WORDS = {
|
|
5060
|
+
worse: "Worse",
|
|
5061
|
+
warning: "Slightly worse",
|
|
5062
|
+
better: "Better",
|
|
5063
|
+
"no-change": "No change",
|
|
5064
|
+
"cant-tell": "Can’t tell"
|
|
5065
|
+
};
|
|
5066
|
+
var NOT_BLOCKING = "not blocking";
|
|
5499
5067
|
// packages/present/dist/src/values.js
|
|
5500
5068
|
var NO_VALUE = "–";
|
|
5501
5069
|
var FALLBACK_DECIMALS = 1;
|
|
@@ -5532,13 +5100,72 @@ function format(unit, decimals, v) {
|
|
|
5532
5100
|
const text = String(roundTo(v * shown.scale, decimals) + 0);
|
|
5533
5101
|
return shown.unit ? `${text} ${shown.unit}` : text;
|
|
5534
5102
|
}
|
|
5103
|
+
function valueText(measure, v) {
|
|
5104
|
+
const def = isMeasureId(measure) ? MEASURES[measure] : BREAKDOWN_MEASURES[measure];
|
|
5105
|
+
return format(def.unit, def.decimals, v);
|
|
5106
|
+
}
|
|
5535
5107
|
var KEY_SEPARATOR = ":";
|
|
5108
|
+
function breakdownMeasure(field) {
|
|
5109
|
+
const at = field.indexOf(KEY_SEPARATOR);
|
|
5110
|
+
if (at < 0 || !Object.hasOwn(DIMENSIONS, field.slice(0, at)))
|
|
5111
|
+
return null;
|
|
5112
|
+
const measure = field.slice(field.lastIndexOf(".") + 1);
|
|
5113
|
+
return Object.hasOwn(BREAKDOWN_MEASURES, measure) ? measure : null;
|
|
5114
|
+
}
|
|
5536
5115
|
function fieldValueText(field, v) {
|
|
5116
|
+
const breakdown = breakdownMeasure(field);
|
|
5117
|
+
if (breakdown)
|
|
5118
|
+
return valueText(breakdown, v);
|
|
5537
5119
|
const def = fieldDef(field.split(KEY_SEPARATOR)[0]);
|
|
5538
5120
|
return format(def?.unit, def?.decimals ?? FALLBACK_DECIMALS, v);
|
|
5539
5121
|
}
|
|
5122
|
+
// packages/present/dist/src/stability.js
|
|
5123
|
+
var UNSTABLE_WORDS = {
|
|
5124
|
+
short: "main varies too much here",
|
|
5125
|
+
label: `${STATE_WORDS["cant-tell"]} · main varies too much here`,
|
|
5126
|
+
line: "Main varies too much here to tell",
|
|
5127
|
+
ratioTip: "1.0 = main’s run-to-run noise alone would pass the warning step about once in 100 comparisons.",
|
|
5128
|
+
what: "Main’s own runs vary too much here to tell, so a comparison there would mostly measure noise. It is shown with its values, never judged, never blocks and is never counted. It is judged again once main settles."
|
|
5129
|
+
};
|
|
5130
|
+
// apps/agent/src/summary.ts
|
|
5131
|
+
var fieldWord = (field) => measureDef(field)?.short ?? field;
|
|
5132
|
+
function plainMetric(metric) {
|
|
5133
|
+
const i = metric.lastIndexOf(".");
|
|
5134
|
+
if (i <= 0)
|
|
5135
|
+
return metric;
|
|
5136
|
+
const word = measureDef(metric.slice(i + 1))?.short;
|
|
5137
|
+
return word ? `${metric.slice(0, i)}: ${word}` : metric;
|
|
5138
|
+
}
|
|
5139
|
+
var WORD_OF = {
|
|
5140
|
+
worse: STATE_WORDS.worse,
|
|
5141
|
+
warn: STATE_WORDS.warning,
|
|
5142
|
+
better: STATE_WORDS.better,
|
|
5143
|
+
"no-change": STATE_WORDS["no-change"],
|
|
5144
|
+
inconclusive: STATE_WORDS["cant-tell"]
|
|
5145
|
+
};
|
|
5146
|
+
var stateWord = (state) => WORD_OF[state];
|
|
5147
|
+
var WORD_WIDTH = Math.max(...Object.values(WORD_OF).map((w) => w.length));
|
|
5148
|
+
var countsWords = (c) => `${STATE_WORDS.worse}: ${c.worse} ${c.warnings === undefined ? "" : `${STATE_WORDS.warning}: ${c.warnings} `}${STATE_WORDS.better}: ${c.better} ${STATE_WORDS["no-change"]}: ${c.noChange} ${STATE_WORDS["cant-tell"]}: ${c.inconclusive}`;
|
|
5149
|
+
function summarize(input) {
|
|
5150
|
+
const valid = input.runs.filter((r) => r.valid).length;
|
|
5151
|
+
const lines = [`${valid}/${input.runs.length} runs valid`];
|
|
5152
|
+
const dropped = input.runs.filter((r) => !r.valid);
|
|
5153
|
+
if (dropped.length) {
|
|
5154
|
+
const reasons = new Map;
|
|
5155
|
+
for (const r of dropped)
|
|
5156
|
+
reasons.set(r.invalid_reason, (reasons.get(r.invalid_reason) ?? 0) + 1);
|
|
5157
|
+
lines[0] += ` (dropped: ${[...reasons].map(([k, n]) => `${n} ${k}`).join(", ")})`;
|
|
5158
|
+
}
|
|
5159
|
+
if (valid === 0)
|
|
5160
|
+
lines.push("No valid runs: nothing to upload. See <out>/<run>/test.log for the first failure.");
|
|
5161
|
+
lines.push(`Results: ${input.outDir}`);
|
|
5162
|
+
return lines.join(`
|
|
5163
|
+
`);
|
|
5164
|
+
}
|
|
5165
|
+
|
|
5540
5166
|
// apps/agent/src/remote.ts
|
|
5541
5167
|
var isBlocking = (s) => !!s?.verdicts && (s.verdicts.blocking ?? s.verdicts.state === "worse");
|
|
5168
|
+
var midLine = (label, metric) => !metric.startsWith("component:") && /^[A-Z][a-z]*(\s|$)/.test(label) ? `${label[0].toLowerCase()}${label.slice(1)}` : label;
|
|
5542
5169
|
function movedItems(v, sc) {
|
|
5543
5170
|
const flat = v.items.filter((i) => i.scenario === sc.scenario && (!sc.device || !i.device_id || i.device_id === sc.device.id));
|
|
5544
5171
|
const h = sc.headline;
|
|
@@ -5556,18 +5183,23 @@ function movedItems(v, sc) {
|
|
|
5556
5183
|
baseline: g.baseline ?? same?.baseline ?? null,
|
|
5557
5184
|
candidate: g.candidate ?? same?.candidate ?? null,
|
|
5558
5185
|
change_text: g.change,
|
|
5559
|
-
...g.
|
|
5186
|
+
...g.finding ? { finding: g.finding } : {},
|
|
5187
|
+
...g.label ? { label_text: midLine(g.label, g.metric) } : {},
|
|
5560
5188
|
...sc.device ? { device_id: sc.device.id } : {}
|
|
5561
5189
|
};
|
|
5562
5190
|
});
|
|
5563
5191
|
return [...graded, ...flat.filter((i) => i.state === "better" && (i.judged ?? "outcome") === "outcome")];
|
|
5564
5192
|
}
|
|
5193
|
+
var placeOfRow = (g) => g.finding ?? placeRef(g.window).stem;
|
|
5565
5194
|
function movedCounts(sc) {
|
|
5566
5195
|
const h = sc.headline;
|
|
5567
5196
|
if (h?.errors === undefined || !h.counts)
|
|
5568
5197
|
return { worse: sc.counts.worse, warnings: 0, better: sc.counts.better, noChange: sc.counts.no_change, inconclusive: sc.counts.inconclusive };
|
|
5569
|
-
|
|
5198
|
+
const worse = new Set(h.errors.map(placeOfRow));
|
|
5199
|
+
const slightly = new Set((h.warnings ?? []).map(placeOfRow).filter((p) => !worse.has(p)));
|
|
5200
|
+
return { worse: worse.size, warnings: slightly.size, better: h.counts.better, noChange: h.counts.no_change, inconclusive: h.counts.inconclusive };
|
|
5570
5201
|
}
|
|
5202
|
+
var placeCount = (rows) => new Set(rows.map((i) => `${i.scenario}|${i.device_id ?? ""}|${i.finding ?? placeRef(i.step ?? "*").stem}`)).size;
|
|
5571
5203
|
var addCounts = (xs) => xs.reduce((a, c) => ({ worse: a.worse + c.worse, warnings: a.warnings + c.warnings, better: a.better + c.better, noChange: a.noChange + c.noChange, inconclusive: a.inconclusive + c.inconclusive }), { worse: 0, warnings: 0, better: 0, noChange: 0, inconclusive: 0 });
|
|
5572
5204
|
var setupLine = (h) => h?.measured_differently?.length ? `Measured differently (${h.measured_differently.join(", ")}) — ${NOT_BLOCKING}` : null;
|
|
5573
5205
|
var keyHeader = (t) => ({ "x-keepsnappy-key": t.key });
|
|
@@ -5593,17 +5225,10 @@ async function finishIngestRun(t, runId, invalidReason, farmSession) {
|
|
|
5593
5225
|
async function getRunSetRuns(t, id) {
|
|
5594
5226
|
return await call(t.fetch ?? fetch, t.url, `/v1/run-sets/${encodeURIComponent(id)}/runs`, keyHeader(t), undefined, "GET");
|
|
5595
5227
|
}
|
|
5596
|
-
async function fetchBaseline(t, q) {
|
|
5597
|
-
const p = new URLSearchParams;
|
|
5598
|
-
for (const [k, v] of Object.entries(q))
|
|
5599
|
-
if (v !== undefined && v !== "")
|
|
5600
|
-
p.set(k, String(v));
|
|
5601
|
-
return await call(t.fetch ?? fetch, t.url, `/v1/baselines?${p}`, keyHeader(t), undefined, "GET");
|
|
5602
|
-
}
|
|
5603
5228
|
var FEW_RUNS = "Few runs — result may be noisy";
|
|
5604
5229
|
var word = (s, server, h) => {
|
|
5605
|
-
const w = server
|
|
5606
|
-
return h?.blocking === false && h.measured_differently?.length && (w ===
|
|
5230
|
+
const w = server || stateWord(s ?? "inconclusive");
|
|
5231
|
+
return h?.blocking === false && h.measured_differently?.length && (w === STATE_WORDS.worse || s === "worse") ? `${w} (${NOT_BLOCKING})` : w;
|
|
5607
5232
|
};
|
|
5608
5233
|
var countsLine = (c) => countsWords(c);
|
|
5609
5234
|
var noCounts = (c) => !c.worse && !c.warnings && !c.better && !c.noChange && !c.inconclusive;
|
|
@@ -5640,8 +5265,8 @@ function formatRunSet(s, top = 5) {
|
|
|
5640
5265
|
lines.push(` ${countsLine(movedCounts(sc))}`);
|
|
5641
5266
|
const moved = movedItems(v, sc).sort((a, b) => Number(b.state === "worse") - Number(a.state === "worse") || Number(a.grade === "warn") - Number(b.grade === "warn")).slice(0, top);
|
|
5642
5267
|
for (const i of moved) {
|
|
5643
|
-
const what = i.label_text ? `${i.step && i.step !== "*" ? `${
|
|
5644
|
-
lines.push(` ${(i.state === "worse" && i.grade === "warn" ?
|
|
5268
|
+
const what = i.label_text ? `${i.step && i.step !== "*" ? `${placeLabel(i.step)}: ` : ""}${i.label_text}` : i.step ? `${placeLabel(i.step)}: ${fieldWord(i.metric)}` : plainMetric(i.metric);
|
|
5269
|
+
lines.push(` ${stateWord(i.state === "worse" && i.grade === "warn" ? "warn" : i.state).padEnd(WORD_WIDTH)} ${what}${i.change_text ? `: ${i.change_text}` : change(i.metric, i.baseline, i.candidate)}`);
|
|
5645
5270
|
}
|
|
5646
5271
|
if (sc.dashboard_url)
|
|
5647
5272
|
lines.push(` ${sc.dashboard_url}`);
|
|
@@ -5673,7 +5298,7 @@ Runs uploaded for \`${fallback.branch}\`. The result appears on the dashboard: [
|
|
|
5673
5298
|
...setupLine(v.headline) ? [`${setupLine(v.headline)}.`, ""] : [],
|
|
5674
5299
|
`\`${s.branch}\`${s.commit_sha ? ` @ \`${s.commit_sha.slice(0, 7)}\`` : ""}, ${s.runs.valid}/${s.runs.total} runs used.${v.low_data ? ` ${FEW_RUNS}.` : ""}`,
|
|
5675
5300
|
"",
|
|
5676
|
-
`| Journey | Device | Result | ${
|
|
5301
|
+
`| Journey | Device | Result | ${STATE_WORDS.worse} / ${STATE_WORDS.warning} / ${STATE_WORDS.better} / ${STATE_WORDS["no-change"]} / ${STATE_WORDS["cant-tell"]} | Details |`,
|
|
5677
5302
|
"|---|---|---|---|---|",
|
|
5678
5303
|
...rows,
|
|
5679
5304
|
"",
|
|
@@ -5683,11 +5308,8 @@ Runs uploaded for \`${fallback.branch}\`. The result appears on the dashboard: [
|
|
|
5683
5308
|
`);
|
|
5684
5309
|
}
|
|
5685
5310
|
|
|
5686
|
-
// apps/agent/src/remote-verdict.ts
|
|
5687
|
-
import { join as join10 } from "node:path";
|
|
5688
|
-
|
|
5689
5311
|
// apps/agent/src/run.ts
|
|
5690
|
-
import { join as
|
|
5312
|
+
import { join as join8 } from "node:path";
|
|
5691
5313
|
// apps/agent/src/extract.ts
|
|
5692
5314
|
import { join as join6 } from "node:path";
|
|
5693
5315
|
|
|
@@ -8560,7 +8182,7 @@ function buildWindowTaskCards(input, windows, options = {}) {
|
|
|
8560
8182
|
if (tasks.length === 0)
|
|
8561
8183
|
return;
|
|
8562
8184
|
const durationMs = Math.max(1, window.t1Ms - window.t0Ms);
|
|
8563
|
-
out.push({ window, name:
|
|
8185
|
+
out.push({ window, name: placeLabel(window.key), set: cardSet(input, folded, classes, tasks, durationMs, offset) });
|
|
8564
8186
|
});
|
|
8565
8187
|
const outsideMs = outside.reduce((sum, task) => sum + task.durationMs, 0);
|
|
8566
8188
|
return { windows: out, outside: cardSet(input, folded, classes, outside, Math.max(1, outsideMs), offset) };
|
|
@@ -9189,53 +8811,6 @@ ${r.stderr}` : ""}`);
|
|
|
9189
8811
|
return { runs, rows };
|
|
9190
8812
|
}
|
|
9191
8813
|
|
|
9192
|
-
// apps/agent/src/verdict.ts
|
|
9193
|
-
import { join as join8 } from "node:path";
|
|
9194
|
-
var g2 = (x, d = 4) => x === undefined || !Number.isFinite(x) ? "-" : Number(x.toPrecision(d)).toString();
|
|
9195
|
-
var pad = (rows) => {
|
|
9196
|
-
const w = rows[0]?.map((_, i) => Math.max(...rows.map((r) => (r[i] ?? "").length))) ?? [];
|
|
9197
|
-
return rows.map((r) => r.map((c, i) => c.padEnd(w[i] ?? 0)).join(" ").trimEnd()).join(`
|
|
9198
|
-
`);
|
|
9199
|
-
};
|
|
9200
|
-
async function load(deps, outDir) {
|
|
9201
|
-
return readRunDataset(await deps.fs.readFile(join8(outDir, "metrics.csv")));
|
|
9202
|
-
}
|
|
9203
|
-
async function verdict(deps, outDir, opts = {}) {
|
|
9204
|
-
const dataset = await load(deps, outDir);
|
|
9205
|
-
const arms = [...new Set([...dataset.metrics.values()].flatMap((m) => [...m.keys()]))].sort();
|
|
9206
|
-
const baseline = opts.baseline ?? arms[0] ?? "A";
|
|
9207
|
-
const candidate = opts.candidate ?? arms.find((a) => a !== baseline) ?? "B";
|
|
9208
|
-
const result = compareRuns({ dataset, baseline, candidate, config: { autoCount: true, countFloor: DEFAULT_COUNT_FLOOR, ...opts.config } });
|
|
9209
|
-
const rows = [["metric", "base", "cand", "ratio", "ci95", "p", "p_adj", "verdict"]];
|
|
9210
|
-
for (const c of result.comparisons) {
|
|
9211
|
-
const r = c.result;
|
|
9212
|
-
if (!r)
|
|
9213
|
-
rows.push([c.metric, "-", "-", "-", "-", "-", "-", `skipped (${c.skipped})`]);
|
|
9214
|
-
else
|
|
9215
|
-
rows.push([c.metric, g2(r.baseline.location), g2(r.candidate.location), g2(r.ratio), `${g2(r.ratioCi95.lo, 3)}..${g2(r.ratioCi95.hi, 3)}`, g2(r.p, 3), g2(c.pAdjusted, 3), r.verdict]);
|
|
9216
|
-
}
|
|
9217
|
-
const table = `${baseline} -> ${candidate}: ${result.nRuns} runs, ${result.nInvalidRuns} invalid
|
|
9218
|
-
${pad(rows)}`;
|
|
9219
|
-
await deps.fs.writeFile(join8(outDir, "verdict.json"), `${JSON.stringify({ kind: "change", baseline, candidate, ...result }, null, 2)}
|
|
9220
|
-
`);
|
|
9221
|
-
return { baseline, candidate, result, table };
|
|
9222
|
-
}
|
|
9223
|
-
async function aa(deps, outDir, opts = {}) {
|
|
9224
|
-
const dataset = await load(deps, outDir);
|
|
9225
|
-
const results = aaRuns({ dataset, autoCount: true, config: { margin: relativeMargin(0.05), ...opts.config } });
|
|
9226
|
-
const rows = [["metric", "n", "median", "robustCV", "MDE@n", "suggested margin"]];
|
|
9227
|
-
for (const r of results) {
|
|
9228
|
-
if (r.skipped)
|
|
9229
|
-
rows.push([r.metric, String(r.pool.n), "-", "-", "-", r.skipped]);
|
|
9230
|
-
else
|
|
9231
|
-
rows.push([r.metric, String(r.pool.n), g2(r.pool.median), g2(r.pool.robustCv, 3), r.rows.map((x) => `${x.n}:${x.mde === null ? ">max" : g2(x.mde, 2)}`).join(" "), r.suggestedMargin.reason]);
|
|
9232
|
-
}
|
|
9233
|
-
const table = pad(rows);
|
|
9234
|
-
await deps.fs.writeFile(join8(outDir, "verdict.json"), `${JSON.stringify({ kind: "aa", results }, null, 2)}
|
|
9235
|
-
`);
|
|
9236
|
-
return { results, table };
|
|
9237
|
-
}
|
|
9238
|
-
|
|
9239
8814
|
// apps/agent/src/run.ts
|
|
9240
8815
|
function renameMetrics(rows, rename) {
|
|
9241
8816
|
const pairs = Object.entries(rename).flatMap(([a, b]) => [[a, b], [`step:${a}`, `step:${b}`]]);
|
|
@@ -9251,49 +8826,6 @@ function renameMetrics(rows, rename) {
|
|
|
9251
8826
|
return r;
|
|
9252
8827
|
});
|
|
9253
8828
|
}
|
|
9254
|
-
var csvCell = (s) => /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
|
|
9255
|
-
async function scenariosOf(deps, outDir) {
|
|
9256
|
-
const out = new Map;
|
|
9257
|
-
let text = "";
|
|
9258
|
-
try {
|
|
9259
|
-
text = await deps.fs.readFile(join9(outDir, "runs.ndjson"));
|
|
9260
|
-
} catch {
|
|
9261
|
-
return out;
|
|
9262
|
-
}
|
|
9263
|
-
for (const line of text.split(`
|
|
9264
|
-
`)) {
|
|
9265
|
-
if (!line.trim())
|
|
9266
|
-
continue;
|
|
9267
|
-
const r = JSON.parse(line);
|
|
9268
|
-
if (r.scenario)
|
|
9269
|
-
out.set(r.run_id, r.scenario);
|
|
9270
|
-
}
|
|
9271
|
-
return out;
|
|
9272
|
-
}
|
|
9273
|
-
async function writeComparisonDataset(deps, baselineDir, outDir) {
|
|
9274
|
-
const dir = join9(outDir, "compare");
|
|
9275
|
-
const rows = ["run_id,arm,block,metric,value,valid,invalid_reason"];
|
|
9276
|
-
const maps = [await scenariosOf(deps, baselineDir), await scenariosOf(deps, outDir)];
|
|
9277
|
-
const several = new Set([...maps[0].values(), ...maps[1].values()]).size > 1;
|
|
9278
|
-
for (const [i, [src, arm, tag]] of [[baselineDir, "baseline", "b"], [outDir, "candidate", "c"]].entries()) {
|
|
9279
|
-
let text;
|
|
9280
|
-
try {
|
|
9281
|
-
text = await deps.fs.readFile(join9(src, "metrics.csv"));
|
|
9282
|
-
} catch {
|
|
9283
|
-
throw new Error(`no metrics.csv in ${src}: --baseline must be the out dir of an earlier run`);
|
|
9284
|
-
}
|
|
9285
|
-
for (const { cells: c } of parseCsv(text).rows) {
|
|
9286
|
-
const flow = maps[i].get(c.run_id ?? "");
|
|
9287
|
-
const metric = several && flow ? `${flow}/${c.metric ?? ""}` : c.metric ?? "";
|
|
9288
|
-
rows.push([`${tag}:${c.run_id ?? ""}`, arm, `${tag}${c.block ?? ""}`, metric, c.value ?? "", c.valid ?? "", c.invalid_reason ?? ""].map(csvCell).join(","));
|
|
9289
|
-
}
|
|
9290
|
-
}
|
|
9291
|
-
await deps.fs.mkdir(dir);
|
|
9292
|
-
await deps.fs.writeFile(join9(dir, "metrics.csv"), `${rows.join(`
|
|
9293
|
-
`)}
|
|
9294
|
-
`);
|
|
9295
|
-
return dir;
|
|
9296
|
-
}
|
|
9297
8829
|
async function keepsnappyRun(deps, source, opts) {
|
|
9298
8830
|
const { runs, rows: raw } = await measure(deps, source, undefined, {
|
|
9299
8831
|
arms: [{ name: "A" }],
|
|
@@ -9307,20 +8839,15 @@ async function keepsnappyRun(deps, source, opts) {
|
|
|
9307
8839
|
...opts.clockOffsetMs === undefined ? {} : { clockOffsetMs: opts.clockOffsetMs }
|
|
9308
8840
|
});
|
|
9309
8841
|
const rows = renameMetrics(raw, opts.config.steps.rename);
|
|
9310
|
-
await deps.fs.writeFile(
|
|
9311
|
-
|
|
9312
|
-
|
|
9313
|
-
const dir = await writeComparisonDataset(deps, opts.baselineDir, opts.outDir);
|
|
9314
|
-
v = await verdict(deps, dir, { baseline: "baseline", candidate: "candidate" });
|
|
9315
|
-
}
|
|
9316
|
-
const summary = summarize2({ runs, outDir: opts.outDir, ...opts.baselineDir ? { baselineDir: opts.baselineDir } : {}, ...v ? { comparison: v.result } : {}, pins: opts.config.steps.pin });
|
|
9317
|
-
return { runs, rows, ...v ? { verdict: v } : {}, summary };
|
|
8842
|
+
await deps.fs.writeFile(join8(opts.outDir, "metrics.csv"), toCsv(rows));
|
|
8843
|
+
const summary = summarize({ runs, outDir: opts.outDir });
|
|
8844
|
+
return { runs, rows, summary };
|
|
9318
8845
|
}
|
|
9319
8846
|
async function replayRuns(deps, sources, outDir, scenario) {
|
|
9320
8847
|
await deps.fs.mkdir(outDir);
|
|
9321
8848
|
const runs = [];
|
|
9322
8849
|
for (const src of sources) {
|
|
9323
|
-
const text = await deps.fs.readFile(
|
|
8850
|
+
const text = await deps.fs.readFile(join8(src.dir, "runs.ndjson")).catch(() => {
|
|
9324
8851
|
throw new Error(`--replay ${src.dir}: no runs.ndjson (not a keepsnappy out dir)`);
|
|
9325
8852
|
});
|
|
9326
8853
|
for (const line of text.split(`
|
|
@@ -9331,75 +8858,24 @@ async function replayRuns(deps, sources, outDir, scenario) {
|
|
|
9331
8858
|
if (!r.valid || src.arm && r.arm !== src.arm)
|
|
9332
8859
|
continue;
|
|
9333
8860
|
const id = `${String(runs.length + 1).padStart(3, "0")}-A`;
|
|
9334
|
-
await deps.fs.mkdir(
|
|
8861
|
+
await deps.fs.mkdir(join8(outDir, id));
|
|
9335
8862
|
for (const f of ["keepsnappy.log", "markers.ndjson", "test.log"]) {
|
|
9336
|
-
const body = await deps.fs.readFile(
|
|
8863
|
+
const body = await deps.fs.readFile(join8(src.dir, r.run_id, f)).catch(() => null);
|
|
9337
8864
|
if (body !== null)
|
|
9338
|
-
await deps.fs.writeFile(
|
|
8865
|
+
await deps.fs.writeFile(join8(outDir, id, f), body);
|
|
9339
8866
|
}
|
|
9340
8867
|
runs.push({ ...r, run_id: id, arm: "A", block: runs.length, scenario });
|
|
9341
8868
|
}
|
|
9342
8869
|
}
|
|
9343
8870
|
if (runs.length === 0)
|
|
9344
8871
|
throw new Error(`--replay: no valid runs in ${sources.map((x) => x.dir + (x.arm ? `:${x.arm}` : "")).join(", ")}`);
|
|
9345
|
-
await deps.fs.writeFile(
|
|
8872
|
+
await deps.fs.writeFile(join8(outDir, "runs.ndjson"), runs.map((r) => JSON.stringify(r)).join(`
|
|
9346
8873
|
`) + `
|
|
9347
8874
|
`);
|
|
9348
8875
|
await extract(deps, outDir, undefined);
|
|
9349
8876
|
return runs;
|
|
9350
8877
|
}
|
|
9351
8878
|
|
|
9352
|
-
// apps/agent/src/remote-verdict.ts
|
|
9353
|
-
var RUN_INFO = "keepsnappy-run.json";
|
|
9354
|
-
async function remoteVerdict(deps, outDir, api, opts = {}) {
|
|
9355
|
-
let info = {};
|
|
9356
|
-
try {
|
|
9357
|
-
info = JSON.parse(await deps.fs.readFile(join10(outDir, RUN_INFO)));
|
|
9358
|
-
} catch {}
|
|
9359
|
-
const flows = await scenariosOf(deps, outDir);
|
|
9360
|
-
const local = parseCsv(await deps.fs.readFile(join10(outDir, "metrics.csv"))).rows.map((r) => r.cells);
|
|
9361
|
-
const names = flows.size ? [...new Set(flows.values())].sort() : [opts.scenario];
|
|
9362
|
-
if (names.some((n) => !n))
|
|
9363
|
-
throw new Error("the runs in this out dir have no flow name: pass --scenario <flow file stem>");
|
|
9364
|
-
const results = [];
|
|
9365
|
-
const parts = [];
|
|
9366
|
-
for (const scenario of names) {
|
|
9367
|
-
const base = await fetchBaseline(api, {
|
|
9368
|
-
scenario,
|
|
9369
|
-
...info.device?.model ? { device_model: info.device.model } : {},
|
|
9370
|
-
...info.device?.os ? { os: info.device.os } : {},
|
|
9371
|
-
...opts.branch ? { branch: opts.branch } : {},
|
|
9372
|
-
...opts.limit ? { limit: opts.limit } : {}
|
|
9373
|
-
});
|
|
9374
|
-
if (base.rows.length === 0) {
|
|
9375
|
-
results.push({ scenario, baselineRuns: 0 });
|
|
9376
|
-
parts.push(`${scenario}: no baseline on ${base.branch} yet${info.device?.model ? ` for ${info.device.model}` : ""} (upload runs from ${base.branch} first)`);
|
|
9377
|
-
continue;
|
|
9378
|
-
}
|
|
9379
|
-
const judged = new Set(base.rows.map((r) => r.metric));
|
|
9380
|
-
const lines = ["run_id,arm,block,metric,value,valid,invalid_reason"];
|
|
9381
|
-
for (const r of base.rows)
|
|
9382
|
-
lines.push([`b:${r.run_id}`, "baseline", "b", r.metric, String(r.value), "true", ""].map(csvCell).join(","));
|
|
9383
|
-
for (const c of local) {
|
|
9384
|
-
if (flows.size && flows.get(c.run_id ?? "") !== scenario || !judged.has(c.metric ?? ""))
|
|
9385
|
-
continue;
|
|
9386
|
-
lines.push([`c:${c.run_id ?? ""}`, "candidate", `c${c.block ?? ""}`, c.metric ?? "", c.value ?? "", c.valid ?? "", c.invalid_reason ?? ""].map(csvCell).join(","));
|
|
9387
|
-
}
|
|
9388
|
-
const dir = join10(outDir, "remote-compare", scenario);
|
|
9389
|
-
await deps.fs.mkdir(dir);
|
|
9390
|
-
await deps.fs.writeFile(join10(dir, "metrics.csv"), `${lines.join(`
|
|
9391
|
-
`)}
|
|
9392
|
-
`);
|
|
9393
|
-
const output = await verdict(deps, dir, { baseline: "baseline", candidate: "candidate" });
|
|
9394
|
-
results.push({ scenario, baselineRuns: base.runs.length, output });
|
|
9395
|
-
parts.push(`${scenario}: ${base.runs.length} baseline runs from ${base.branch}${base.device ? ` on ${base.device.model}` : ""}
|
|
9396
|
-
${output.table}`);
|
|
9397
|
-
}
|
|
9398
|
-
return { results, table: parts.join(`
|
|
9399
|
-
|
|
9400
|
-
`) };
|
|
9401
|
-
}
|
|
9402
|
-
|
|
9403
8879
|
// apps/agent/src/defaults.ts
|
|
9404
8880
|
var DEFAULT_API_URL = "https://keepsnappy-api-922918702810.europe-west1.run.app";
|
|
9405
8881
|
|
|
@@ -9485,6 +8961,47 @@ async function pullRequestInfo(deps, env) {
|
|
|
9485
8961
|
const url = typeof pr.html_url === "string" && /^https?:\/\//.test(pr.html_url) ? pr.html_url : undefined;
|
|
9486
8962
|
return number || url ? { ...number ? { number } : {}, ...url ? { url } : {} } : undefined;
|
|
9487
8963
|
}
|
|
8964
|
+
var REMOTE = /^(?:refs\/remotes\/origin\/|refs\/heads\/|origin\/)/;
|
|
8965
|
+
function baseRefs(env) {
|
|
8966
|
+
const forced = env.KEEPSNAPPY_BASE_REF?.trim();
|
|
8967
|
+
if (forced)
|
|
8968
|
+
return [forced];
|
|
8969
|
+
const target = env.GITHUB_BASE_REF?.trim();
|
|
8970
|
+
return target ? [`origin/${target}`] : ["origin/main", "main"];
|
|
8971
|
+
}
|
|
8972
|
+
var SHA = /^[0-9a-f]{40,64}$/;
|
|
8973
|
+
var FIX = "fetch main and the commit, in CI with fetch-depth: 0, or pass --base-commit SHA";
|
|
8974
|
+
async function defaultBaseCommit(deps, env, o) {
|
|
8975
|
+
const refs = baseRefs(env).filter((r) => !r.startsWith("-"));
|
|
8976
|
+
if (o.branch && refs.some((r) => r.replace(REMOTE, "") === o.branch))
|
|
8977
|
+
return {};
|
|
8978
|
+
const tip = o.commit?.trim();
|
|
8979
|
+
if (!tip)
|
|
8980
|
+
return { warning: "base commit not recorded: no commit is recorded for the run set (pass --commit, or run in GitHub Actions)" };
|
|
8981
|
+
const shown = tip.slice(0, 12);
|
|
8982
|
+
if (tip.startsWith("-"))
|
|
8983
|
+
return { warning: `base commit not recorded: "${shown}" is not a commit` };
|
|
8984
|
+
const there = await deps.spawner.run("git", ["rev-parse", "--verify", "--quiet", `${tip}^{commit}`], { timeoutMs: 5000 }).catch(() => null);
|
|
8985
|
+
const tipSha = there && there.code === 0 ? there.stdout.trim() : "";
|
|
8986
|
+
if (!SHA.test(tipSha))
|
|
8987
|
+
return { warning: `base commit not recorded: the commit ${shown} is not in the checkout (${FIX})` };
|
|
8988
|
+
for (const ref of refs) {
|
|
8989
|
+
const r = await deps.spawner.run("git", ["merge-base", ref, tipSha], { timeoutMs: 5000 }).catch(() => null);
|
|
8990
|
+
const sha = r && r.code === 0 ? r.stdout.trim() : "";
|
|
8991
|
+
if (!SHA.test(sha))
|
|
8992
|
+
continue;
|
|
8993
|
+
return sha === tipSha ? {} : { sha };
|
|
8994
|
+
}
|
|
8995
|
+
return { warning: `base commit not recorded: git merge-base ${refs.join(" / ") || "<no ref>"} ${shown} failed (${FIX})` };
|
|
8996
|
+
}
|
|
8997
|
+
async function baseCommitFor(deps, env, o, print) {
|
|
8998
|
+
if (o.explicit)
|
|
8999
|
+
return o.explicit;
|
|
9000
|
+
const d = await defaultBaseCommit(deps, env, o);
|
|
9001
|
+
if (d.warning)
|
|
9002
|
+
print(d.warning);
|
|
9003
|
+
return d.sha;
|
|
9004
|
+
}
|
|
9488
9005
|
var wire = (c) => c.title || c.author || c.committedAt ? { ...c.title ? { title: c.title } : {}, ...c.author ? { author: c.author } : {}, ...c.committedAt ? { committed_at: c.committedAt } : {} } : undefined;
|
|
9489
9006
|
async function commitFields(deps, env, shas) {
|
|
9490
9007
|
const commit = shas.commit ? wire(await commitInfo(deps, shas.commit)) : undefined;
|
|
@@ -9495,6 +9012,12 @@ async function commitFields(deps, env, shas) {
|
|
|
9495
9012
|
|
|
9496
9013
|
// apps/agent/src/run-command.ts
|
|
9497
9014
|
var EXIT = { ok: 0, infra: 1, config: 2, worse: 3 };
|
|
9015
|
+
var RUN_INFO = "keepsnappy-run.json";
|
|
9016
|
+
var LOCAL_VERDICT_REMOVED = "there is no local verdict any more: use --upload; the server judges";
|
|
9017
|
+
function verdictCommand(print = console.error) {
|
|
9018
|
+
print(`keepsnappy: ${LOCAL_VERDICT_REMOVED}`);
|
|
9019
|
+
return EXIT.config;
|
|
9020
|
+
}
|
|
9498
9021
|
function gitContext(flags, env) {
|
|
9499
9022
|
const branch = flags.branch ?? (env.GITHUB_HEAD_REF || undefined) ?? env.GITHUB_REF_NAME;
|
|
9500
9023
|
const commit = flags.commit ?? env.GITHUB_SHA;
|
|
@@ -9549,11 +9072,12 @@ async function runCommand(deps, o, positionals, rt = { env: process.env, print:
|
|
|
9549
9072
|
const replayed = replay ? await replayRuns(deps, replay, out, flows[0].name) : null;
|
|
9550
9073
|
let runSet;
|
|
9551
9074
|
if (api && git.branch) {
|
|
9075
|
+
const base = await baseCommitFor(deps, rt.env, { explicit: o["base-commit"], branch: git.branch, commit: git.commit }, rt.print);
|
|
9552
9076
|
runSet = await createRunSet(api, {
|
|
9553
9077
|
branch: git.branch,
|
|
9554
9078
|
...git.commit ? { commit_sha: git.commit } : {},
|
|
9555
|
-
...
|
|
9556
|
-
...await commitFields(deps, rt.env, { commit: git.commit, base
|
|
9079
|
+
...base ? { base_commit_sha: base } : {},
|
|
9080
|
+
...await commitFields(deps, rt.env, { commit: git.commit, base }),
|
|
9557
9081
|
device: { ...device, provider },
|
|
9558
9082
|
scenarios: flows.map((f) => f.name),
|
|
9559
9083
|
journeys: await journeysFor(flows, { readFile: (p) => deps.fs.readFile(p), root: await repoRoot(deps) }),
|
|
@@ -9584,7 +9108,6 @@ async function runCommand(deps, o, positionals, rt = { env: process.env, print:
|
|
|
9584
9108
|
flows,
|
|
9585
9109
|
outDir: out,
|
|
9586
9110
|
repetitions: repeat,
|
|
9587
|
-
...o.baseline ? { baselineDir: resolve3(o.baseline) } : {},
|
|
9588
9111
|
...markers ? { markers } : {},
|
|
9589
9112
|
...o["clock-offset-ms"] ? { clockOffsetMs: Number(o["clock-offset-ms"]) } : {}
|
|
9590
9113
|
});
|
|
@@ -9594,7 +9117,7 @@ async function runCommand(deps, o, positionals, rt = { env: process.env, print:
|
|
|
9594
9117
|
${outcome.summary}`);
|
|
9595
9118
|
}
|
|
9596
9119
|
const info = { device, ...git.branch ? { branch: git.branch } : {}, ...git.commit ? { commit: git.commit } : {} };
|
|
9597
|
-
await deps.fs.writeFile(
|
|
9120
|
+
await deps.fs.writeFile(join9(out, RUN_INFO), `${JSON.stringify({ ...info, ...runSet ? { run_set_id: runSet.run_set_id } : {} }, null, 2)}
|
|
9598
9121
|
`);
|
|
9599
9122
|
if (!api)
|
|
9600
9123
|
return EXIT.ok;
|
|
@@ -9622,7 +9145,7 @@ ${formatRunSet(summary)}`);
|
|
|
9622
9145
|
}
|
|
9623
9146
|
await writeResult(deps, out, rt, runSet, summary, git.branch);
|
|
9624
9147
|
if (o["fail-on-worse"] && isBlocking(summary)) {
|
|
9625
|
-
rt.print(
|
|
9148
|
+
rt.print(`--fail-on-worse: a journey is ${STATE_WORDS.worse.toLowerCase()} than its baseline and blocks (exit 3)`);
|
|
9626
9149
|
return EXIT.worse;
|
|
9627
9150
|
}
|
|
9628
9151
|
return EXIT.ok;
|
|
@@ -9643,7 +9166,7 @@ var infra = (what) => (e) => {
|
|
|
9643
9166
|
};
|
|
9644
9167
|
async function writeResult(deps, out, rt, runSet, summary, branch) {
|
|
9645
9168
|
const url = summary?.dashboard_url ?? runSet.dashboard_url;
|
|
9646
|
-
await deps.fs.writeFile(
|
|
9169
|
+
await deps.fs.writeFile(join9(out, "keepsnappy-result.json"), `${JSON.stringify({ run_set_id: runSet.run_set_id, dashboard_url: url, state: summary?.verdicts?.state ?? null, summary }, null, 2)}
|
|
9647
9170
|
`);
|
|
9648
9171
|
const step = rt.env.GITHUB_STEP_SUMMARY;
|
|
9649
9172
|
if (step)
|
|
@@ -9764,8 +9287,35 @@ var nodeFs = {
|
|
|
9764
9287
|
var systemClock = { now: () => Date.now() };
|
|
9765
9288
|
var nodeDeps = (log = console.log) => ({ spawner: nodeSpawner, clock: systemClock, fs: nodeFs, log });
|
|
9766
9289
|
|
|
9290
|
+
// apps/agent/src/aa.ts
|
|
9291
|
+
import { join as join10 } from "node:path";
|
|
9292
|
+
var g = (x, d = 4) => x === undefined || !Number.isFinite(x) ? "-" : Number(x.toPrecision(d)).toString();
|
|
9293
|
+
var pad = (rows) => {
|
|
9294
|
+
const w = rows[0]?.map((_, i) => Math.max(...rows.map((r) => (r[i] ?? "").length))) ?? [];
|
|
9295
|
+
return rows.map((r) => r.map((c, i) => c.padEnd(w[i] ?? 0)).join(" ").trimEnd()).join(`
|
|
9296
|
+
`);
|
|
9297
|
+
};
|
|
9298
|
+
async function load(deps, outDir) {
|
|
9299
|
+
return readRunDataset(await deps.fs.readFile(join10(outDir, "metrics.csv")));
|
|
9300
|
+
}
|
|
9301
|
+
async function aa(deps, outDir, opts = {}) {
|
|
9302
|
+
const dataset = await load(deps, outDir);
|
|
9303
|
+
const results = aaRuns({ dataset, autoCount: true, config: { margin: relativeMargin(0.05), ...opts.config } });
|
|
9304
|
+
const rows = [["metric", "n", "median", "robustCV", "MDE@n", "suggested margin"]];
|
|
9305
|
+
for (const r of results) {
|
|
9306
|
+
if (r.skipped)
|
|
9307
|
+
rows.push([r.metric, String(r.pool.n), "-", "-", "-", r.skipped]);
|
|
9308
|
+
else
|
|
9309
|
+
rows.push([r.metric, String(r.pool.n), g(r.pool.median), g(r.pool.robustCv, 3), r.rows.map((x) => `${x.n}:${x.mde === null ? ">max" : g(x.mde, 2)}`).join(" "), r.suggestedMargin.reason]);
|
|
9310
|
+
}
|
|
9311
|
+
const table = pad(rows);
|
|
9312
|
+
await deps.fs.writeFile(join10(outDir, "aa.json"), `${JSON.stringify({ kind: "aa", results }, null, 2)}
|
|
9313
|
+
`);
|
|
9314
|
+
return { results, table };
|
|
9315
|
+
}
|
|
9316
|
+
|
|
9767
9317
|
// apps/agent/src/farm.ts
|
|
9768
|
-
import { dirname as dirname3, join as
|
|
9318
|
+
import { dirname as dirname3, join as join12, resolve as resolve5 } from "node:path";
|
|
9769
9319
|
|
|
9770
9320
|
// packages/appium-driver/dist/src/links.js
|
|
9771
9321
|
var ALERT_POLL_MS = 250;
|
|
@@ -9785,7 +9335,7 @@ function linkArgs(args = "") {
|
|
|
9785
9335
|
return [...fields].map(([k, v]) => `${k}=${v}`).join("&");
|
|
9786
9336
|
}
|
|
9787
9337
|
var activationLink = (scheme, token, args = "") => `${route(scheme)}?activate=1&${linkArgs(args)}&runToken=${token}`;
|
|
9788
|
-
var flushLink = (scheme) => `${route(scheme)}?flush=1&activate=0`;
|
|
9338
|
+
var flushLink = (scheme, ack) => `${route(scheme)}?flush=1&activate=0${ack === undefined ? "" : `&ack=${ack}`}`;
|
|
9789
9339
|
var gcLink = (scheme, n) => `${route(scheme)}?cmd=gc&n=${n}`;
|
|
9790
9340
|
|
|
9791
9341
|
// packages/appium-driver/dist/src/selectors.js
|
|
@@ -9981,9 +9531,9 @@ class Screen {
|
|
|
9981
9531
|
var modelOf2 = (device) => device.replace(/-[^-]*$/, "");
|
|
9982
9532
|
var versionOf = (device) => device.replace(/^.*-/, "");
|
|
9983
9533
|
var FLUSH_ID = "keepsnappy.flushed";
|
|
9984
|
-
function parseBroadcastAnswer(out, n) {
|
|
9534
|
+
function parseBroadcastAnswer(out, n, cmd = "gc") {
|
|
9985
9535
|
const data = /data="([^"]*)"/.exec(String(out ?? ""))?.[1];
|
|
9986
|
-
return data !== undefined && data.startsWith(
|
|
9536
|
+
return data !== undefined && data.startsWith(`${cmd}:${n} `) ? data : null;
|
|
9987
9537
|
}
|
|
9988
9538
|
var NO_WAIT_MS = 0;
|
|
9989
9539
|
var NEW_COMMAND_TIMEOUT_S = 300;
|
|
@@ -10024,6 +9574,7 @@ var ANDROID = {
|
|
|
10024
9574
|
flush: {
|
|
10025
9575
|
lookup: { using: "-android uiautomator", value: `new UiSelector().description("${FLUSH_ID}")` },
|
|
10026
9576
|
statusAttribute: "text",
|
|
9577
|
+
detailAttribute: "hint",
|
|
10027
9578
|
fallback: { lookup: { using: "xpath", value: `//*[@content-desc="${FLUSH_ID}"]` }, setting: "allowInvisibleElements" }
|
|
10028
9579
|
},
|
|
10029
9580
|
async readScreen(driver) {
|
|
@@ -10040,16 +9591,18 @@ var ANDROID = {
|
|
|
10040
9591
|
singleFindFirst: true,
|
|
10041
9592
|
anyOfIds: androidAnyOfIds,
|
|
10042
9593
|
uploadFields: {},
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
action: `${driver.bundleId}.keepsnappy.COMMAND`,
|
|
10046
|
-
package: driver.bundleId,
|
|
10047
|
-
extras: [["s", "cmd", "gc"], ["i", "n", String(n)]],
|
|
10048
|
-
flags: "0x10000000"
|
|
10049
|
-
});
|
|
10050
|
-
return parseBroadcastAnswer(out, n);
|
|
10051
|
-
}
|
|
9594
|
+
gcCommand: (driver, n) => androidCommand(driver, "gc", n),
|
|
9595
|
+
memReadCommand: (driver, n) => androidCommand(driver, "memread", n)
|
|
10052
9596
|
};
|
|
9597
|
+
async function androidCommand(driver, cmd, n) {
|
|
9598
|
+
const out = await driver.execute("mobile: broadcast", {
|
|
9599
|
+
action: `${driver.bundleId}.keepsnappy.COMMAND`,
|
|
9600
|
+
package: driver.bundleId,
|
|
9601
|
+
extras: [["s", "cmd", cmd], ["i", "n", String(n)]],
|
|
9602
|
+
flags: "0x10000000"
|
|
9603
|
+
});
|
|
9604
|
+
return parseBroadcastAnswer(out, n, cmd);
|
|
9605
|
+
}
|
|
10053
9606
|
|
|
10054
9607
|
// packages/appium-driver/dist/src/webdriver.js
|
|
10055
9608
|
class WdError extends Error {
|
|
@@ -10063,18 +9616,19 @@ class WdError extends Error {
|
|
|
10063
9616
|
var SDK_FLUSH_WORST_CASE_MS = 315000;
|
|
10064
9617
|
var FLUSH_MARGIN_MS = 30000;
|
|
10065
9618
|
var FLUSH_WAIT_MS = SDK_FLUSH_WORST_CASE_MS + FLUSH_MARGIN_MS;
|
|
9619
|
+
var FLUSH_STATUSES = new Set(["ok", "error", "off"]);
|
|
10066
9620
|
// packages/appium-driver/dist/src/version.js
|
|
10067
9621
|
import { createHash } from "node:crypto";
|
|
10068
9622
|
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
10069
|
-
import { dirname as dirname2, join as
|
|
9623
|
+
import { dirname as dirname2, join as join11, relative as relative2, resolve as resolve4 } from "node:path";
|
|
10070
9624
|
import { fileURLToPath } from "node:url";
|
|
10071
9625
|
function driverVersion() {
|
|
10072
|
-
const built = "0.1.0-alpha.
|
|
9626
|
+
const built = "0.1.0-alpha.14";
|
|
10073
9627
|
if (built)
|
|
10074
9628
|
return built;
|
|
10075
9629
|
let dir = dirname2(fileURLToPath(import.meta.url));
|
|
10076
9630
|
for (let i = 0;i < 3; i++, dir = dirname2(dir)) {
|
|
10077
|
-
const file =
|
|
9631
|
+
const file = join11(dir, "package.json");
|
|
10078
9632
|
if (!existsSync(file))
|
|
10079
9633
|
continue;
|
|
10080
9634
|
try {
|
|
@@ -10118,14 +9672,10 @@ function journeyVersion(modulePath) {
|
|
|
10118
9672
|
hash.update(key).update("\x00").update(text).update("\x00");
|
|
10119
9673
|
return hash.digest("hex").slice(0, 12);
|
|
10120
9674
|
}
|
|
10121
|
-
// packages/verdicts/dist/src/heat.js
|
|
10122
|
-
var HEAT_METRIC = "sampler.thermal.heats_up";
|
|
10123
|
-
var HEAT_LABEL = MEASURES[HEAT_METRIC].label;
|
|
10124
|
-
|
|
10125
9675
|
// apps/agent/src/slack.ts
|
|
10126
9676
|
var COLOR = { error: "#E01E5A", warn: "#ECB22E", better: "#2EB67D", none: "#9AA0A6" };
|
|
10127
9677
|
var MAX_ROWS = 3;
|
|
10128
|
-
var NAME_W =
|
|
9678
|
+
var NAME_W = 30;
|
|
10129
9679
|
var COLUMN_GAP = 1;
|
|
10130
9680
|
function metricWord(item) {
|
|
10131
9681
|
if (item.label_text)
|
|
@@ -10153,15 +9703,12 @@ var worseOrder = (a, b) => Number(a.grade === "warn") - Number(b.grade === "warn
|
|
|
10153
9703
|
var isWorse = (i) => i.state === "worse";
|
|
10154
9704
|
var isBetter = (i) => i.state === "better";
|
|
10155
9705
|
var differs = (sc) => sc?.headline?.measured_differently ?? [];
|
|
10156
|
-
var
|
|
9706
|
+
var scenarioBlocks = (sc) => sc?.headline?.blocking !== false;
|
|
9707
|
+
var isRed = (i, sc) => i.grade !== "warn" && scenarioBlocks(sc);
|
|
10157
9708
|
var mark = (i, sc) => isBetter(i) ? "\uD83D\uDFE2" : isRed(i, sc) ? "\uD83D\uDD34" : "\uD83D\uDFE0";
|
|
10158
9709
|
var fit = (s, w) => s.length > w ? `${s.slice(0, w - 1)}…` : s.padEnd(w);
|
|
10159
9710
|
var right = (s, w) => s.padStart(w);
|
|
10160
9711
|
var phone = (sc) => sc?.device ? `${sc.device.model}${sc.device.os ? ` · ${sc.device.os}` : ""}` : "unknown phone";
|
|
10161
|
-
var kindWord = (step) => {
|
|
10162
|
-
const k = placeKindWord(step ?? "");
|
|
10163
|
-
return k === "Window" || k === PLACE_KIND_WORDS.journey ? "" : `${k.toLowerCase()} · `;
|
|
10164
|
-
};
|
|
10165
9712
|
function commitLine(ctx) {
|
|
10166
9713
|
const c = ctx.commit;
|
|
10167
9714
|
if (!c)
|
|
@@ -10177,8 +9724,8 @@ function prTable(items, sc) {
|
|
|
10177
9724
|
const w = [NAME_W, widest("main", (c) => c.main), widest("PR", (c) => c.pr), widest("change", (c) => c.change)];
|
|
10178
9725
|
const head = `${" ".repeat(3)}${fit("", w[0])}${right("main", w[1])}${right("PR", w[2])}${right("change", w[3])}`;
|
|
10179
9726
|
const rows = cells.flatMap((c) => [
|
|
10180
|
-
`${mark(c.i, sc)} ${fit(
|
|
10181
|
-
` ${
|
|
9727
|
+
`${mark(c.i, sc)} ${fit(placeLabel(c.i.step ?? "*"), w[0])}${right(c.main, w[1])}${right(c.pr, w[2])}${right(c.change, w[3])}`,
|
|
9728
|
+
` ${metricWord(c.i)}`
|
|
10182
9729
|
]);
|
|
10183
9730
|
return ["```", head, ...rows, "```"].join(`
|
|
10184
9731
|
`);
|
|
@@ -10187,7 +9734,7 @@ function mainCard(i) {
|
|
|
10187
9734
|
const label = (s) => ` ${s.padEnd(16)}`;
|
|
10188
9735
|
return [
|
|
10189
9736
|
"```",
|
|
10190
|
-
|
|
9737
|
+
placeLabel(i.step ?? "*"),
|
|
10191
9738
|
`${label(metricWord(i))}${right(fieldValueText(i.metric, i.candidate), 9)}`,
|
|
10192
9739
|
`${label("usually")}${right(fieldValueText(i.metric, i.baseline), 9)}`,
|
|
10193
9740
|
`${label("change")}${right(change2(i), 9)}`,
|
|
@@ -10200,7 +9747,7 @@ function blockedMessage(s, ctx) {
|
|
|
10200
9747
|
const blocks = [
|
|
10201
9748
|
section(`*${text}*
|
|
10202
9749
|
${ctx.blocked?.reason ?? "No iteration sent any data."}
|
|
10203
|
-
Nothing was judged, so this is not a "no
|
|
9750
|
+
Nothing was judged, so this is not a "${STATE_WORDS["no-change"].toLowerCase()}".`),
|
|
10204
9751
|
...commitLine(ctx) ? [context(commitLine(ctx))] : [],
|
|
10205
9752
|
...s ? [section(`\uD83D\uDC49 <${s.dashboard_url}|See what happened>`)] : []
|
|
10206
9753
|
];
|
|
@@ -10212,21 +9759,24 @@ function slackMessage(s, ctx) {
|
|
|
10212
9759
|
const v = s?.verdicts;
|
|
10213
9760
|
if (!s || !v)
|
|
10214
9761
|
return null;
|
|
10215
|
-
const items = v.scenarios.flatMap((sc) => movedItems(v, sc));
|
|
9762
|
+
const items = v.scenarios.flatMap((sc) => sc.state === "inconclusive" ? [] : movedItems(v, sc));
|
|
10216
9763
|
const scenarioOf = (i) => v.scenarios.find((sc) => sc.scenario === i.scenario && (!i.device_id || sc.device?.id === i.device_id)) ?? v.scenarios.find((sc) => sc.scenario === i.scenario);
|
|
10217
9764
|
const isPr = v.scenarios.some((sc) => sc.role === "comparison");
|
|
10218
9765
|
const worse = items.filter(isWorse).sort(worseOrder);
|
|
10219
9766
|
const better = items.filter(isBetter).sort(bySize);
|
|
10220
9767
|
const commit = commitLine(ctx);
|
|
9768
|
+
const errorRows = worse.filter((i) => i.grade !== "warn");
|
|
9769
|
+
const warningRows = worse.filter((i) => i.grade === "warn");
|
|
9770
|
+
const errors = placeCount(errorRows);
|
|
9771
|
+
const warns = placeCount(worse) - errors;
|
|
10221
9772
|
if (isPr) {
|
|
10222
9773
|
if (!worse.length)
|
|
10223
9774
|
return null;
|
|
10224
|
-
const errors = worse.filter((i) => i.grade !== "warn").length;
|
|
10225
|
-
const warns = worse.length - errors;
|
|
10226
9775
|
const red = worse.some((i) => isRed(i, scenarioOf(i)));
|
|
10227
9776
|
const graded = worse.some((i) => i.grade);
|
|
10228
9777
|
const notBlocking = errors > 0 && !red;
|
|
10229
|
-
const
|
|
9778
|
+
const places = (word, n) => `${word.toLowerCase()} in ${n} place${n > 1 ? "s" : ""}`;
|
|
9779
|
+
const counts = graded ? [errors ? `${places(STATE_WORDS.worse, errors)}${notBlocking ? ` (${NOT_BLOCKING})` : ""}` : "", warns ? places(STATE_WORDS.warning, warns) : ""].filter(Boolean).join(", ") : places(STATE_WORDS.worse, placeCount(worse));
|
|
10230
9780
|
const head = `${red ? "\uD83D\uDD34" : "\uD83D\uDFE0"} ${ctx.branch}: ${counts}`;
|
|
10231
9781
|
const blocks = [section(`*${head}*`)];
|
|
10232
9782
|
const groups = new Map;
|
|
@@ -10242,7 +9792,7 @@ function slackMessage(s, ctx) {
|
|
|
10242
9792
|
const nC = Math.max(...rows.map((r) => r.n_candidate ?? 0)) || (sc?.runs?.valid ?? 0);
|
|
10243
9793
|
const runs = nB && nC ? ` · ${nC} PR run${nC === 1 ? "" : "s"} vs ${nB} main run${nB === 1 ? "" : "s"}` : "";
|
|
10244
9794
|
const note = differs(sc).length ? `
|
|
10245
|
-
_Measured differently (${differs(sc).join(", ")}) — ${NOT_BLOCKING}_` : "";
|
|
9795
|
+
_Measured differently (${differs(sc).join(", ")})${scenarioBlocks(sc) ? "" : ` — ${NOT_BLOCKING}`}_` : "";
|
|
10246
9796
|
blocks.push(section(`*${sc?.scenario ?? "journey"}* · ${phone(sc)}${runs}${note}
|
|
10247
9797
|
${prTable(rows, sc)}`));
|
|
10248
9798
|
}
|
|
@@ -10254,15 +9804,15 @@ ${prTable(rows, sc)}`));
|
|
|
10254
9804
|
}
|
|
10255
9805
|
if (!worse.length && !better.length && !ctx.always)
|
|
10256
9806
|
return null;
|
|
10257
|
-
const parts = [worse.
|
|
10258
|
-
const quiet = (v.state === null || v.state === "inconclusive") && v.word
|
|
9807
|
+
const parts = [errors ? `${errors} ${STATE_WORDS.worse.toLowerCase()}` : "", warns ? `${warns} ${STATE_WORDS.warning.toLowerCase()}` : "", better.length ? `${placeCount(better)} ${STATE_WORDS.better.toLowerCase()}` : ""].filter(Boolean);
|
|
9808
|
+
const quiet = (v.state === null || v.state === "inconclusive") && v.word ? v.word : STATE_WORDS["no-change"].toLowerCase();
|
|
10259
9809
|
const status = parts.length ? `${ctx.branch}: ${parts.join(", ")}` : quiet.startsWith("Main: ") && ctx.branch === "main" ? quiet : `${ctx.branch}: ${quiet}`;
|
|
10260
|
-
const head = `${worse.length ? worse.some((i) => i
|
|
9810
|
+
const head = `${worse.length ? worse.some((i) => isRed(i, scenarioOf(i))) ? "\uD83D\uDD34" : "\uD83D\uDFE0" : better.length ? "\uD83D\uDFE2" : "⚪"} ${status}`;
|
|
10261
9811
|
const journeys = new Set(v.scenarios.map((sc) => `${sc.scenario}|${sc.device?.id}`)).size;
|
|
10262
9812
|
const blocks = [section(`*${head}*
|
|
10263
9813
|
${journeys} journey${journeys === 1 ? "" : "s"} checked`)];
|
|
10264
9814
|
let shown = 0;
|
|
10265
|
-
for (const [title, list] of [[
|
|
9815
|
+
for (const [title, list] of [[STATE_WORDS.worse, errorRows], [STATE_WORDS.warning, warningRows], [STATE_WORDS.better, better]]) {
|
|
10266
9816
|
for (const i of list) {
|
|
10267
9817
|
if (shown >= MAX_ROWS)
|
|
10268
9818
|
break;
|
|
@@ -10276,7 +9826,7 @@ ${mainCard(i)}`));
|
|
|
10276
9826
|
if (commit)
|
|
10277
9827
|
blocks.push(context(commit));
|
|
10278
9828
|
blocks.push(section(`\uD83D\uDC49 <${s.dashboard_url}|Open KeepSnappy>${more > 0 ? ` · ${more} more changes` : ""}`));
|
|
10279
|
-
const color = worse.length ? worse.some((i) => i
|
|
9829
|
+
const color = worse.length ? worse.some((i) => isRed(i, scenarioOf(i))) ? COLOR.error : COLOR.warn : better.length ? COLOR.better : COLOR.none;
|
|
10280
9830
|
return { text: head, attachments: [{ color, blocks }] };
|
|
10281
9831
|
}
|
|
10282
9832
|
async function postToSlack(url, payload, fetchImpl = fetch) {
|
|
@@ -10459,11 +10009,12 @@ async function farmOpen(deps, f, rt) {
|
|
|
10459
10009
|
const journeyInfo = await farmJourneys(deps, f, rt, journeys);
|
|
10460
10010
|
const withInstrument = { ...f, ...instrumentFlags(f) };
|
|
10461
10011
|
const setup = Object.fromEntries(SETUP_FLAGS.flatMap(([flag, key]) => withInstrument[flag]?.trim() ? [[key, withInstrument[flag].trim().slice(0, 200)]] : []));
|
|
10012
|
+
const base = await baseCommitFor(deps, rt.env, { explicit: f["base-commit"], branch: git.branch, commit: git.commit }, rt.print);
|
|
10462
10013
|
const rs = await createRunSet(api, {
|
|
10463
10014
|
branch: git.branch,
|
|
10464
10015
|
...git.commit ? { commit_sha: git.commit } : {},
|
|
10465
|
-
...
|
|
10466
|
-
...await commitFields(deps, rt.env, { commit: git.commit, base
|
|
10016
|
+
...base ? { base_commit_sha: base } : {},
|
|
10017
|
+
...await commitFields(deps, rt.env, { commit: git.commit, base }),
|
|
10467
10018
|
device: { model: device.model, ...device.os ? { os: device.os } : {}, provider: device.provider, ...f["device-id"]?.trim() ? { id: f["device-id"].trim() } : {} },
|
|
10468
10019
|
scenarios: journeys,
|
|
10469
10020
|
journeys: journeyInfo,
|
|
@@ -10509,7 +10060,7 @@ async function farmOpen(deps, f, rt) {
|
|
|
10509
10060
|
dashboard_url: rs.dashboard_url,
|
|
10510
10061
|
branch: git.branch,
|
|
10511
10062
|
commit: git.commit ?? null,
|
|
10512
|
-
base_commit:
|
|
10063
|
+
base_commit: base ?? null,
|
|
10513
10064
|
device,
|
|
10514
10065
|
journeys,
|
|
10515
10066
|
repeat,
|
|
@@ -10712,7 +10263,7 @@ async function closeRunSet(deps, f, rt, state, statePath) {
|
|
|
10712
10263
|
await deps.fs.writeFile(statePath, `${JSON.stringify(state2, null, 2)}
|
|
10713
10264
|
`);
|
|
10714
10265
|
await deps.fs.mkdir(out);
|
|
10715
|
-
await deps.fs.writeFile(
|
|
10266
|
+
await deps.fs.writeFile(join12(out, "keepsnappy-result.json"), `${JSON.stringify({ run_set_id: state.run_set_id, dashboard_url: summary.dashboard_url, state: state2.result.state, session: { status: session.status, message: session.message }, summary }, null, 2)}
|
|
10716
10267
|
`);
|
|
10717
10268
|
await githubOutputs(deps, rt, { state: state2.result.state ?? "none", dashboard_url: summary.dashboard_url, run_set_id: state.run_set_id });
|
|
10718
10269
|
const step = rt.env.GITHUB_STEP_SUMMARY;
|
|
@@ -10740,7 +10291,7 @@ ${tail.join(" ")}
|
|
|
10740
10291
|
always: envValue(rt.env, "SLACK_ALWAYS") === "1"
|
|
10741
10292
|
});
|
|
10742
10293
|
if (!msg)
|
|
10743
|
-
rt.print(f.wait ?
|
|
10294
|
+
rt.print(f.wait ? `Slack: nothing to report (${STATE_WORDS["no-change"].toLowerCase()})` : "Slack: skipped, the message needs --wait");
|
|
10744
10295
|
else {
|
|
10745
10296
|
const r = await postToSlack(hook, msg, rt.fetch ?? fetch);
|
|
10746
10297
|
rt.print(r.ok ? "Slack: posted" : `Slack: not posted (${r.error}); the job is not affected`);
|
|
@@ -10749,7 +10300,7 @@ ${tail.join(" ")}
|
|
|
10749
10300
|
if (blocked)
|
|
10750
10301
|
return EXIT.infra;
|
|
10751
10302
|
if (f["fail-on-worse"] && isBlocking(summary)) {
|
|
10752
|
-
rt.print(
|
|
10303
|
+
rt.print(`--fail-on-worse: a journey is ${STATE_WORDS.worse.toLowerCase()} than its baseline and blocks (exit 3)`);
|
|
10753
10304
|
return EXIT.worse;
|
|
10754
10305
|
}
|
|
10755
10306
|
return EXIT.ok;
|
|
@@ -10758,11 +10309,11 @@ ${tail.join(" ")}
|
|
|
10758
10309
|
// apps/agent/src/farm-run.ts
|
|
10759
10310
|
import { existsSync as existsSync2 } from "node:fs";
|
|
10760
10311
|
import { createRequire } from "node:module";
|
|
10761
|
-
import { dirname as dirname4, join as
|
|
10312
|
+
import { dirname as dirname4, join as join15, resolve as resolve6 } from "node:path";
|
|
10762
10313
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
10763
10314
|
|
|
10764
10315
|
// apps/agent/src/app-source.ts
|
|
10765
|
-
import { join as
|
|
10316
|
+
import { join as join13 } from "node:path";
|
|
10766
10317
|
var EAS_CLI_VERSION = "24.6.0";
|
|
10767
10318
|
var isEas = (spec) => spec.startsWith("eas:");
|
|
10768
10319
|
|
|
@@ -10798,7 +10349,7 @@ class DefaultAppResolver {
|
|
|
10798
10349
|
const url = build.artifacts?.buildUrl ?? build.artifacts?.applicationArchiveUrl;
|
|
10799
10350
|
if (!url)
|
|
10800
10351
|
throw new Error(`EAS build ${id} has no downloadable artifact (is it finished?)`);
|
|
10801
|
-
const file =
|
|
10352
|
+
const file = join13(o.outDir, `app-${id.slice(0, 8)}${build.platform === "ANDROID" ? ".apk" : ".ipa"}`);
|
|
10802
10353
|
await this.deps.fs.mkdir(o.outDir);
|
|
10803
10354
|
const dl = await this.deps.spawner.run("curl", ["-fsSL", "-o", file, url]);
|
|
10804
10355
|
if (dl.code !== 0)
|
|
@@ -10809,7 +10360,7 @@ class DefaultAppResolver {
|
|
|
10809
10360
|
|
|
10810
10361
|
// apps/agent/src/browserstack.ts
|
|
10811
10362
|
import { randomUUID } from "node:crypto";
|
|
10812
|
-
import { basename as basename2, join as
|
|
10363
|
+
import { basename as basename2, join as join14 } from "node:path";
|
|
10813
10364
|
import { readFile as readFile3 } from "node:fs/promises";
|
|
10814
10365
|
var BASE = "https://api-cloud.browserstack.com/app-automate/maestro/v2";
|
|
10815
10366
|
var TERMINAL = new Set(["passed", "failed", "error", "timedout", "skipped", "stopped"]);
|
|
@@ -10893,7 +10444,7 @@ class BrowserStack {
|
|
|
10893
10444
|
}
|
|
10894
10445
|
async runMaestro(b) {
|
|
10895
10446
|
const appUrl = await this.uploadApp(b.app, b.customId);
|
|
10896
|
-
const work =
|
|
10447
|
+
const work = join14(this.o.tmpDir ?? "/tmp", `keepsnappy-suite-${(this.o.uniqueId ?? randomUUID)()}`);
|
|
10897
10448
|
let suiteUrl;
|
|
10898
10449
|
try {
|
|
10899
10450
|
const zip = await this.zipSuite(b, work);
|
|
@@ -10926,17 +10477,17 @@ class BrowserStack {
|
|
|
10926
10477
|
return this.poll(buildId, b);
|
|
10927
10478
|
}
|
|
10928
10479
|
async zipSuite(b, work) {
|
|
10929
|
-
const zipPath =
|
|
10930
|
-
const stage =
|
|
10480
|
+
const zipPath = join14(work, "suite.zip");
|
|
10481
|
+
const stage = join14(work, "stage");
|
|
10931
10482
|
await this.deps.fs.mkdir(stage);
|
|
10932
10483
|
const cp = await this.deps.spawner.run("cp", ["-R", b.suiteDir.replace(/\/$/, ""), stage]);
|
|
10933
10484
|
if (cp.code !== 0)
|
|
10934
10485
|
throw new FarmApiError(`cannot copy the suite ${b.suiteDir}: ${cp.stderr || cp.error}`);
|
|
10935
10486
|
const folder = basename2(b.suiteDir.replace(/\/$/, ""));
|
|
10936
|
-
if (!await this.deps.fs.exists(
|
|
10487
|
+
if (!await this.deps.fs.exists(join14(stage, folder, "config.yaml")))
|
|
10937
10488
|
throw new FarmApiError(`${b.suiteDir} has no config.yaml: is that the Maestro workspace?`);
|
|
10938
|
-
const target =
|
|
10939
|
-
await this.deps.fs.mkdir(
|
|
10489
|
+
const target = join14(stage, folder, b.tokensPath);
|
|
10490
|
+
await this.deps.fs.mkdir(join14(target, ".."));
|
|
10940
10491
|
await this.deps.fs.writeFile(target, await this.deps.fs.readFile(b.tokensJs));
|
|
10941
10492
|
const include = SUITE_EXTENSIONS.flatMap((e) => ["-i", `${folder}/*${e}`, "-i", `${folder}/**/*${e}`]);
|
|
10942
10493
|
const z = await this.deps.spawner.run("sh", ["-c", 'cd "$1" && shift && zip -qr "$@"', "sh", stage, zipPath, folder, ...include, "-x", `${folder}/runs/*`, "-x", "*/node_modules/*"]);
|
|
@@ -10995,9 +10546,9 @@ class BrowserStackLocal {
|
|
|
10995
10546
|
const url = TUNNEL_ZIP[this.o.platform ?? process.platform];
|
|
10996
10547
|
if (!url)
|
|
10997
10548
|
throw new FarmApiError(`no BrowserStackLocal binary known for ${this.o.platform ?? process.platform}`);
|
|
10998
|
-
const dir =
|
|
10999
|
-
const bin =
|
|
11000
|
-
const zip =
|
|
10549
|
+
const dir = join14(this.o.dir, identifier);
|
|
10550
|
+
const bin = join14(dir, "BrowserStackLocal");
|
|
10551
|
+
const zip = join14(dir, "bs-local.zip");
|
|
11001
10552
|
await this.deps.fs.mkdir(dir);
|
|
11002
10553
|
for (const [cmd, args] of [
|
|
11003
10554
|
["curl", ["-fsSL", "-o", zip, url]],
|
|
@@ -11031,12 +10582,12 @@ ${r.stderr}`;
|
|
|
11031
10582
|
// apps/agent/src/farm-run.ts
|
|
11032
10583
|
function appiumDriverEntry(o = {}) {
|
|
11033
10584
|
const exists = o.exists ?? existsSync2;
|
|
11034
|
-
const bundled =
|
|
10585
|
+
const bundled = join15(o.selfDir ?? dirname4(fileURLToPath2(import.meta.url)), "appium-driver.js");
|
|
11035
10586
|
if (exists(bundled))
|
|
11036
10587
|
return bundled;
|
|
11037
10588
|
const pkgDir = dirname4(o.packageJson ? o.packageJson() : createRequire(import.meta.url).resolve("@keepsnappy/appium-driver/package.json"));
|
|
11038
|
-
const built =
|
|
11039
|
-
return exists(built) ? built :
|
|
10589
|
+
const built = join15(pkgDir, "dist", "src", "cli.js");
|
|
10590
|
+
return exists(built) ? built : join15(pkgDir, "src", "cli.ts");
|
|
11040
10591
|
}
|
|
11041
10592
|
var driverRuntime = (entry) => entry.endsWith(".ts") ? process.execPath : "node";
|
|
11042
10593
|
function nodeFarmRunPorts(deps, rt) {
|
|
@@ -11045,7 +10596,7 @@ function nodeFarmRunPorts(deps, rt) {
|
|
|
11045
10596
|
if (!user || !key)
|
|
11046
10597
|
throw new ConfigError(["BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY must be set"]);
|
|
11047
10598
|
const tmp = rt.env.RUNNER_TEMP ?? rt.env.TMPDIR ?? "/tmp";
|
|
11048
|
-
const dir =
|
|
10599
|
+
const dir = join15(tmp, "keepsnappy-bs-local");
|
|
11049
10600
|
return {
|
|
11050
10601
|
apps: new DefaultAppResolver(deps, rt.env),
|
|
11051
10602
|
provider: new BrowserStack(deps, { credentials: { user, key }, tmpDir: tmp, ...rt.fetch ? { fetch: rt.fetch } : {}, ...rt.sleep ? { sleep: rt.sleep } : {}, ...rt.now ? { now: rt.now } : {} }),
|
|
@@ -11158,7 +10709,7 @@ function printPlan(p, f, rt) {
|
|
|
11158
10709
|
line("keepsnappy farm run: dry run, no request is made and nothing is written");
|
|
11159
10710
|
line(` 1. validate: ${p.journeys.join(", ")} x ${p.repeat} on ${p.device} (${p.platform}), driver ${p.driver}`);
|
|
11160
10711
|
line(` 2. resolve the app: ${p.app.startsWith("eas:") ? `eas-cli build:view ${p.app.slice(4)}, download into ${p.out}` : p.app.startsWith("bs://") ? "bs:// passes through" : `${p.app} (local file)`}`);
|
|
11161
|
-
line(` 3. farm open -> ${
|
|
10712
|
+
line(` 3. farm open -> ${join15(p.out, "keepsnappy-tokens.js")}, ${join15(p.out, "keepsnappy-farm.json")} (POST /v1/run-sets, then POST /v1/runs x ${p.journeys.length * p.repeat})`);
|
|
11162
10713
|
line(` 4. tunnel: ${p.ports.length ? `BrowserStackLocal for bs-local.com ports ${p.ports.join(", ")}, identifier = run id` : "none (no --tunnel-for)"}`);
|
|
11163
10714
|
if (p.driver === "maestro") {
|
|
11164
10715
|
line(` 5. maestro on BrowserStack: upload app, zip ${f["suite-dir"]} with the tokens at ${f["tokens-path"] ?? "flows/keepsnappy/tokens.js"}, upload suite, POST build (execute ${f.execute}), poll`);
|
|
@@ -11167,10 +10718,11 @@ function printPlan(p, f, rt) {
|
|
|
11167
10718
|
}
|
|
11168
10719
|
for (const [k, v] of p.env)
|
|
11169
10720
|
line(` env ${k}=${substitute(v, {})}`);
|
|
11170
|
-
line(` 6. farm close${f.wait ? " --wait" : ""}${f["fail-on-worse"] ? " --fail-on-worse" : ""} with ${
|
|
10721
|
+
line(` 6. farm close${f.wait ? " --wait" : ""}${f["fail-on-worse"] ? " --fail-on-worse" : ""} with ${join15(p.out, `browserstack.${p.platform}.json`)}`);
|
|
11171
10722
|
line(" 7. always: stop the tunnel, put the empty stub back over the tokens file");
|
|
11172
10723
|
line(" exit: 0 any verdict, 1 infrastructure / driver failed / Blocked, 2 configuration, 3 --fail-on-worse");
|
|
11173
10724
|
}
|
|
10725
|
+
var ANNOTATION = /^::(error|warning|notice)( [^:]*)?::/;
|
|
11174
10726
|
async function runDriver2(deps, rt, f, p, ports, ctx) {
|
|
11175
10727
|
const env = Object.fromEntries(p.env.map(([k, v]) => [k, substitute(v, ctx)]));
|
|
11176
10728
|
if (p.driver === "maestro") {
|
|
@@ -11190,13 +10742,13 @@ async function runDriver2(deps, rt, f, p, ports, ctx) {
|
|
|
11190
10742
|
timeoutMinutes: p.timeoutMinutes,
|
|
11191
10743
|
pollSeconds: p.pollSeconds,
|
|
11192
10744
|
onStarted: async (buildId) => {
|
|
11193
|
-
await deps.fs.writeFile(
|
|
10745
|
+
await deps.fs.writeFile(join15(ctx.out, `browserstack.${p.platform}.json`), `${JSON.stringify({ buildId, dashboard: `https://app-automate.browserstack.com/dashboard/v2/builds/${buildId}`, note: "written at submit; replaced by the full detail once the build ends", status: "unknown" }, null, 2)}
|
|
11194
10746
|
`);
|
|
11195
10747
|
rt.print(`build ${buildId} started: https://app-automate.browserstack.com/dashboard/v2/builds/${buildId}`);
|
|
11196
10748
|
},
|
|
11197
10749
|
onStatus: (buildId, status) => rt.print(`build ${buildId}: ${status}`)
|
|
11198
10750
|
});
|
|
11199
|
-
await deps.fs.writeFile(
|
|
10751
|
+
await deps.fs.writeFile(join15(ctx.out, `browserstack.${p.platform}.json`), `${redact(JSON.stringify(detail.detail, null, 2), secretsOf(p, f, rt.env))}
|
|
11200
10752
|
`);
|
|
11201
10753
|
return detail.status === "passed" ? { ok: true, message: null } : { ok: false, message: `the device build ${detail.buildId} ended ${detail.status}` };
|
|
11202
10754
|
}
|
|
@@ -11229,7 +10781,7 @@ async function runDriver2(deps, rt, f, p, ports, ctx) {
|
|
|
11229
10781
|
const secrets = secretsOf(p, f, rt.env);
|
|
11230
10782
|
let streamed = false;
|
|
11231
10783
|
let pending = "";
|
|
11232
|
-
const emit = (l) => l.trim() && rt.print(` driver: ${redact(l, secrets)}`);
|
|
10784
|
+
const emit = (l) => l.trim() && rt.print(ANNOTATION.test(l) ? redact(l, secrets) : ` driver: ${redact(l, secrets)}`);
|
|
11233
10785
|
const r = await deps.spawner.run(f["driver-script"] ? "node" : driverRuntime(entry), args, {
|
|
11234
10786
|
env,
|
|
11235
10787
|
timeoutMs: p.timeoutMinutes * 60000,
|
|
@@ -11251,7 +10803,7 @@ async function runDriver2(deps, rt, f, p, ports, ctx) {
|
|
|
11251
10803
|
}
|
|
11252
10804
|
async function farmRun(deps, f, rt, ports) {
|
|
11253
10805
|
const p = planFarmRun(f);
|
|
11254
|
-
const statePath =
|
|
10806
|
+
const statePath = join15(p.out, "keepsnappy-farm.json");
|
|
11255
10807
|
if (await deps.fs.exists(statePath))
|
|
11256
10808
|
throw new ConfigError([`${statePath} exists: a run set was opened here before; use another --out or remove ${p.out} (close it first with keepsnappy farm close)`]);
|
|
11257
10809
|
if (f["dry-run"]) {
|
|
@@ -11259,7 +10811,7 @@ async function farmRun(deps, f, rt, ports) {
|
|
|
11259
10811
|
return EXIT.ok;
|
|
11260
10812
|
}
|
|
11261
10813
|
const farm = ports ?? nodeFarmRunPorts(deps, rt);
|
|
11262
|
-
const tokensJs =
|
|
10814
|
+
const tokensJs = join15(p.out, "keepsnappy-tokens.js");
|
|
11263
10815
|
await deps.fs.mkdir(p.out);
|
|
11264
10816
|
const resolved = await farm.apps.resolve(p.app, { outDir: p.out });
|
|
11265
10817
|
const commit = f.commit ?? resolved.commit ?? undefined;
|
|
@@ -11290,9 +10842,9 @@ async function farmRun(deps, f, rt, ports) {
|
|
|
11290
10842
|
const runId = f["run-id"] ?? `keepsnappy-${p.platform}-${state.run_set_id.slice(0, 8)}`;
|
|
11291
10843
|
const ctx = { runId, out: p.out, tokensJs, runSetId: state.run_set_id, app: resolved.app };
|
|
11292
10844
|
try {
|
|
11293
|
-
await deps.spawner.run("rm", ["-f",
|
|
10845
|
+
await deps.spawner.run("rm", ["-f", join15(p.out, `browserstack.${p.platform}.json`)]);
|
|
11294
10846
|
if (p.ports.length) {
|
|
11295
|
-
tunnel = await farm.tunnel.open({ identifier: runId, ports: p.ports, logFile:
|
|
10847
|
+
tunnel = await farm.tunnel.open({ identifier: runId, ports: p.ports, logFile: join15(p.out, "browserstack-local.log") });
|
|
11296
10848
|
rt.print(`tunnel ${runId}: bs-local.com ports ${p.ports.join(", ")}`);
|
|
11297
10849
|
}
|
|
11298
10850
|
driver = await runDriver2(deps, rt, f, p, farm, { ...ctx, tunnelId: tunnel?.identifier ?? null });
|
|
@@ -11303,7 +10855,7 @@ async function farmRun(deps, f, rt, ports) {
|
|
|
11303
10855
|
rt.print(`device run failed: ${driver.message}; closing the run set with what arrived`);
|
|
11304
10856
|
closeCode = await farmClose(deps, {
|
|
11305
10857
|
state: statePath,
|
|
11306
|
-
"bs-build":
|
|
10858
|
+
"bs-build": join15(p.out, `browserstack.${p.platform}.json`),
|
|
11307
10859
|
out: p.out,
|
|
11308
10860
|
...f.wait ? { wait: true } : {},
|
|
11309
10861
|
...f["fail-on-worse"] ? { "fail-on-worse": true } : {},
|
|
@@ -11531,9 +11083,7 @@ function countsLine2(l) {
|
|
|
11531
11083
|
const unjudged = l.measured.lanes.filter((x) => !x.judged);
|
|
11532
11084
|
const head = parts.length ? `${parts.join(", ")}${state === "all" ? "" : `, ${STATE_WORDS2[state] ?? state}`}` : state !== "present" && state !== "all" ? `No finding ${STATE_WORDS2[state] ?? state}` : unjudged.length === l.measured.lanes.length ? `No finding, but nothing was compared yet (${[...new Set(unjudged.map((x) => x.word))].join(", ")}): measure again before calling it clean` : "Nothing worse, no warning, nothing better";
|
|
11533
11085
|
const heated = c.heat_runs ?? 0;
|
|
11534
|
-
const never = c.unsettled ?? 0;
|
|
11535
11086
|
const more = [
|
|
11536
|
-
never ? `${plural(never, "finding")} about taps or scrolls that never settle, so their time cannot be measured (information, layer 3)` : "",
|
|
11537
11087
|
heated ? `${plural(heated, "run")} heated up the phone (run health, layer 3)` : "",
|
|
11538
11088
|
state === "present" && l.states.not_judged ? `${l.states.not_judged} more not judged in the newest run set (state not-judged)` : ""
|
|
11539
11089
|
].filter(Boolean);
|
|
@@ -11638,7 +11188,7 @@ async function get(t, path, query = {}) {
|
|
|
11638
11188
|
}
|
|
11639
11189
|
var listFindings = (t, f = {}) => get(t, "v1/findings", { branch: f.branch, main: f.main ? "true" : undefined, journey: f.journey, phone: f.phone, state: f.state, limit: f.limit, offset: f.offset || undefined });
|
|
11640
11190
|
var getFinding = (t, id) => get(t, `v1/findings/${encodeURIComponent(id)}`);
|
|
11641
|
-
var
|
|
11191
|
+
var compareRuns = (t, a, b) => get(t, "v1/compare", { a, b });
|
|
11642
11192
|
var windowHistory = (t, q) => get(t, "v1/history", q);
|
|
11643
11193
|
var commitsBetween = (t, q) => get(t, "v1/commits", q);
|
|
11644
11194
|
var layers = (t) => get(t, "v1/layers");
|
|
@@ -11647,7 +11197,7 @@ var layers = (t) => get(t, "v1/layers");
|
|
|
11647
11197
|
var FINDINGS_USAGE = `keepsnappy findings [--branch NAME | --main] [--journey KEY] [--phone MODEL] [--state present|not-judged|resolved|all] [--limit N] [--offset N] [--json]
|
|
11648
11198
|
finding <id> [--prompt] [--json] (--prompt: only the hand-off for a coding agent; with --json the whole finding)
|
|
11649
11199
|
compare <runA> <runB> [--json] (run ids or external ids; one run against one, no statistics)
|
|
11650
|
-
history <journey> <window> [--phone MODEL] [--json] (window id or its name, e.g. "Tap
|
|
11200
|
+
history <journey> <window> [--phone MODEL] [--json] (window id or its name, e.g. "Tap · “live chat send”")
|
|
11651
11201
|
commits <from> [<to>] [--branch NAME] [--json] (measured commits between two run sets or commit hashes)
|
|
11652
11202
|
layers [--json]
|
|
11653
11203
|
(key: KEEPSNAPPY_KEY or --key-env NAME; API: --api URL or KEEPSNAPPY_API)`;
|
|
@@ -11734,7 +11284,7 @@ async function findingsCommand(cmd, argv, rt) {
|
|
|
11734
11284
|
}
|
|
11735
11285
|
case "compare":
|
|
11736
11286
|
need(2, "<runA> <runB>");
|
|
11737
|
-
return out(await
|
|
11287
|
+
return out(await compareRuns(t, p[0], p[1]), formatCompare);
|
|
11738
11288
|
case "history":
|
|
11739
11289
|
need(2, "<journey> <window>");
|
|
11740
11290
|
return out(await windowHistory(t, { journey: p[0], window: p.slice(1).join(" "), ...o.phone ? { phone: o.phone } : {}, ...limit ? { limit } : {} }), formatHistory);
|
|
@@ -11750,7 +11300,7 @@ async function findingsCommand(cmd, argv, rt) {
|
|
|
11750
11300
|
|
|
11751
11301
|
// apps/agent/src/cli.ts
|
|
11752
11302
|
var USAGE = `keepsnappy <command>
|
|
11753
|
-
run [--config keepsnappy.yml] [--
|
|
11303
|
+
run [--config keepsnappy.yml] [--repeat N] [--out DIR]
|
|
11754
11304
|
[--upload [--wait] [--fail-on-worse] [--ingest-url URL] [--commit SHA] [--branch NAME] [--base-commit SHA]]
|
|
11755
11305
|
[--device-model NAME --device-os VERSION] [--scenario NAME] [--replay <out dir>[:ARM] ...]
|
|
11756
11306
|
[--no-markers] [-- <e2e command...>]
|
|
@@ -11771,8 +11321,6 @@ var USAGE = `keepsnappy <command>
|
|
|
11771
11321
|
appium: --journey-module FILE | --driver-script FILE [--driver-arg ARG ...]
|
|
11772
11322
|
[--branch NAME --commit SHA] [--wait [--fail-on-worse]] [--out DIR] [--dry-run] (open + tunnel + device + close; docs/plans/54a)
|
|
11773
11323
|
extract <out-dir> --ri-capture observations.ndjson [--mem-window NAME]
|
|
11774
|
-
verdict <out-dir> [--baseline A --candidate B]
|
|
11775
|
-
verdict <out-dir> --baseline remote [--config keepsnappy.yml] [--branch main] [--limit N] [--scenario NAME]
|
|
11776
11324
|
aa <out-dir> [--reps N] [--boot N]
|
|
11777
11325
|
doctor
|
|
11778
11326
|
${SOURCEMAPS_USAGE.replace("keepsnappy ", "")}
|
|
@@ -11796,6 +11344,11 @@ async function main2(argv) {
|
|
|
11796
11344
|
console.log(USAGE);
|
|
11797
11345
|
return 0;
|
|
11798
11346
|
}
|
|
11347
|
+
if (cmd === "verdict")
|
|
11348
|
+
return verdictCommand();
|
|
11349
|
+
if (cmd === "run" && rest.slice(0, rest.indexOf("--") < 0 ? rest.length : rest.indexOf("--")).some((a) => a === "--baseline" || a.startsWith("--baseline="))) {
|
|
11350
|
+
throw new ConfigError([`--baseline: ${LOCAL_VERDICT_REMOVED}`]);
|
|
11351
|
+
}
|
|
11799
11352
|
if (cmd === "sourcemaps")
|
|
11800
11353
|
return sourcemapsCommand(rest, { env: process.env, print: console.log });
|
|
11801
11354
|
if (cmd && FINDINGS_COMMANDS.has(cmd))
|
|
@@ -11815,8 +11368,6 @@ async function main2(argv) {
|
|
|
11815
11368
|
out: { type: "string" },
|
|
11816
11369
|
"bundle-id": { type: "string" },
|
|
11817
11370
|
timeout: { type: "string" },
|
|
11818
|
-
baseline: { type: "string" },
|
|
11819
|
-
candidate: { type: "string" },
|
|
11820
11371
|
reps: { type: "string" },
|
|
11821
11372
|
boot: { type: "string" },
|
|
11822
11373
|
"ri-capture": { type: "string" },
|
|
@@ -11836,7 +11387,6 @@ async function main2(argv) {
|
|
|
11836
11387
|
"device-os": { type: "string" },
|
|
11837
11388
|
"device-id": { type: "string" },
|
|
11838
11389
|
replay: { type: "string", multiple: true },
|
|
11839
|
-
limit: { type: "string" },
|
|
11840
11390
|
api: { type: "string" },
|
|
11841
11391
|
"key-env": { type: "string" },
|
|
11842
11392
|
"device-provider": { type: "string" },
|
|
@@ -11937,37 +11487,12 @@ async function main2(argv) {
|
|
|
11937
11487
|
console.log(`${rows.length} rows -> ${resolve7(dir, "metrics.csv")}`);
|
|
11938
11488
|
return 0;
|
|
11939
11489
|
}
|
|
11940
|
-
if (cmd === "
|
|
11490
|
+
if (cmd === "aa") {
|
|
11941
11491
|
const dir = positionals[0];
|
|
11942
11492
|
if (!dir)
|
|
11943
|
-
throw new Error(
|
|
11944
|
-
|
|
11945
|
-
|
|
11946
|
-
const text = await readProjectConfig((p) => readFile5(p, "utf8"), o.config).catch(() => {
|
|
11947
|
-
throw new ConfigError([`cannot read ${configPath}: --baseline remote takes the API URL and key from keepsnappy.yml (use --config <file>)`]);
|
|
11948
|
-
});
|
|
11949
|
-
const config = parseConfig(text, { commandGiven: true });
|
|
11950
|
-
const url = o["ingest-url"] ?? config.project.ingestUrl ?? envValue(process.env, "API") ?? DEFAULT_API_URL;
|
|
11951
|
-
const key = projectKey(process.env, config.project.keyEnv);
|
|
11952
|
-
if (!url)
|
|
11953
|
-
throw new ConfigError(["--baseline remote needs project.ingest_url in keepsnappy.yml (or --ingest-url)"]);
|
|
11954
|
-
if (!key)
|
|
11955
|
-
throw new ConfigError([`--baseline remote needs the project key in the environment variable ${config.project.keyEnv}`]);
|
|
11956
|
-
const limit = o.limit ? Number(o.limit) : undefined;
|
|
11957
|
-
if (limit !== undefined && (!Number.isInteger(limit) || limit < 1))
|
|
11958
|
-
throw new ConfigError(["--limit must be a whole number >= 1"]);
|
|
11959
|
-
const out = await remoteVerdict(deps, resolve7(dir), { url, key }, {
|
|
11960
|
-
branch: o.branch ?? config.baselineBranch,
|
|
11961
|
-
...limit ? { limit } : {},
|
|
11962
|
-
...o.scenario ? { scenario: o.scenario } : {}
|
|
11963
|
-
});
|
|
11964
|
-
console.log(out.table);
|
|
11965
|
-
} else if (cmd === "verdict") {
|
|
11966
|
-
console.log((await verdict(deps, resolve7(dir), { ...o.baseline ? { baseline: o.baseline } : {}, ...o.candidate ? { candidate: o.candidate } : {} })).table);
|
|
11967
|
-
} else {
|
|
11968
|
-
const config = { ...o.reps ? { reps: Number(o.reps) } : {}, ...o.boot ? { nBoot: Number(o.boot) } : {} };
|
|
11969
|
-
console.log((await aa(deps, resolve7(dir), { config })).table);
|
|
11970
|
-
}
|
|
11493
|
+
throw new Error("aa needs <out-dir>");
|
|
11494
|
+
const config = { ...o.reps ? { reps: Number(o.reps) } : {}, ...o.boot ? { nBoot: Number(o.boot) } : {} };
|
|
11495
|
+
console.log((await aa(deps, resolve7(dir), { config })).table);
|
|
11971
11496
|
return 0;
|
|
11972
11497
|
}
|
|
11973
11498
|
console.error(USAGE);
|