@keepsnappy/agent 0.1.0-alpha.13 → 0.1.0-alpha.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -16
- package/dist/appium-driver.js +340 -81
- package/dist/keepsnappy.js +425 -876
- package/examples/farm/flows/keepsnappy/end-run.yaml +4 -3
- package/examples/keepsnappy.yml +1 -3
- package/package.json +1 -1
package/dist/keepsnappy.js
CHANGED
|
@@ -1,32 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
-
// apps/agent/src/compat.ts
|
|
4
|
-
import { resolve } from "node:path";
|
|
5
|
-
function envValue(env, name) {
|
|
6
|
-
return env[`KEEPSNAPPY_${name}`];
|
|
7
|
-
}
|
|
8
|
-
function projectKey(env, name = "KEEPSNAPPY_KEY") {
|
|
9
|
-
return name === "KEEPSNAPPY_KEY" ? envValue(env, "KEY") : env[name];
|
|
10
|
-
}
|
|
11
|
-
var CONFIG_NAMES = ["keepsnappy.yml", "keepsnappy.yaml"];
|
|
12
|
-
async function readProjectConfig(read, explicit) {
|
|
13
|
-
if (explicit !== undefined)
|
|
14
|
-
return read(resolve(explicit));
|
|
15
|
-
for (const name of CONFIG_NAMES) {
|
|
16
|
-
try {
|
|
17
|
-
return await read(resolve(name));
|
|
18
|
-
} catch (error) {
|
|
19
|
-
if (error.code !== "ENOENT")
|
|
20
|
-
throw error;
|
|
21
|
-
}
|
|
22
|
-
}
|
|
23
|
-
return read(resolve(CONFIG_NAMES[0]));
|
|
24
|
-
}
|
|
25
|
-
|
|
26
3
|
// apps/agent/src/cli.ts
|
|
27
4
|
import { resolve as resolve7 } from "node:path";
|
|
28
5
|
import { parseArgs as parseArgs4 } from "node:util";
|
|
29
|
-
import { readFile as readFile5 } from "node:fs/promises";
|
|
30
6
|
|
|
31
7
|
// apps/agent/src/config.ts
|
|
32
8
|
import { readdir } from "node:fs/promises";
|
|
@@ -238,7 +214,7 @@ class ConfigError extends Error {
|
|
|
238
214
|
}
|
|
239
215
|
var TITLE_MAX = 200;
|
|
240
216
|
var DESCRIPTION_MAX = 2000;
|
|
241
|
-
var TOP = ["project", "app", "platform", "device", "repetitions", "
|
|
217
|
+
var TOP = ["project", "app", "platform", "device", "repetitions", "flows", "steps", "output", "timeout_seconds"];
|
|
242
218
|
var isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
243
219
|
function splitCommand(s) {
|
|
244
220
|
const out = [];
|
|
@@ -341,7 +317,6 @@ function validateConfig(raw, opts = {}) {
|
|
|
341
317
|
else
|
|
342
318
|
repetitions = raw.repetitions;
|
|
343
319
|
}
|
|
344
|
-
const baselineBranch = str("baseline_branch", raw.baseline_branch, false, "main");
|
|
345
320
|
const flows = section("flows", ["runner", "maestro_version", "tags", "exclude_tags", "paths", "command"], !given);
|
|
346
321
|
const runner = str("flows.runner", flows.runner, false, "maestro");
|
|
347
322
|
if (runner !== "maestro" && runner !== "command")
|
|
@@ -398,7 +373,7 @@ function validateConfig(raw, opts = {}) {
|
|
|
398
373
|
if (runner === "command" && (tags.length || excludeTags.length || paths.length))
|
|
399
374
|
p.push("flows.tags/exclude_tags/paths only apply to runner: maestro");
|
|
400
375
|
const maestroVersion = flows.maestro_version === undefined ? undefined : String(flows.maestro_version);
|
|
401
|
-
const steps = section("steps", ["rename"
|
|
376
|
+
const steps = section("steps", ["rename"], false);
|
|
402
377
|
const rename = {};
|
|
403
378
|
if (steps.rename !== undefined && steps.rename !== null) {
|
|
404
379
|
if (!isObj(steps.rename))
|
|
@@ -411,7 +386,6 @@ function validateConfig(raw, opts = {}) {
|
|
|
411
386
|
rename[k] = v;
|
|
412
387
|
}
|
|
413
388
|
}
|
|
414
|
-
const pin = strList("steps.pin", steps.pin);
|
|
415
389
|
const output = section("output", ["dir"], false);
|
|
416
390
|
const outDir = str("output.dir", output.dir, false, "keepsnappy-out");
|
|
417
391
|
let timeoutSeconds;
|
|
@@ -429,9 +403,8 @@ function validateConfig(raw, opts = {}) {
|
|
|
429
403
|
platform,
|
|
430
404
|
device,
|
|
431
405
|
repetitions,
|
|
432
|
-
baselineBranch,
|
|
433
406
|
flows: { runner, ...maestroVersion ? { maestroVersion } : {}, tags, excludeTags, paths, info, command },
|
|
434
|
-
steps: { rename
|
|
407
|
+
steps: { rename },
|
|
435
408
|
outDir,
|
|
436
409
|
...timeoutSeconds ? { timeoutSeconds } : {}
|
|
437
410
|
};
|
|
@@ -513,9 +486,32 @@ async function expandGlob(pattern, listDir) {
|
|
|
513
486
|
return [...new Set(out)].sort();
|
|
514
487
|
}
|
|
515
488
|
|
|
489
|
+
// apps/agent/src/compat.ts
|
|
490
|
+
import { resolve } from "node:path";
|
|
491
|
+
function envValue(env, name) {
|
|
492
|
+
return env[`KEEPSNAPPY_${name}`];
|
|
493
|
+
}
|
|
494
|
+
function projectKey(env, name = "KEEPSNAPPY_KEY") {
|
|
495
|
+
return name === "KEEPSNAPPY_KEY" ? envValue(env, "KEY") : env[name];
|
|
496
|
+
}
|
|
497
|
+
var CONFIG_NAMES = ["keepsnappy.yml", "keepsnappy.yaml"];
|
|
498
|
+
async function readProjectConfig(read, explicit) {
|
|
499
|
+
if (explicit !== undefined)
|
|
500
|
+
return read(resolve(explicit));
|
|
501
|
+
for (const name of CONFIG_NAMES) {
|
|
502
|
+
try {
|
|
503
|
+
return await read(resolve(name));
|
|
504
|
+
} catch (error) {
|
|
505
|
+
if (error.code !== "ENOENT")
|
|
506
|
+
throw error;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return read(resolve(CONFIG_NAMES[0]));
|
|
510
|
+
}
|
|
511
|
+
|
|
516
512
|
// apps/agent/src/run-command.ts
|
|
517
513
|
import { readFile } from "node:fs/promises";
|
|
518
|
-
import { join as
|
|
514
|
+
import { join as join9, resolve as resolve3 } from "node:path";
|
|
519
515
|
|
|
520
516
|
// apps/agent/src/data-source.ts
|
|
521
517
|
import { join as join2 } from "node:path";
|
|
@@ -2053,6 +2049,13 @@ var NOT_FACTS = {
|
|
|
2053
2049
|
decimals: 0,
|
|
2054
2050
|
what: "Whether the test cleaned up the app’s memory once at the end of the journey, outside any measured window, before the end reading"
|
|
2055
2051
|
},
|
|
2052
|
+
"sdk.end_read": {
|
|
2053
|
+
unit: "flag",
|
|
2054
|
+
label: "End memory read taken on request after the clean-up (1 = yes)",
|
|
2055
|
+
short: "end memory read on request",
|
|
2056
|
+
decimals: 0,
|
|
2057
|
+
what: "Whether the end memory reading is the one the app took when the test asked for it right after the clean-up (1), or the clean-up’s own reading (0, an app or run without the request)"
|
|
2058
|
+
},
|
|
2056
2059
|
"sdk.appstate_changes": {
|
|
2057
2060
|
unit: "count",
|
|
2058
2061
|
label: "App went to the background or came back",
|
|
@@ -2092,7 +2095,23 @@ var NOT_FACTS = {
|
|
|
2092
2095
|
};
|
|
2093
2096
|
var isNotFactId = (id) => Object.hasOwn(NOT_FACTS, id);
|
|
2094
2097
|
var fieldDef = (id) => isMeasureId(id) ? MEASURES[id] : isNotFactId(id) ? NOT_FACTS[id] : undefined;
|
|
2098
|
+
var DIMENSIONS = {
|
|
2099
|
+
component: { label: "Component", rankBy: "self_ms" },
|
|
2100
|
+
updater: { label: "Update scheduled by", rankBy: "share_of_commits" },
|
|
2101
|
+
slow_commit_component: { label: "Component in the slowest commit", rankBy: "self_ms" },
|
|
2102
|
+
slow_commit_updater: { label: "Update of the slowest commit scheduled by", rankBy: "scheduled" }
|
|
2103
|
+
};
|
|
2104
|
+
var BREAKDOWN_MEASURES = {
|
|
2105
|
+
self_ms: { unit: "ms", worse: "higher", label: "render time", decimals: 1 },
|
|
2106
|
+
total_ms: { unit: "ms", worse: "higher", label: "total render time", decimals: 1 },
|
|
2107
|
+
renders: { unit: "count", worse: "higher", label: "renders", decimals: 0 },
|
|
2108
|
+
mounts: { unit: "count", worse: "higher", label: "mounts", decimals: 0 },
|
|
2109
|
+
share_of_commits: { unit: "share", worse: "none", label: "share of commits", decimals: 0 },
|
|
2110
|
+
scheduled: { unit: "count", worse: "none", label: "updates scheduled", decimals: 0 }
|
|
2111
|
+
};
|
|
2095
2112
|
var measureDef = (id) => isMeasureId(id) ? MEASURES[id] : undefined;
|
|
2113
|
+
var HEAT_METRIC = "sampler.thermal.heats_up";
|
|
2114
|
+
var HEAT_LABEL = MEASURES[HEAT_METRIC].label;
|
|
2096
2115
|
// packages/facts/dist/src/places.js
|
|
2097
2116
|
var AFTER_3S = "+3s";
|
|
2098
2117
|
var kindOf = (stem) => {
|
|
@@ -2432,32 +2451,6 @@ function mannWhitney(a, b, exactLimit = MWU_EXACT_LIMIT) {
|
|
|
2432
2451
|
const z = Math.max((Math.abs(u - mu) - 0.5) / Math.sqrt(v), 0);
|
|
2433
2452
|
return { u, p: Math.min(1, 2 * (1 - normCdf(z))), cles, exact: false };
|
|
2434
2453
|
}
|
|
2435
|
-
function hodgesLehmann(a, b) {
|
|
2436
|
-
const d = [];
|
|
2437
|
-
for (const bj of b)
|
|
2438
|
-
for (const ai of a)
|
|
2439
|
-
d.push(bj - ai);
|
|
2440
|
-
return median(d);
|
|
2441
|
-
}
|
|
2442
|
-
// packages/stats/dist/src/inference/wilcoxon.js
|
|
2443
|
-
function wilcoxonSignedRank(d) {
|
|
2444
|
-
const nz = d.filter((v) => v !== 0);
|
|
2445
|
-
const n = nz.length;
|
|
2446
|
-
if (n === 0)
|
|
2447
|
-
return 1;
|
|
2448
|
-
const { ranks, tieTerm } = averageRanks(nz.map(Math.abs));
|
|
2449
|
-
let w = 0;
|
|
2450
|
-
nz.forEach((v, i) => {
|
|
2451
|
-
if (v > 0)
|
|
2452
|
-
w += ranks[i];
|
|
2453
|
-
});
|
|
2454
|
-
const mu = n * (n + 1) / 4;
|
|
2455
|
-
const v = n * (n + 1) * (2 * n + 1) / 24 - tieTerm / 48;
|
|
2456
|
-
if (v <= 0)
|
|
2457
|
-
return 1;
|
|
2458
|
-
const z = Math.max((Math.abs(w - mu) - 0.5) / Math.sqrt(v), 0);
|
|
2459
|
-
return Math.min(1, 2 * (1 - normCdf(z)));
|
|
2460
|
-
}
|
|
2461
2454
|
// packages/stats/dist/src/inference/bootstrap.js
|
|
2462
2455
|
var NAN_INTERVAL = { lo: Number.NaN, hi: Number.NaN };
|
|
2463
2456
|
function locationInPlace(scratch, kind) {
|
|
@@ -2472,88 +2465,6 @@ function locationInPlace(scratch, kind) {
|
|
|
2472
2465
|
function location(x, kind) {
|
|
2473
2466
|
return kind === "count" ? mean(x) : locationInPlace(Float64Array.from(x), kind);
|
|
2474
2467
|
}
|
|
2475
|
-
function resampleLocation(x, kind, rng, buf) {
|
|
2476
|
-
for (let i = 0;i < buf.length; i++)
|
|
2477
|
-
buf[i] = x[rng.int(x.length)];
|
|
2478
|
-
return locationInPlace(buf, kind);
|
|
2479
|
-
}
|
|
2480
|
-
function bootLogRatio(a, b, kind, nBoot, rng) {
|
|
2481
|
-
const logRatio = new Float64Array(nBoot);
|
|
2482
|
-
const diff = new Float64Array(nBoot);
|
|
2483
|
-
const ba = new Float64Array(a.length);
|
|
2484
|
-
const bb = new Float64Array(b.length);
|
|
2485
|
-
for (let r = 0;r < nBoot; r++) {
|
|
2486
|
-
const la = resampleLocation(a, kind, rng, ba);
|
|
2487
|
-
const lb = resampleLocation(b, kind, rng, bb);
|
|
2488
|
-
logRatio[r] = Math.log(lb) - Math.log(la);
|
|
2489
|
-
diff[r] = lb - la;
|
|
2490
|
-
}
|
|
2491
|
-
return { logRatio, diff };
|
|
2492
|
-
}
|
|
2493
|
-
function bootPairedLogRatio(blockLogRatios, nBoot, rng) {
|
|
2494
|
-
const out = new Float64Array(nBoot);
|
|
2495
|
-
const buf = new Float64Array(blockLogRatios.length);
|
|
2496
|
-
for (let r = 0;r < nBoot; r++)
|
|
2497
|
-
out[r] = resampleLocation(blockLogRatios, "continuous", rng, buf);
|
|
2498
|
-
return out;
|
|
2499
|
-
}
|
|
2500
|
-
function percentileCi(draws, level) {
|
|
2501
|
-
const lo = (1 - level) / 2;
|
|
2502
|
-
const [a, b] = finiteQuantiles(draws, [lo, 1 - lo]);
|
|
2503
|
-
return { lo: a ?? Number.NaN, hi: b ?? Number.NaN };
|
|
2504
|
-
}
|
|
2505
|
-
function bcaCi(draws, thetaHat, jackknife, level) {
|
|
2506
|
-
const d = [];
|
|
2507
|
-
for (let i = 0;i < draws.length; i++) {
|
|
2508
|
-
const v = draws[i];
|
|
2509
|
-
if (Number.isFinite(v))
|
|
2510
|
-
d.push(v);
|
|
2511
|
-
}
|
|
2512
|
-
if (d.length === 0 || !Number.isFinite(thetaHat))
|
|
2513
|
-
return NAN_INTERVAL;
|
|
2514
|
-
let below = 0;
|
|
2515
|
-
let equal = 0;
|
|
2516
|
-
for (const v of d) {
|
|
2517
|
-
if (v < thetaHat)
|
|
2518
|
-
below++;
|
|
2519
|
-
else if (v === thetaHat)
|
|
2520
|
-
equal++;
|
|
2521
|
-
}
|
|
2522
|
-
let prop = (below + 0.5 * equal) / d.length;
|
|
2523
|
-
prop = Math.min(Math.max(prop, 1 / (d.length + 1)), 1 - 1 / (d.length + 1));
|
|
2524
|
-
const z0 = normPpf(prop);
|
|
2525
|
-
const jack = jackknife.filter(Number.isFinite);
|
|
2526
|
-
const jm = mean(jack);
|
|
2527
|
-
let num = 0;
|
|
2528
|
-
let sq = 0;
|
|
2529
|
-
for (const v of jack) {
|
|
2530
|
-
num += (jm - v) ** 3;
|
|
2531
|
-
sq += (jm - v) ** 2;
|
|
2532
|
-
}
|
|
2533
|
-
const den = 6 * sq ** 1.5;
|
|
2534
|
-
const acc = den > 0 ? num / den : 0;
|
|
2535
|
-
const qs = [(1 - level) / 2, 1 - (1 - level) / 2].map((q) => {
|
|
2536
|
-
const zq = normPpf(q);
|
|
2537
|
-
const adj = normCdf(z0 + (z0 + zq) / (1 - acc * (z0 + zq)));
|
|
2538
|
-
return Math.min(Math.max(adj, 0), 1);
|
|
2539
|
-
});
|
|
2540
|
-
const [lo, hi] = finiteQuantiles(d, qs);
|
|
2541
|
-
return { lo: lo ?? Number.NaN, hi: hi ?? Number.NaN };
|
|
2542
|
-
}
|
|
2543
|
-
function jackknifeLogRatio(a, b, kind) {
|
|
2544
|
-
const out = [];
|
|
2545
|
-
const la = location(a, kind);
|
|
2546
|
-
const lb = location(b, kind);
|
|
2547
|
-
for (let i = 0;i < a.length; i++) {
|
|
2548
|
-
const l = location(a.filter((_, j) => j !== i), kind);
|
|
2549
|
-
out.push(l > 0 && lb > 0 ? Math.log(lb / l) : Number.NaN);
|
|
2550
|
-
}
|
|
2551
|
-
for (let i = 0;i < b.length; i++) {
|
|
2552
|
-
const l = location(b.filter((_, j) => j !== i), kind);
|
|
2553
|
-
out.push(la > 0 && l > 0 ? Math.log(l / la) : Number.NaN);
|
|
2554
|
-
}
|
|
2555
|
-
return out;
|
|
2556
|
-
}
|
|
2557
2468
|
// packages/stats/dist/src/inference/equivalence.js
|
|
2558
2469
|
var relativeMargin = (value) => ({ kind: "relative", value });
|
|
2559
2470
|
var absoluteMargin = (value) => ({ kind: "absolute", value });
|
|
@@ -2563,78 +2474,6 @@ function equivalenceBounds(margin) {
|
|
|
2563
2474
|
function isEquivalent(ci, bounds) {
|
|
2564
2475
|
return Number.isFinite(ci.lo) && Number.isFinite(ci.hi) && ci.lo > bounds.lo && ci.hi < bounds.hi;
|
|
2565
2476
|
}
|
|
2566
|
-
function excludesZero(ci) {
|
|
2567
|
-
return ci.lo > 0 || ci.hi < 0;
|
|
2568
|
-
}
|
|
2569
|
-
// packages/stats/dist/src/inference/fisher.js
|
|
2570
|
-
function logFactorials(n) {
|
|
2571
|
-
const out = new Float64Array(n + 1);
|
|
2572
|
-
for (let i = 2;i <= n; i++)
|
|
2573
|
-
out[i] = out[i - 1] + Math.log(i);
|
|
2574
|
-
return out;
|
|
2575
|
-
}
|
|
2576
|
-
function fisherExact(t) {
|
|
2577
|
-
const r1 = t.a + t.b;
|
|
2578
|
-
const r2 = t.c + t.d;
|
|
2579
|
-
const c1 = t.a + t.c;
|
|
2580
|
-
const n = r1 + r2;
|
|
2581
|
-
if (r1 === 0 || r2 === 0 || c1 === 0 || c1 === n)
|
|
2582
|
-
return 1;
|
|
2583
|
-
const lf = logFactorials(n);
|
|
2584
|
-
const f = (k) => lf[k];
|
|
2585
|
-
const fixed = f(r1) + f(r2) + f(c1) + f(n - c1) - f(n);
|
|
2586
|
-
const logP = (x) => fixed - f(x) - f(r1 - x) - f(c1 - x) - f(r2 - c1 + x);
|
|
2587
|
-
const observed = logP(t.a);
|
|
2588
|
-
let p = 0;
|
|
2589
|
-
for (let x = Math.max(0, c1 - r2);x <= Math.min(r1, c1); x++) {
|
|
2590
|
-
const lp = logP(x);
|
|
2591
|
-
if (lp <= observed + 0.0000001)
|
|
2592
|
-
p += Math.exp(lp);
|
|
2593
|
-
}
|
|
2594
|
-
return Math.min(1, p);
|
|
2595
|
-
}
|
|
2596
|
-
// packages/stats/dist/src/inference/multiplicity.js
|
|
2597
|
-
function order(p) {
|
|
2598
|
-
return p.map((_, i) => i).sort((i, j) => p[i] - p[j] || i - j);
|
|
2599
|
-
}
|
|
2600
|
-
function bonferroni(p) {
|
|
2601
|
-
return p.map((v) => Math.min(1, v * p.length));
|
|
2602
|
-
}
|
|
2603
|
-
function holm(p) {
|
|
2604
|
-
const m = p.length;
|
|
2605
|
-
const out = new Array(m).fill(1);
|
|
2606
|
-
let running = 0;
|
|
2607
|
-
order(p).forEach((idx, rank) => {
|
|
2608
|
-
running = Math.max(running, Math.min(1, (m - rank) * p[idx]));
|
|
2609
|
-
out[idx] = running;
|
|
2610
|
-
});
|
|
2611
|
-
return out;
|
|
2612
|
-
}
|
|
2613
|
-
function benjaminiHochberg(p) {
|
|
2614
|
-
const m = p.length;
|
|
2615
|
-
const out = new Array(m).fill(1);
|
|
2616
|
-
const ord = order(p);
|
|
2617
|
-
let running = 1;
|
|
2618
|
-
for (let rank = m - 1;rank >= 0; rank--) {
|
|
2619
|
-
const idx = ord[rank];
|
|
2620
|
-
running = Math.min(running, Math.min(1, p[idx] * m / (rank + 1)));
|
|
2621
|
-
out[idx] = running;
|
|
2622
|
-
}
|
|
2623
|
-
return out;
|
|
2624
|
-
}
|
|
2625
|
-
function adjustPValues(p, method) {
|
|
2626
|
-
const finite = p.map((v) => Number.isFinite(v) ? v : 1);
|
|
2627
|
-
switch (method) {
|
|
2628
|
-
case "none":
|
|
2629
|
-
return [...p];
|
|
2630
|
-
case "bonferroni":
|
|
2631
|
-
return bonferroni(finite);
|
|
2632
|
-
case "holm":
|
|
2633
|
-
return holm(finite);
|
|
2634
|
-
case "bh":
|
|
2635
|
-
return benjaminiHochberg(finite);
|
|
2636
|
-
}
|
|
2637
|
-
}
|
|
2638
2477
|
// packages/stats/dist/src/modes/config.js
|
|
2639
2478
|
var DEFAULT_MODE_CONFIG = {
|
|
2640
2479
|
keys: ["mounts", "renders", "commits"],
|
|
@@ -2666,9 +2505,6 @@ var DEFAULT_CHANGE_CONFIG = {
|
|
|
2666
2505
|
seed: 12345,
|
|
2667
2506
|
modes: DEFAULT_MODE_CONFIG
|
|
2668
2507
|
};
|
|
2669
|
-
function withChangeDefaults(partial = {}) {
|
|
2670
|
-
return { ...DEFAULT_CHANGE_CONFIG, ...partial };
|
|
2671
|
-
}
|
|
2672
2508
|
// packages/stats/dist/src/change/decide.js
|
|
2673
2509
|
function decide(ci95, ci90, p, alpha, bounds, higherIsBetter) {
|
|
2674
2510
|
if (isEquivalent(ci90, bounds))
|
|
@@ -2680,51 +2516,6 @@ function decide(ci95, ci90, p, alpha, bounds, higherIsBetter) {
|
|
|
2680
2516
|
return higherIsBetter ? "worse" : "better";
|
|
2681
2517
|
return "inconclusive";
|
|
2682
2518
|
}
|
|
2683
|
-
// packages/stats/dist/src/change/paired.js
|
|
2684
|
-
function groupByBlock(values, blocks) {
|
|
2685
|
-
const out = new Map;
|
|
2686
|
-
values.forEach((v, i) => {
|
|
2687
|
-
const k = blocks[i];
|
|
2688
|
-
const g = out.get(k);
|
|
2689
|
-
if (g)
|
|
2690
|
-
g.push(v);
|
|
2691
|
-
else
|
|
2692
|
-
out.set(k, [v]);
|
|
2693
|
-
});
|
|
2694
|
-
return out;
|
|
2695
|
-
}
|
|
2696
|
-
function pairedAnalysis(a, aBlocks, b, bBlocks, kind, alpha, nBoot, minBlocks, rng) {
|
|
2697
|
-
const ga = groupByBlock(a, aBlocks);
|
|
2698
|
-
const gb = groupByBlock(b, bBlocks);
|
|
2699
|
-
const common = [...ga.keys()].filter((k) => gb.has(k)).sort();
|
|
2700
|
-
if (common.length < minBlocks)
|
|
2701
|
-
return null;
|
|
2702
|
-
const lr = [];
|
|
2703
|
-
const df = [];
|
|
2704
|
-
for (const k of common) {
|
|
2705
|
-
const la = location(ga.get(k), kind);
|
|
2706
|
-
const lb = location(gb.get(k), kind);
|
|
2707
|
-
if (la > 0 && lb > 0) {
|
|
2708
|
-
lr.push(Math.log(lb / la));
|
|
2709
|
-
df.push(lb - la);
|
|
2710
|
-
}
|
|
2711
|
-
}
|
|
2712
|
-
if (lr.length < minBlocks)
|
|
2713
|
-
return null;
|
|
2714
|
-
const lrDraws = bootPairedLogRatio(lr, nBoot, rng);
|
|
2715
|
-
const dfDraws = bootPairedLogRatio(df, nBoot, rng);
|
|
2716
|
-
return {
|
|
2717
|
-
blocks: lr.length,
|
|
2718
|
-
logRatio: median(lr),
|
|
2719
|
-
logRatioCi95: percentileCi(lrDraws, 1 - alpha),
|
|
2720
|
-
logRatioCi90: percentileCi(lrDraws, 1 - 2 * alpha),
|
|
2721
|
-
diff: median(df),
|
|
2722
|
-
diffCi95: percentileCi(dfDraws, 1 - alpha),
|
|
2723
|
-
diffCi90: percentileCi(dfDraws, 1 - 2 * alpha),
|
|
2724
|
-
p: wilcoxonSignedRank(lr)
|
|
2725
|
-
};
|
|
2726
|
-
}
|
|
2727
|
-
|
|
2728
2519
|
// packages/stats/dist/src/change/compare-metric.js
|
|
2729
2520
|
function effectiveMargin(a, o) {
|
|
2730
2521
|
const floor = o.countFloor;
|
|
@@ -2733,89 +2524,6 @@ function effectiveMargin(a, o) {
|
|
|
2733
2524
|
const units = o.margin.value * Math.abs(location(a, "count"));
|
|
2734
2525
|
return units < floor ? absoluteMargin(floor) : o.margin;
|
|
2735
2526
|
}
|
|
2736
|
-
var expInterval = (ci) => ({
|
|
2737
|
-
lo: Number.isFinite(ci.lo) ? Math.exp(ci.lo) : Number.NaN,
|
|
2738
|
-
hi: Number.isFinite(ci.hi) ? Math.exp(ci.hi) : Number.NaN
|
|
2739
|
-
});
|
|
2740
|
-
function summarize(x, kind) {
|
|
2741
|
-
const loc = location(x, kind);
|
|
2742
|
-
const base = { n: x.length, location: loc, mad: mad(x), robustCv: robustCv(x) };
|
|
2743
|
-
if (kind !== "count")
|
|
2744
|
-
return base;
|
|
2745
|
-
return { ...base, dispersion: x.length > 1 && loc > 0 ? variance(x, 1) / loc : Number.NaN };
|
|
2746
|
-
}
|
|
2747
|
-
function unpairedEstimates(a, b, o, pMwu, rng) {
|
|
2748
|
-
const la = location(a, o.kind);
|
|
2749
|
-
const lb = location(b, o.kind);
|
|
2750
|
-
const draws = bootLogRatio(a, b, o.kind, o.nBoot, rng);
|
|
2751
|
-
const logRatio = la > 0 && lb > 0 ? Math.log(lb / la) : Number.NaN;
|
|
2752
|
-
let lr95;
|
|
2753
|
-
let lr90;
|
|
2754
|
-
if (o.ci === "bca") {
|
|
2755
|
-
const jack = jackknifeLogRatio(a, b, o.kind);
|
|
2756
|
-
lr95 = bcaCi(draws.logRatio, logRatio, jack, 1 - o.alpha);
|
|
2757
|
-
lr90 = bcaCi(draws.logRatio, logRatio, jack, 1 - 2 * o.alpha);
|
|
2758
|
-
} else {
|
|
2759
|
-
lr95 = percentileCi(draws.logRatio, 1 - o.alpha);
|
|
2760
|
-
lr90 = percentileCi(draws.logRatio, 1 - 2 * o.alpha);
|
|
2761
|
-
}
|
|
2762
|
-
return {
|
|
2763
|
-
logRatio,
|
|
2764
|
-
lr95,
|
|
2765
|
-
lr90,
|
|
2766
|
-
diff: lb - la,
|
|
2767
|
-
diff95: percentileCi(draws.diff, 1 - o.alpha),
|
|
2768
|
-
diff90: percentileCi(draws.diff, 1 - 2 * o.alpha),
|
|
2769
|
-
p: pMwu,
|
|
2770
|
-
paired: null
|
|
2771
|
-
};
|
|
2772
|
-
}
|
|
2773
|
-
function compareMetric(a, b, o, rng, blocks) {
|
|
2774
|
-
const margin = effectiveMargin(a, o);
|
|
2775
|
-
const mw = mannWhitney(a, b);
|
|
2776
|
-
const pairedRes = o.paired && blocks ? pairedAnalysis(a, blocks.a, b, blocks.b, o.kind, o.alpha, o.nBoot, o.minPairedBlocks, rng) : null;
|
|
2777
|
-
const est = pairedRes ? {
|
|
2778
|
-
logRatio: pairedRes.logRatio,
|
|
2779
|
-
lr95: pairedRes.logRatioCi95,
|
|
2780
|
-
lr90: pairedRes.logRatioCi90,
|
|
2781
|
-
diff: pairedRes.diff,
|
|
2782
|
-
diff95: pairedRes.diffCi95,
|
|
2783
|
-
diff90: pairedRes.diffCi90,
|
|
2784
|
-
p: pairedRes.p,
|
|
2785
|
-
paired: { blocks: pairedRes.blocks, p: pairedRes.p }
|
|
2786
|
-
} : unpairedEstimates(a, b, o, mw.p, rng);
|
|
2787
|
-
const absolute = margin.kind === "absolute";
|
|
2788
|
-
const ci95 = absolute ? est.diff95 : est.lr95;
|
|
2789
|
-
const ci90 = absolute ? est.diff90 : est.lr90;
|
|
2790
|
-
let verdict = decide(ci95, ci90, est.p, o.alpha, equivalenceBounds(margin), o.higherIsBetter);
|
|
2791
|
-
let note;
|
|
2792
|
-
if (verdict === "no-change" && est.p < o.alpha && excludesZero(ci95)) {
|
|
2793
|
-
note = `detectable shift, but inside the ${absolute ? `+/-${margin.value}` : `+/-${Math.round(margin.value * 100)}%`} margin`;
|
|
2794
|
-
}
|
|
2795
|
-
if (Math.min(a.length, b.length) < o.minN) {
|
|
2796
|
-
verdict = "inconclusive";
|
|
2797
|
-
note = `n < ${o.minN}`;
|
|
2798
|
-
}
|
|
2799
|
-
return {
|
|
2800
|
-
kind: o.kind,
|
|
2801
|
-
baseline: summarize(a, o.kind),
|
|
2802
|
-
candidate: summarize(b, o.kind),
|
|
2803
|
-
mannWhitney: { u: mw.u, p: mw.p, cles: mw.cles },
|
|
2804
|
-
hodgesLehmann: hodgesLehmann(a, b),
|
|
2805
|
-
paired: est.paired !== null,
|
|
2806
|
-
...est.paired ? { blocks: est.paired.blocks, pPaired: est.paired.p } : {},
|
|
2807
|
-
p: est.p,
|
|
2808
|
-
ratio: Number.isFinite(est.logRatio) ? Math.exp(est.logRatio) : Number.NaN,
|
|
2809
|
-
ratioCi95: expInterval(est.lr95),
|
|
2810
|
-
ratioCi90: expInterval(est.lr90),
|
|
2811
|
-
diff: est.diff,
|
|
2812
|
-
diffCi95: est.diff95,
|
|
2813
|
-
diffCi90: est.diff90,
|
|
2814
|
-
margin,
|
|
2815
|
-
verdict,
|
|
2816
|
-
...note ? { note } : {}
|
|
2817
|
-
};
|
|
2818
|
-
}
|
|
2819
2527
|
// packages/stats/dist/src/modes/detect.js
|
|
2820
2528
|
function group(x) {
|
|
2821
2529
|
return { n: x.length, median: median(x), min: x[0], max: x[x.length - 1], spread: 1.4826 * mad(x) };
|
|
@@ -2860,22 +2568,19 @@ function splitTwoModes(values, cfg) {
|
|
|
2860
2568
|
return null;
|
|
2861
2569
|
return { threshold: (low.max + high.min) / 2, low, high, gap, spread, separation: gap / spread };
|
|
2862
2570
|
}
|
|
2863
|
-
|
|
2571
|
+
// packages/stats/dist/src/modes/bimodal.js
|
|
2572
|
+
var BIMODAL_CONFIG = withModeDefaults({
|
|
2573
|
+
minModeRuns: 3,
|
|
2574
|
+
minModeShare: 0.15,
|
|
2575
|
+
gapFactor: 3,
|
|
2576
|
+
minRelativeGap: 0.25
|
|
2577
|
+
});
|
|
2864
2578
|
// packages/stats/dist/src/modes/windows.js
|
|
2865
2579
|
var MODE_LABELS = ["low", "high"];
|
|
2866
2580
|
function splitMetricName(metric) {
|
|
2867
2581
|
const i = metric.lastIndexOf(".");
|
|
2868
2582
|
return i < 0 ? { window: metric, measure: "" } : { window: metric.slice(0, i), measure: metric.slice(i + 1) };
|
|
2869
2583
|
}
|
|
2870
|
-
function countByArm(obs, labels) {
|
|
2871
|
-
const out = { low: 0, high: 0 };
|
|
2872
|
-
for (const o of obs ?? []) {
|
|
2873
|
-
const l = labels[o.runId];
|
|
2874
|
-
if (l)
|
|
2875
|
-
out[l] += 1;
|
|
2876
|
-
}
|
|
2877
|
-
return out;
|
|
2878
|
-
}
|
|
2879
2584
|
function splitWindows(dataset, arms, cfg) {
|
|
2880
2585
|
const keyed = new Map;
|
|
2881
2586
|
for (const metric of dataset.metrics.keys()) {
|
|
@@ -2905,126 +2610,6 @@ function splitWindows(dataset, arms, cfg) {
|
|
|
2905
2610
|
}
|
|
2906
2611
|
return out;
|
|
2907
2612
|
}
|
|
2908
|
-
function detectWindowModes(dataset, baseline, candidate, cfg, alpha) {
|
|
2909
|
-
const out = new Map;
|
|
2910
|
-
for (const [window, ws] of splitWindows(dataset, [baseline, candidate], cfg)) {
|
|
2911
|
-
const arms = dataset.metrics.get(`${window}.${ws.key}`);
|
|
2912
|
-
const a = countByArm(arms?.get(baseline), ws.labels);
|
|
2913
|
-
const b = countByArm(arms?.get(candidate), ws.labels);
|
|
2914
|
-
const p = fisherExact({ a: a.low, b: a.high, c: b.low, d: b.high });
|
|
2915
|
-
const mixed = MODE_LABELS.some((l) => a[l] > 0 && b[l] > 0);
|
|
2916
|
-
out.set(window, { ...ws, mix: { baseline: a, candidate: b, p, shifted: p < alpha }, confounded: !mixed });
|
|
2917
|
-
}
|
|
2918
|
-
return out;
|
|
2919
|
-
}
|
|
2920
|
-
|
|
2921
|
-
// packages/stats/dist/src/change/compare-modes.js
|
|
2922
|
-
function combine(modes, confounded, pooled) {
|
|
2923
|
-
if (confounded)
|
|
2924
|
-
return pooled;
|
|
2925
|
-
const tested = modes.filter((m) => m.result);
|
|
2926
|
-
if (tested.some((m) => m.verdict === "worse"))
|
|
2927
|
-
return "worse";
|
|
2928
|
-
if (tested.some((m) => m.verdict === "better"))
|
|
2929
|
-
return "better";
|
|
2930
|
-
if (tested.length === modes.length && tested.every((m) => m.verdict === "no-change"))
|
|
2931
|
-
return "no-change";
|
|
2932
|
-
return "inconclusive";
|
|
2933
|
-
}
|
|
2934
|
-
function explain(m, verdict) {
|
|
2935
|
-
const counts = (r) => `${r.low}/${r.high}`;
|
|
2936
|
-
const mix = `mix low/high ${counts(m.mix.baseline)} vs ${counts(m.mix.candidate)}, Fisher p=${m.mix.p.toPrecision(3)}${m.mix.shifted ? " (mix shifted)" : ""}`;
|
|
2937
|
-
if (m.confounded) {
|
|
2938
|
-
return `modes on ${m.key} coincide with the arms (${mix}); pooled verdict ${verdict.toUpperCase()} kept, but a state difference between arms looks the same`;
|
|
2939
|
-
}
|
|
2940
|
-
const parts = m.modes.map((x) => `${x.label}: ${x.verdict}${x.reason ? ` (${x.reason})` : ""}`);
|
|
2941
|
-
return `state modes on ${m.key}: ${parts.join(", ")}; ${mix}; pooled ${m.pooledVerdict}`;
|
|
2942
|
-
}
|
|
2943
|
-
function compareByMode(metric, A, B, pooled, wm, options, cfg, seed, modeMargins) {
|
|
2944
|
-
const rng = createRng(labelSeed(seed, metric));
|
|
2945
|
-
const pick = (obs, l) => obs.filter((o) => wm.labels[o.runId] === l);
|
|
2946
|
-
const modes = MODE_LABELS.map((label) => {
|
|
2947
|
-
const a = pick(A, label);
|
|
2948
|
-
const b = pick(B, label);
|
|
2949
|
-
const base = { label, keyMedian: wm.split[label].median, nBaseline: a.length, nCandidate: b.length };
|
|
2950
|
-
if (Math.min(a.length, b.length) < cfg.minRunsPerArm) {
|
|
2951
|
-
return { ...base, verdict: "inconclusive", reason: `n ${a.length} vs ${b.length}, needs ${cfg.minRunsPerArm} per arm` };
|
|
2952
|
-
}
|
|
2953
|
-
const own = modeMargins?.[label];
|
|
2954
|
-
const result = compareMetric(a.map((o) => o.value), b.map((o) => o.value), {
|
|
2955
|
-
...options,
|
|
2956
|
-
...own?.margin ? { margin: own.margin } : {},
|
|
2957
|
-
countFloor: own?.countFloor ?? (options.countFloor || DEFAULT_COUNT_FLOOR),
|
|
2958
|
-
paired: false,
|
|
2959
|
-
minN: cfg.minRunsPerArm
|
|
2960
|
-
}, rng);
|
|
2961
|
-
return { ...base, result, verdict: result.verdict, ...result.note ? { reason: result.note } : {} };
|
|
2962
|
-
});
|
|
2963
|
-
const unlabeled = [...A, ...B].filter((o) => !(o.runId in wm.labels)).length;
|
|
2964
|
-
const partial = { window: wm.window, key: wm.key, modes, mix: wm.mix, confounded: wm.confounded, unlabeled, pooledVerdict: pooled.verdict };
|
|
2965
|
-
const verdict = combine(modes, wm.confounded, pooled.verdict);
|
|
2966
|
-
return { ...partial, verdict, why: explain(partial, verdict) };
|
|
2967
|
-
}
|
|
2968
|
-
|
|
2969
|
-
// packages/stats/dist/src/change/compare-runs.js
|
|
2970
|
-
function resolveKind(spec, values, autoCount) {
|
|
2971
|
-
if (spec?.kind)
|
|
2972
|
-
return spec.kind;
|
|
2973
|
-
return autoCount && isCountMetric(values) ? "count" : "continuous";
|
|
2974
|
-
}
|
|
2975
|
-
function compareRuns(input) {
|
|
2976
|
-
const config = withChangeDefaults(input.config);
|
|
2977
|
-
const rng = createRng(config.seed);
|
|
2978
|
-
const windowModes = config.modes ? detectWindowModes(input.dataset, input.baseline, input.candidate, config.modes, config.alpha) : new Map;
|
|
2979
|
-
const names = [...input.dataset.metrics.keys()].sort();
|
|
2980
|
-
const raw = names.map((metric) => {
|
|
2981
|
-
const arms = input.dataset.metrics.get(metric);
|
|
2982
|
-
const spec = input.metrics?.[metric];
|
|
2983
|
-
const higherIsBetter = spec?.higherIsBetter ?? false;
|
|
2984
|
-
const A = arms?.get(input.baseline);
|
|
2985
|
-
const B = arms?.get(input.candidate);
|
|
2986
|
-
if (!A || !B) {
|
|
2987
|
-
return { metric, higherIsBetter, skipped: `arms present: ${[...arms?.keys() ?? []].sort().join(",")}` };
|
|
2988
|
-
}
|
|
2989
|
-
const a = A.map((o) => o.value);
|
|
2990
|
-
const b = B.map((o) => o.value);
|
|
2991
|
-
const options = {
|
|
2992
|
-
kind: resolveKind(spec, [...a, ...b], config.autoCount),
|
|
2993
|
-
higherIsBetter,
|
|
2994
|
-
margin: spec?.margin ?? config.margin,
|
|
2995
|
-
alpha: config.alpha,
|
|
2996
|
-
nBoot: config.nBoot,
|
|
2997
|
-
ci: config.ci,
|
|
2998
|
-
paired: config.paired,
|
|
2999
|
-
minPairedBlocks: config.minPairedBlocks,
|
|
3000
|
-
minN: config.minN,
|
|
3001
|
-
countFloor: spec?.countFloor ?? config.countFloor
|
|
3002
|
-
};
|
|
3003
|
-
const result = compareMetric(a, b, options, rng, { a: A.map((o) => o.block), b: B.map((o) => o.block) });
|
|
3004
|
-
const wm = windowModes.get(splitMetricName(metric).window);
|
|
3005
|
-
if (!wm || !config.modes)
|
|
3006
|
-
return { metric, higherIsBetter, result };
|
|
3007
|
-
const modes = compareByMode(metric, A, B, result, wm, options, config.modes, config.seed, spec?.modeMargins);
|
|
3008
|
-
return { metric, higherIsBetter, result: { ...result, verdict: modes.verdict, note: modes.why }, modes };
|
|
3009
|
-
});
|
|
3010
|
-
const tested = raw.filter((c) => c.result);
|
|
3011
|
-
const adj = adjustPValues(tested.map((c) => c.result?.p ?? Number.NaN), config.multiplicity);
|
|
3012
|
-
const adjBy = new Map(tested.map((c, i) => [c.metric, adj[i]]));
|
|
3013
|
-
return {
|
|
3014
|
-
config,
|
|
3015
|
-
nRuns: input.dataset.nRuns,
|
|
3016
|
-
nInvalidRuns: input.dataset.invalidRuns.length,
|
|
3017
|
-
comparisons: raw.map((c) => adjBy.has(c.metric) ? { ...c, pAdjusted: adjBy.get(c.metric) } : c),
|
|
3018
|
-
windowModes: [...windowModes.values()].sort((x, y) => x.window.localeCompare(y.window))
|
|
3019
|
-
};
|
|
3020
|
-
}
|
|
3021
|
-
// packages/stats/dist/src/modes/bimodal.js
|
|
3022
|
-
var BIMODAL_CONFIG = withModeDefaults({
|
|
3023
|
-
minModeRuns: 3,
|
|
3024
|
-
minModeShare: 0.15,
|
|
3025
|
-
gapFactor: 3,
|
|
3026
|
-
minRelativeGap: 0.25
|
|
3027
|
-
});
|
|
3028
2613
|
// packages/stats/dist/src/calibration/gate.js
|
|
3029
2614
|
class MwuTables {
|
|
3030
2615
|
tables = new Map;
|
|
@@ -3169,6 +2754,11 @@ function suggestMargin(pool, kind, policy = DEFAULT_MARGIN_POLICY) {
|
|
|
3169
2754
|
}
|
|
3170
2755
|
|
|
3171
2756
|
// packages/stats/dist/src/calibration/aa-runs.js
|
|
2757
|
+
function resolveKind(spec, values, autoCount) {
|
|
2758
|
+
if (spec?.kind)
|
|
2759
|
+
return spec.kind;
|
|
2760
|
+
return autoCount && isCountMetric(values) ? "count" : "continuous";
|
|
2761
|
+
}
|
|
3172
2762
|
function calibrateModes(metric, obs, kind, ws, config, policy) {
|
|
3173
2763
|
const modes = MODE_LABELS.map((label) => {
|
|
3174
2764
|
const pool = obs.filter((o) => ws.labels[o.runId] === label).map((o) => o.value);
|
|
@@ -3397,6 +2987,8 @@ var readRunDataset = (csvText) => buildRunDataset(parseRunRows(csvText));
|
|
|
3397
2987
|
var obj2 = (v) => typeof v === "object" && v !== null ? v : {};
|
|
3398
2988
|
var num = (v) => typeof v === "number" && Number.isFinite(v) ? v : undefined;
|
|
3399
2989
|
function inputWall(w) {
|
|
2990
|
+
if (w.touchWall !== undefined)
|
|
2991
|
+
return w.touchWall;
|
|
3400
2992
|
const touch = num(obj2(w.raw.interaction).touchStartT);
|
|
3401
2993
|
const t1 = num(w.raw.t1);
|
|
3402
2994
|
return touch === undefined || t1 === undefined ? undefined : w.endWall - (t1 - touch);
|
|
@@ -3494,17 +3086,21 @@ function clusters(windows) {
|
|
|
3494
3086
|
}
|
|
3495
3087
|
return out;
|
|
3496
3088
|
}
|
|
3497
|
-
function onAppClock(windows) {
|
|
3089
|
+
function onAppClock(windows, version) {
|
|
3498
3090
|
const order = windows.map((w, i) => ({ w, i })).sort((a, b) => a.w.endWall - b.w.endWall || a.i - b.i);
|
|
3499
3091
|
const segment = new Map;
|
|
3500
3092
|
let seg = 0;
|
|
3501
3093
|
let prevId = -Infinity;
|
|
3094
|
+
let seen = new Set;
|
|
3502
3095
|
for (const { w } of order) {
|
|
3503
3096
|
const id = num(obj2(w.raw).id);
|
|
3504
3097
|
if (id === undefined)
|
|
3505
3098
|
continue;
|
|
3506
|
-
if (id <= prevId)
|
|
3099
|
+
if (version >= 23 ? seen.has(id) : id <= prevId) {
|
|
3507
3100
|
seg += 1;
|
|
3101
|
+
seen = new Set;
|
|
3102
|
+
}
|
|
3103
|
+
seen.add(id);
|
|
3508
3104
|
prevId = id;
|
|
3509
3105
|
segment.set(w, seg);
|
|
3510
3106
|
}
|
|
@@ -3524,7 +3120,8 @@ function onAppClock(windows) {
|
|
|
3524
3120
|
const t1 = num(w.raw.t1);
|
|
3525
3121
|
if (o === undefined || t0 === undefined || t1 === undefined)
|
|
3526
3122
|
return w;
|
|
3527
|
-
|
|
3123
|
+
const touch = version >= 23 ? num(obj2(w.raw.interaction).touchStartT) : undefined;
|
|
3124
|
+
return { ...w, startWall: t0 + o, endWall: t1 + o, ...touch === undefined ? {} : { touchWall: touch + o } };
|
|
3528
3125
|
});
|
|
3529
3126
|
}
|
|
3530
3127
|
|
|
@@ -3906,7 +3503,9 @@ function screenWindowModelMetrics(windows) {
|
|
|
3906
3503
|
var t0Of = (w) => num3(w.raw.t0);
|
|
3907
3504
|
var t1Of = (w) => num3(w.raw.t1);
|
|
3908
3505
|
var NEXT_ACTION_ENDS = ["next-press", "touch-lost", "gc-command"];
|
|
3909
|
-
function actionsOf(windows, steps = []) {
|
|
3506
|
+
function actionsOf(windows, steps = [], version) {
|
|
3507
|
+
const onClock = version !== undefined && version >= 23 ? onAppClock(windows, version) : undefined;
|
|
3508
|
+
const placed = onClock && new Map(windows.map((w, i) => [w, onClock[i]]));
|
|
3910
3509
|
const inputs = windows.filter((w) => isInputWindow(w) && t0Of(w) !== undefined).sort((a, b) => t0Of(a) - t0Of(b));
|
|
3911
3510
|
const journeyEnd = Math.max(Number.NEGATIVE_INFINITY, ...windows.map((w) => t1Of(w) ?? Number.NEGATIVE_INFINITY));
|
|
3912
3511
|
const twins = new Map;
|
|
@@ -3928,9 +3527,10 @@ function actionsOf(windows, steps = []) {
|
|
|
3928
3527
|
let endReason = next ? ACTION_END.nextAction : ACTION_END.journeyEnd;
|
|
3929
3528
|
const own1 = t1Of(w);
|
|
3930
3529
|
const ws = windowStart(w);
|
|
3931
|
-
const
|
|
3530
|
+
const holder = placed ? steps.find((s) => s.windows.some((x) => x.raw === w.raw)) : undefined;
|
|
3531
|
+
const step = placed ? holder?.endReason === "end-marker" ? holder : undefined : steps.find((s) => s.endReason === "end-marker" && ws >= s.startWall && ws <= s.endWall);
|
|
3932
3532
|
if (step && own1 !== undefined) {
|
|
3933
|
-
const markerT = own1 - (w.endWall - step.endWall);
|
|
3533
|
+
const markerT = own1 - ((placed?.get(w) ?? w).endWall - step.endWall);
|
|
3934
3534
|
if (markerT > t0 && markerT < endT) {
|
|
3935
3535
|
endT = markerT;
|
|
3936
3536
|
endReason = ACTION_END.stepEnd;
|
|
@@ -3959,9 +3559,9 @@ function actionsOf(windows, steps = []) {
|
|
|
3959
3559
|
return out;
|
|
3960
3560
|
}
|
|
3961
3561
|
var KEPT_FROM_INPUT = ["id", "interaction", "heapAfterGc"];
|
|
3962
|
-
function withActionWindows(windows, steps = []) {
|
|
3562
|
+
function withActionWindows(windows, steps = [], version) {
|
|
3963
3563
|
const byWindow = new Map;
|
|
3964
|
-
for (const a of actionsOf(windows, steps))
|
|
3564
|
+
for (const a of actionsOf(windows, steps, version))
|
|
3965
3565
|
byWindow.set(a.window, a);
|
|
3966
3566
|
return windows.map((w) => {
|
|
3967
3567
|
const a = byWindow.get(w);
|
|
@@ -3980,9 +3580,9 @@ function withActionWindows(windows, steps = []) {
|
|
|
3980
3580
|
return { name: w.name, startWall: w.startWall, endWall: w.endWall, raw };
|
|
3981
3581
|
});
|
|
3982
3582
|
}
|
|
3983
|
-
function actionMetrics(windows, steps = []) {
|
|
3583
|
+
function actionMetrics(windows, steps = [], version) {
|
|
3984
3584
|
const out = new Map;
|
|
3985
|
-
const actions = actionsOf(windows, steps);
|
|
3585
|
+
const actions = actionsOf(windows, steps, version);
|
|
3986
3586
|
const namesOf = new Map;
|
|
3987
3587
|
const updatersOf = (a) => {
|
|
3988
3588
|
const p = windowProfile(a.covered.raw);
|
|
@@ -4660,9 +4260,9 @@ var samplesAround = (samples, a, b) => {
|
|
|
4660
4260
|
const after = samples.find((s) => s.wall > b);
|
|
4661
4261
|
return after ? [after] : [];
|
|
4662
4262
|
};
|
|
4663
|
-
function v6Metrics(windows, samples, t0, t1) {
|
|
4263
|
+
function v6Metrics(windows, samples, t0, t1, version) {
|
|
4664
4264
|
const out = new Map;
|
|
4665
|
-
const placed = onAppClock(windows);
|
|
4265
|
+
const placed = onAppClock(windows, version);
|
|
4666
4266
|
const inputs = placed.filter(isInput).map((w) => w.startWall).sort((a, b) => a - b);
|
|
4667
4267
|
const inRun = samplesInRun(samples, t0, t1);
|
|
4668
4268
|
const gaps = new Map;
|
|
@@ -4728,7 +4328,7 @@ function windowEndFootprints(placed, records) {
|
|
|
4728
4328
|
}
|
|
4729
4329
|
return out;
|
|
4730
4330
|
}
|
|
4731
|
-
function v7Metrics(windows, steps, records) {
|
|
4331
|
+
function v7Metrics(windows, steps, records, version) {
|
|
4732
4332
|
const out = new Map;
|
|
4733
4333
|
const added = new Map;
|
|
4734
4334
|
for (const s of steps) {
|
|
@@ -4738,7 +4338,7 @@ function v7Metrics(windows, steps, records) {
|
|
|
4738
4338
|
}
|
|
4739
4339
|
for (const [name, v] of added)
|
|
4740
4340
|
out.set(`step:${name}.sampler_footprint_delta_mb`, median(v));
|
|
4741
|
-
const placed = onAppClock(windows).filter((w) => w.raw.endReason !== "timeout");
|
|
4341
|
+
const placed = onAppClock(windows, version).filter((w) => w.raw.endReason !== "timeout");
|
|
4742
4342
|
const ends = windowEndFootprints(placed, records);
|
|
4743
4343
|
const withEnd = [...ends.keys()];
|
|
4744
4344
|
const launchWindow = [...withEnd].sort((a, b) => a.startWall - b.startWall || a.endWall - b.endWall)[0];
|
|
@@ -4943,6 +4543,12 @@ function gcMetrics(markers, events, samples, windows, opts = {}) {
|
|
|
4943
4543
|
out.set(`step:${step}.${field}`, median3(vs));
|
|
4944
4544
|
return out;
|
|
4945
4545
|
}
|
|
4546
|
+
var END_READ_METRIC = "sdk.end_read";
|
|
4547
|
+
var manualReadFootprint = (samples, n) => {
|
|
4548
|
+
const reason = `endgc-read:${n}`;
|
|
4549
|
+
const found = samples.filter((s) => s.raw.reason === reason).sort((a, b) => a.wall - b.wall).at(-1);
|
|
4550
|
+
return found === undefined ? undefined : num9(obj9(found.raw.mem).footprintMb);
|
|
4551
|
+
};
|
|
4946
4552
|
var footprintsBySeq = (samples) => {
|
|
4947
4553
|
const bySeq = new Map;
|
|
4948
4554
|
for (const s of samples) {
|
|
@@ -4993,7 +4599,7 @@ function endsHome(windows, endCommandWall) {
|
|
|
4993
4599
|
const last = screens.filter((w) => w.startWall <= end).at(-1) ?? screens[screens.length - 1];
|
|
4994
4600
|
return screenChain(home.name) === screenChain(last.name) ? 1 : 0;
|
|
4995
4601
|
}
|
|
4996
|
-
function endMetrics(markers, events, samples, windows, launchMb) {
|
|
4602
|
+
function endMetrics(markers, events, samples, windows, launchMb, opts = {}) {
|
|
4997
4603
|
const out = new Map;
|
|
4998
4604
|
const n = markers.some((m) => m.phase === "start") ? runEndCommand(markers, events, windows) : null;
|
|
4999
4605
|
const command = n === null ? undefined : events.gc.find((g) => g.n === n && g.status === "ok");
|
|
@@ -5006,8 +4612,12 @@ function endMetrics(markers, events, samples, windows, launchMb) {
|
|
|
5006
4612
|
const g = n === null ? undefined : cleanOf(events, windows).get(n);
|
|
5007
4613
|
if (!g)
|
|
5008
4614
|
return out;
|
|
5009
|
-
const
|
|
4615
|
+
const post = g.post === undefined ? undefined : footprintsBySeq(samples).get(g.post);
|
|
4616
|
+
const manual = opts.manualRead ? manualReadFootprint(samples, g.n) : undefined;
|
|
4617
|
+
const footprint = manual ?? post;
|
|
5010
4618
|
if (footprint !== undefined) {
|
|
4619
|
+
if (opts.manualRead)
|
|
4620
|
+
out.set(END_READ_METRIC, manual === undefined ? 0 : 1);
|
|
5011
4621
|
out.set("mem.end_after_gc_mb", footprint);
|
|
5012
4622
|
if (launchMb !== undefined)
|
|
5013
4623
|
out.set("mem.retained_growth_mb", footprint - launchMb);
|
|
@@ -5068,12 +4678,15 @@ function keyFirstOpens(windows) {
|
|
|
5068
4678
|
}
|
|
5069
4679
|
|
|
5070
4680
|
// packages/derive/dist/src/derive.js
|
|
5071
|
-
var DERIVATION_VERSION =
|
|
5072
|
-
var DERIVATION_VERSIONS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22];
|
|
4681
|
+
var DERIVATION_VERSION = 23;
|
|
4682
|
+
var DERIVATION_VERSIONS = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23];
|
|
5073
4683
|
var windowsForVersion = (windows, version = DERIVATION_VERSION) => version >= 22 ? keyFirstOpens(windows) : windows;
|
|
4684
|
+
function stepWindowsFor(windows, version = DERIVATION_VERSION) {
|
|
4685
|
+
return windowsForVersion(version >= 16 ? withLaunchNames(windows).filter((w) => !isBackgroundWindow(w)) : windows, version);
|
|
4686
|
+
}
|
|
5074
4687
|
function alignRunSteps(markers, windows, opts = {}, version = DERIVATION_VERSION) {
|
|
5075
4688
|
const order = version >= 17 ? { byOrder: true } : {};
|
|
5076
|
-
return version >= 3 ? alignSteps(markers, onAppClock(windows), { startToleranceMs: APP_CLOCK_TOLERANCE_MS, ...order, ...opts }) : alignSteps(markers, windows, opts);
|
|
4689
|
+
return version >= 3 ? alignSteps(markers, onAppClock(windows, version), { startToleranceMs: APP_CLOCK_TOLERANCE_MS, ...order, ...opts }) : alignSteps(markers, windows, opts);
|
|
5077
4690
|
}
|
|
5078
4691
|
function samplerFrameBudgetMs(samples) {
|
|
5079
4692
|
let hz = 0;
|
|
@@ -5094,10 +4707,10 @@ function deriveRun(input) {
|
|
|
5094
4707
|
const renamed = new Map(input.windows.flatMap((w, i) => named[i].name !== w.name ? [[w.name, named[i].name]] : []));
|
|
5095
4708
|
const cpu = input.cpu && renamed.size ? new Map([...input.cpu].map(([k, v]) => [renamed.get(k) ?? k, v])) : input.cpu;
|
|
5096
4709
|
const background = version >= 16 ? named.filter(isBackgroundWindow) : [];
|
|
5097
|
-
const sdkWindows =
|
|
4710
|
+
const sdkWindows = stepWindowsFor(input.windows, version);
|
|
5098
4711
|
const off = clockOffsetMs === undefined ? {} : { clockOffsetMs };
|
|
5099
4712
|
const preSteps = version >= 11 && markers.length ? alignRunSteps(markers, sdkWindows, off, version) : [];
|
|
5100
|
-
const windows = version >= 11 ? withActionWindows(sdkWindows, preSteps) : sdkWindows;
|
|
4713
|
+
const windows = version >= 11 ? withActionWindows(sdkWindows, preSteps, version) : sdkWindows;
|
|
5101
4714
|
const steps = version >= 11 ? preSteps : alignRunSteps(markers, windows, clockOffsetMs === undefined ? {} : { clockOffsetMs }, version);
|
|
5102
4715
|
const valid = run.valid && windows.length > 0;
|
|
5103
4716
|
const final = run.valid && !valid ? { ...run, valid, invalid_reason: "no-windows" } : run;
|
|
@@ -5128,16 +4741,16 @@ function deriveRun(input) {
|
|
|
5128
4741
|
...version >= 3 ? v3Metrics(windows, steps, samples, final.t0, final.t1, ri?.samples) : [],
|
|
5129
4742
|
...version >= 4 ? v4Metrics(windows) : [],
|
|
5130
4743
|
...version >= 5 ? v5Metrics(windows) : [],
|
|
5131
|
-
...version >= 6 ? v6Metrics(windows, samples, final.t0, final.t1) : [],
|
|
4744
|
+
...version >= 6 ? v6Metrics(windows, samples, final.t0, final.t1, version) : [],
|
|
5132
4745
|
...version >= 10 ? unsettledMetrics(sdkWindows) : [],
|
|
5133
|
-
...version >= 11 ? actionMetrics(sdkWindows, preSteps) : [],
|
|
4746
|
+
...version >= 11 ? actionMetrics(sdkWindows, preSteps, version) : [],
|
|
5134
4747
|
...version >= 11 ? windowModelMetrics(sdkWindows) : [],
|
|
5135
4748
|
...version >= 13 ? screenWindowModelMetrics(sdkWindows) : [],
|
|
5136
4749
|
...version >= 11 ? routeChangeMetrics(sdkWindows, preSteps) : [],
|
|
5137
4750
|
...version >= 14 ? v14Metrics(steps, samplesInRun(samples, final.t0, final.t1)) : [],
|
|
5138
4751
|
...version >= 15 ? v15Metrics(windows, steps, samplesInRun(samples, final.t0, final.t1)) : [],
|
|
5139
4752
|
...version >= 21 ? hookMetrics(sdkWindows, samplesInRun(samples, final.t0, final.t1).some((x) => typeof x.raw.reason === "string" && /^window-(start|end):/.test(x.raw.reason))) : [],
|
|
5140
|
-
...version >= 7 ? v7Metrics(windows, steps, version >= 8 ? samplesInRun(samples, final.t0, final.t1) : undefined) : [],
|
|
4753
|
+
...version >= 7 ? v7Metrics(windows, steps, version >= 8 ? samplesInRun(samples, final.t0, final.t1) : undefined, version) : [],
|
|
5141
4754
|
...version >= 16 ? externalMetrics(windows) : [],
|
|
5142
4755
|
...version >= 16 ? causeSplitMetrics(sdkWindows) : [],
|
|
5143
4756
|
...version >= 16 ? bg.metrics : [],
|
|
@@ -5146,7 +4759,7 @@ function deriveRun(input) {
|
|
|
5146
4759
|
if (version >= 19) {
|
|
5147
4760
|
const events = input.events ?? { gc: [], appState: [] };
|
|
5148
4761
|
const launch = extra.get("mem.launch_mb");
|
|
5149
|
-
for (const [k, v] of [...endMetrics(markers, events, samplesInRun(samples, final.t0, final.t1), sdkWindows, launch), ...jsHeapMetrics(events, sdkWindows)])
|
|
4762
|
+
for (const [k, v] of [...endMetrics(markers, events, samplesInRun(samples, final.t0, final.t1), sdkWindows, launch, { manualRead: version >= 23 }), ...jsHeapMetrics(events, sdkWindows)])
|
|
5150
4763
|
extra.set(k, v);
|
|
5151
4764
|
}
|
|
5152
4765
|
const base = { run_id: final.run_id, arm: final.arm, block: final.block, valid: final.valid, invalid_reason: final.invalid_reason };
|
|
@@ -5211,7 +4824,11 @@ function parseWindows(logText) {
|
|
|
5211
4824
|
}
|
|
5212
4825
|
return out;
|
|
5213
4826
|
}
|
|
5214
|
-
function windowsInRun(windows, t0, t1) {
|
|
4827
|
+
function windowsInRun(windows, t0, t1, version) {
|
|
4828
|
+
if (version !== undefined && version >= 23) {
|
|
4829
|
+
const placed = onAppClock(windows, version);
|
|
4830
|
+
return windows.filter((_, i) => placed[i].startWall <= t1);
|
|
4831
|
+
}
|
|
5215
4832
|
return windows.filter((w) => w.startWall >= t0 && w.startWall <= t1);
|
|
5216
4833
|
}
|
|
5217
4834
|
// apps/agent/src/markers.ts
|
|
@@ -5345,74 +4962,6 @@ async function uploadRuns(deps, outDir, runs, opts) {
|
|
|
5345
4962
|
return { uploaded, skipped };
|
|
5346
4963
|
}
|
|
5347
4964
|
|
|
5348
|
-
// packages/verdicts/dist/src/headline.js
|
|
5349
|
-
var HEAD_WORD = {
|
|
5350
|
-
worse: "Worse",
|
|
5351
|
-
warn: "Warning",
|
|
5352
|
-
better: "Better",
|
|
5353
|
-
"no-change": "No change",
|
|
5354
|
-
inconclusive: "Not enough data yet",
|
|
5355
|
-
"cant-compare": "Can’t compare"
|
|
5356
|
-
};
|
|
5357
|
-
var NOT_BLOCKING = "not blocking";
|
|
5358
|
-
var NOT_JUDGED = "Not judged";
|
|
5359
|
-
|
|
5360
|
-
// apps/agent/src/summary.ts
|
|
5361
|
-
var fieldWord = (field) => measureDef(field)?.short ?? field;
|
|
5362
|
-
function plainMetric(metric) {
|
|
5363
|
-
const i = metric.lastIndexOf(".");
|
|
5364
|
-
if (i <= 0)
|
|
5365
|
-
return metric;
|
|
5366
|
-
const word = measureDef(metric.slice(i + 1))?.short;
|
|
5367
|
-
return word ? `${metric.slice(0, i)}: ${word}` : metric;
|
|
5368
|
-
}
|
|
5369
|
-
var g = (x) => Number(x.toPrecision(3)).toString();
|
|
5370
|
-
var pct = (ratio) => `${ratio >= 1 ? "+" : "-"}${Math.round(Math.abs(ratio - 1) * 100)}%`;
|
|
5371
|
-
var countsWords = (c) => `${HEAD_WORD.worse}: ${c.worse} ${c.warnings === undefined ? "" : `${HEAD_WORD.warn}: ${c.warnings} `}${HEAD_WORD.better}: ${c.better} ${HEAD_WORD["no-change"]}: ${c.noChange} ${HEAD_WORD.inconclusive}: ${c.inconclusive}`;
|
|
5372
|
-
var isPinned = (metric, pins) => pins.some((p) => metric === p || metric.startsWith(`${p}.`) || metric.startsWith(`step:${p}.`));
|
|
5373
|
-
function summarize2(input) {
|
|
5374
|
-
const valid = input.runs.filter((r) => r.valid).length;
|
|
5375
|
-
const lines = [`${valid}/${input.runs.length} runs valid`];
|
|
5376
|
-
const dropped = input.runs.filter((r) => !r.valid);
|
|
5377
|
-
if (dropped.length) {
|
|
5378
|
-
const reasons = new Map;
|
|
5379
|
-
for (const r of dropped)
|
|
5380
|
-
reasons.set(r.invalid_reason, (reasons.get(r.invalid_reason) ?? 0) + 1);
|
|
5381
|
-
lines[0] += ` (dropped: ${[...reasons].map(([k, n]) => `${n} ${k}`).join(", ")})`;
|
|
5382
|
-
}
|
|
5383
|
-
if (valid === 0)
|
|
5384
|
-
lines.push("No valid runs: nothing to compare. See <out>/<run>/test.log for the first failure.");
|
|
5385
|
-
const cmp = input.comparison;
|
|
5386
|
-
if (!cmp) {
|
|
5387
|
-
lines.push(`No baseline given; pass --baseline <earlier out dir> to see what changed. Results: ${input.outDir}`);
|
|
5388
|
-
return lines.join(`
|
|
5389
|
-
`);
|
|
5390
|
-
}
|
|
5391
|
-
const rows = cmp.comparisons.filter((c) => c.metric !== "run.status");
|
|
5392
|
-
const count = { worse: 0, better: 0, "no-change": 0, inconclusive: 0 };
|
|
5393
|
-
for (const c of rows)
|
|
5394
|
-
count[c.result ? c.result.verdict : "inconclusive"]++;
|
|
5395
|
-
lines.push(`Compared with ${input.baselineDir ?? "baseline"}: ${rows.length} metrics`);
|
|
5396
|
-
const changes = rows.filter((c) => c.result && (c.result.verdict === "worse" || c.result.verdict === "better") && Number.isFinite(c.result.ratio)).sort((a, b) => Math.abs(Math.log(b.result?.ratio || 1)) - Math.abs(Math.log(a.result?.ratio || 1))).slice(0, input.top ?? 5);
|
|
5397
|
-
if (changes.length) {
|
|
5398
|
-
lines.push("Top changes:");
|
|
5399
|
-
for (const c of changes) {
|
|
5400
|
-
const r = c.result;
|
|
5401
|
-
lines.push(` ${HEAD_WORD[r.verdict].padEnd(9)} ${plainMetric(c.metric)} ${pct(r.ratio)} (${g(r.baseline.location)} -> ${g(r.candidate.location)})`);
|
|
5402
|
-
}
|
|
5403
|
-
} else
|
|
5404
|
-
lines.push("No metric changed beyond its noise.");
|
|
5405
|
-
const pinned = rows.filter((c) => input.pins && isPinned(c.metric, input.pins) && !changes.includes(c));
|
|
5406
|
-
if (pinned.length) {
|
|
5407
|
-
lines.push("Pinned:");
|
|
5408
|
-
for (const c of pinned)
|
|
5409
|
-
lines.push(` ${HEAD_WORD[c.result ? c.result.verdict : "inconclusive"].padEnd(19)} ${plainMetric(c.metric)}`);
|
|
5410
|
-
}
|
|
5411
|
-
lines.push(countsWords({ worse: count.worse, better: count.better, noChange: count["no-change"], inconclusive: count.inconclusive }));
|
|
5412
|
-
return lines.join(`
|
|
5413
|
-
`);
|
|
5414
|
-
}
|
|
5415
|
-
|
|
5416
4965
|
// packages/present/dist/src/places.js
|
|
5417
4966
|
var PLACE_KIND_WORDS = {
|
|
5418
4967
|
screen: "Screen",
|
|
@@ -5491,11 +5040,31 @@ function placeKindWord(id) {
|
|
|
5491
5040
|
const { kind } = placeRef(id);
|
|
5492
5041
|
return kind === "other" ? "Window" : KIND_WORD[kind];
|
|
5493
5042
|
}
|
|
5494
|
-
|
|
5495
|
-
|
|
5043
|
+
var ELEMENT_SEPARATORS = /(?<!\d)\.|\.(?!\d)|[_-]+/g;
|
|
5044
|
+
var elementWords = (testId) => testId.replace(ELEMENT_SEPARATORS, " ").replace(/\s+/g, " ").trim();
|
|
5045
|
+
function placeLabel(id, screen) {
|
|
5046
|
+
if (id === "flow")
|
|
5047
|
+
return PLACE_KIND_WORDS.journey;
|
|
5048
|
+
const ref = placeRef(id);
|
|
5049
|
+
const after = ref.segment === "after3s" ? ` · ${PLACE_KIND_WORDS.after3s}` : "";
|
|
5050
|
+
if (ref.kind === "press" || ref.kind === "drag" || ref.kind === "step") {
|
|
5051
|
+
const element = elementWords(ref.stem.slice(ref.stem.indexOf(":") + 1).replace(OPEN_MARK, ""));
|
|
5052
|
+
return `${KIND_WORD[ref.kind]} · “${element}”${screen ? ` · ${screen}` : ""}${after}`;
|
|
5053
|
+
}
|
|
5054
|
+
const name = windowName(ref.stem);
|
|
5496
5055
|
const colon = name.indexOf(": ");
|
|
5497
|
-
return colon > 0 &&
|
|
5056
|
+
return `${colon > 0 && ref.kind !== "other" ? `${name.slice(0, colon)} · ${name.slice(colon + 2)}` : name}${after}`;
|
|
5498
5057
|
}
|
|
5058
|
+
// packages/present/dist/src/states.js
|
|
5059
|
+
var STATE_WORDS = {
|
|
5060
|
+
worse: "Worse",
|
|
5061
|
+
warning: "Slightly worse",
|
|
5062
|
+
better: "Better",
|
|
5063
|
+
"no-change": "No change",
|
|
5064
|
+
"cant-tell": "Can’t tell"
|
|
5065
|
+
};
|
|
5066
|
+
var INFO_WORDS = { heat: "Phone heated up" };
|
|
5067
|
+
var NOT_BLOCKING = "not blocking";
|
|
5499
5068
|
// packages/present/dist/src/values.js
|
|
5500
5069
|
var NO_VALUE = "–";
|
|
5501
5070
|
var FALLBACK_DECIMALS = 1;
|
|
@@ -5532,13 +5101,79 @@ function format(unit, decimals, v) {
|
|
|
5532
5101
|
const text = String(roundTo(v * shown.scale, decimals) + 0);
|
|
5533
5102
|
return shown.unit ? `${text} ${shown.unit}` : text;
|
|
5534
5103
|
}
|
|
5104
|
+
function valueText(measure, v) {
|
|
5105
|
+
const def = isMeasureId(measure) ? MEASURES[measure] : BREAKDOWN_MEASURES[measure];
|
|
5106
|
+
return format(def.unit, def.decimals, v);
|
|
5107
|
+
}
|
|
5535
5108
|
var KEY_SEPARATOR = ":";
|
|
5109
|
+
function breakdownMeasure(field) {
|
|
5110
|
+
const at = field.indexOf(KEY_SEPARATOR);
|
|
5111
|
+
if (at < 0 || !Object.hasOwn(DIMENSIONS, field.slice(0, at)))
|
|
5112
|
+
return null;
|
|
5113
|
+
const measure = field.slice(field.lastIndexOf(".") + 1);
|
|
5114
|
+
return Object.hasOwn(BREAKDOWN_MEASURES, measure) ? measure : null;
|
|
5115
|
+
}
|
|
5536
5116
|
function fieldValueText(field, v) {
|
|
5117
|
+
const breakdown = breakdownMeasure(field);
|
|
5118
|
+
if (breakdown)
|
|
5119
|
+
return valueText(breakdown, v);
|
|
5537
5120
|
const def = fieldDef(field.split(KEY_SEPARATOR)[0]);
|
|
5538
5121
|
return format(def?.unit, def?.decimals ?? FALLBACK_DECIMALS, v);
|
|
5539
5122
|
}
|
|
5123
|
+
// packages/present/dist/src/stability.js
|
|
5124
|
+
var UNSTABLE_WORDS = {
|
|
5125
|
+
short: "main varies too much here",
|
|
5126
|
+
label: `${STATE_WORDS["cant-tell"]} · main varies too much here`,
|
|
5127
|
+
line: "Main varies too much here to tell",
|
|
5128
|
+
ratioTip: "1.0 = main’s run-to-run noise alone would pass the warning step about once in 100 comparisons.",
|
|
5129
|
+
what: "Main’s own runs vary too much here to tell, so a comparison there would mostly measure noise. It is shown with its values, never judged, never blocks and is never counted. It is judged again once main settles."
|
|
5130
|
+
};
|
|
5131
|
+
// apps/agent/src/summary.ts
|
|
5132
|
+
var fieldWord = (field) => measureDef(field)?.short ?? field;
|
|
5133
|
+
function plainMetric(metric) {
|
|
5134
|
+
const i = metric.lastIndexOf(".");
|
|
5135
|
+
if (i <= 0)
|
|
5136
|
+
return metric;
|
|
5137
|
+
const word = measureDef(metric.slice(i + 1))?.short;
|
|
5138
|
+
return word ? `${metric.slice(0, i)}: ${word}` : metric;
|
|
5139
|
+
}
|
|
5140
|
+
var WORD_OF = {
|
|
5141
|
+
worse: STATE_WORDS.worse,
|
|
5142
|
+
warn: STATE_WORDS.warning,
|
|
5143
|
+
better: STATE_WORDS.better,
|
|
5144
|
+
"no-change": STATE_WORDS["no-change"],
|
|
5145
|
+
inconclusive: STATE_WORDS["cant-tell"]
|
|
5146
|
+
};
|
|
5147
|
+
var stateWord = (state) => WORD_OF[state];
|
|
5148
|
+
var WORD_WIDTH = Math.max(...Object.values(WORD_OF).map((w) => w.length));
|
|
5149
|
+
var countsWords = (c) => `${STATE_WORDS.worse}: ${c.worse} ${c.warnings === undefined ? "" : `${STATE_WORDS.warning}: ${c.warnings} `}${STATE_WORDS.better}: ${c.better} ${STATE_WORDS["no-change"]}: ${c.noChange} ${STATE_WORDS["cant-tell"]}: ${c.inconclusive}`;
|
|
5150
|
+
function summarize(input) {
|
|
5151
|
+
const valid = input.runs.filter((r) => r.valid).length;
|
|
5152
|
+
const lines = [`${valid}/${input.runs.length} runs valid`];
|
|
5153
|
+
const dropped = input.runs.filter((r) => !r.valid);
|
|
5154
|
+
if (dropped.length) {
|
|
5155
|
+
const reasons = new Map;
|
|
5156
|
+
for (const r of dropped)
|
|
5157
|
+
reasons.set(r.invalid_reason, (reasons.get(r.invalid_reason) ?? 0) + 1);
|
|
5158
|
+
lines[0] += ` (dropped: ${[...reasons].map(([k, n]) => `${n} ${k}`).join(", ")})`;
|
|
5159
|
+
}
|
|
5160
|
+
if (valid === 0)
|
|
5161
|
+
lines.push("No valid runs: nothing to upload. See <out>/<run>/test.log for the first failure.");
|
|
5162
|
+
lines.push(`Results: ${input.outDir}`);
|
|
5163
|
+
return lines.join(`
|
|
5164
|
+
`);
|
|
5165
|
+
}
|
|
5166
|
+
|
|
5540
5167
|
// apps/agent/src/remote.ts
|
|
5541
5168
|
var isBlocking = (s) => !!s?.verdicts && (s.verdicts.blocking ?? s.verdicts.state === "worse");
|
|
5169
|
+
var midLine = (label, metric) => !metric.startsWith("component:") && /^[A-Z][a-z]*(\s|$)/.test(label) ? `${label[0].toLowerCase()}${label.slice(1)}` : label;
|
|
5170
|
+
function heatNote(sc) {
|
|
5171
|
+
const n = sc.heated_runs ?? 0;
|
|
5172
|
+
if (!n)
|
|
5173
|
+
return null;
|
|
5174
|
+
const phone = sc.device ? `${sc.device.model}${sc.device.os ? ` · ${sc.device.os}` : ""}` : "unknown phone";
|
|
5175
|
+
return `${INFO_WORDS.heat} in ${n} of ${sc.runs.valid + sc.runs.invalid} runs · ${sc.scenario ?? "journey"} · ${phone}`;
|
|
5176
|
+
}
|
|
5542
5177
|
function movedItems(v, sc) {
|
|
5543
5178
|
const flat = v.items.filter((i) => i.scenario === sc.scenario && (!sc.device || !i.device_id || i.device_id === sc.device.id));
|
|
5544
5179
|
const h = sc.headline;
|
|
@@ -5556,18 +5191,23 @@ function movedItems(v, sc) {
|
|
|
5556
5191
|
baseline: g.baseline ?? same?.baseline ?? null,
|
|
5557
5192
|
candidate: g.candidate ?? same?.candidate ?? null,
|
|
5558
5193
|
change_text: g.change,
|
|
5559
|
-
...g.
|
|
5194
|
+
...g.finding ? { finding: g.finding } : {},
|
|
5195
|
+
...g.label ? { label_text: midLine(g.label, g.metric) } : {},
|
|
5560
5196
|
...sc.device ? { device_id: sc.device.id } : {}
|
|
5561
5197
|
};
|
|
5562
5198
|
});
|
|
5563
5199
|
return [...graded, ...flat.filter((i) => i.state === "better" && (i.judged ?? "outcome") === "outcome")];
|
|
5564
5200
|
}
|
|
5201
|
+
var placeOfRow = (g) => g.finding ?? placeRef(g.window).stem;
|
|
5565
5202
|
function movedCounts(sc) {
|
|
5566
5203
|
const h = sc.headline;
|
|
5567
5204
|
if (h?.errors === undefined || !h.counts)
|
|
5568
5205
|
return { worse: sc.counts.worse, warnings: 0, better: sc.counts.better, noChange: sc.counts.no_change, inconclusive: sc.counts.inconclusive };
|
|
5569
|
-
|
|
5206
|
+
const worse = new Set(h.errors.map(placeOfRow));
|
|
5207
|
+
const slightly = new Set((h.warnings ?? []).map(placeOfRow).filter((p) => !worse.has(p)));
|
|
5208
|
+
return { worse: worse.size, warnings: slightly.size, better: h.counts.better, noChange: h.counts.no_change, inconclusive: h.counts.inconclusive };
|
|
5570
5209
|
}
|
|
5210
|
+
var placeCount = (rows) => new Set(rows.map((i) => `${i.scenario}|${i.device_id ?? ""}|${i.finding ?? placeRef(i.step ?? "*").stem}`)).size;
|
|
5571
5211
|
var addCounts = (xs) => xs.reduce((a, c) => ({ worse: a.worse + c.worse, warnings: a.warnings + c.warnings, better: a.better + c.better, noChange: a.noChange + c.noChange, inconclusive: a.inconclusive + c.inconclusive }), { worse: 0, warnings: 0, better: 0, noChange: 0, inconclusive: 0 });
|
|
5572
5212
|
var setupLine = (h) => h?.measured_differently?.length ? `Measured differently (${h.measured_differently.join(", ")}) — ${NOT_BLOCKING}` : null;
|
|
5573
5213
|
var keyHeader = (t) => ({ "x-keepsnappy-key": t.key });
|
|
@@ -5593,17 +5233,10 @@ async function finishIngestRun(t, runId, invalidReason, farmSession) {
|
|
|
5593
5233
|
async function getRunSetRuns(t, id) {
|
|
5594
5234
|
return await call(t.fetch ?? fetch, t.url, `/v1/run-sets/${encodeURIComponent(id)}/runs`, keyHeader(t), undefined, "GET");
|
|
5595
5235
|
}
|
|
5596
|
-
async function fetchBaseline(t, q) {
|
|
5597
|
-
const p = new URLSearchParams;
|
|
5598
|
-
for (const [k, v] of Object.entries(q))
|
|
5599
|
-
if (v !== undefined && v !== "")
|
|
5600
|
-
p.set(k, String(v));
|
|
5601
|
-
return await call(t.fetch ?? fetch, t.url, `/v1/baselines?${p}`, keyHeader(t), undefined, "GET");
|
|
5602
|
-
}
|
|
5603
5236
|
var FEW_RUNS = "Few runs — result may be noisy";
|
|
5604
5237
|
var word = (s, server, h) => {
|
|
5605
|
-
const w = server
|
|
5606
|
-
return h?.blocking === false && h.measured_differently?.length && (w ===
|
|
5238
|
+
const w = server || stateWord(s ?? "inconclusive");
|
|
5239
|
+
return h?.blocking === false && h.measured_differently?.length && (w === STATE_WORDS.worse || s === "worse") ? `${w} (${NOT_BLOCKING})` : w;
|
|
5607
5240
|
};
|
|
5608
5241
|
var countsLine = (c) => countsWords(c);
|
|
5609
5242
|
var noCounts = (c) => !c.worse && !c.warnings && !c.better && !c.noChange && !c.inconclusive;
|
|
@@ -5638,11 +5271,16 @@ function formatRunSet(s, top = 5) {
|
|
|
5638
5271
|
lines.push(` ${FEW_RUNS}`);
|
|
5639
5272
|
if (sc.role !== "adhoc" && !noCounts(movedCounts(sc)))
|
|
5640
5273
|
lines.push(` ${countsLine(movedCounts(sc))}`);
|
|
5641
|
-
|
|
5274
|
+
if (heatNote(sc))
|
|
5275
|
+
lines.push(` ${heatNote(sc)}`);
|
|
5276
|
+
const allMoved = movedItems(v, sc).sort((a, b) => Number(b.state === "worse") - Number(a.state === "worse") || Number(a.grade === "warn") - Number(b.grade === "warn"));
|
|
5277
|
+
const moved = allMoved.slice(0, top);
|
|
5642
5278
|
for (const i of moved) {
|
|
5643
|
-
const what = i.label_text ? `${i.step && i.step !== "*" ? `${
|
|
5644
|
-
lines.push(` ${(i.state === "worse" && i.grade === "warn" ?
|
|
5279
|
+
const what = i.label_text ? `${i.step && i.step !== "*" ? `${placeLabel(i.step)}: ` : ""}${i.label_text}` : i.step ? `${placeLabel(i.step)}: ${fieldWord(i.metric)}` : plainMetric(i.metric);
|
|
5280
|
+
lines.push(` ${stateWord(i.state === "worse" && i.grade === "warn" ? "warn" : i.state).padEnd(WORD_WIDTH)} ${what}${i.change_text ? `: ${i.change_text}` : change(i.metric, i.baseline, i.candidate)}`);
|
|
5645
5281
|
}
|
|
5282
|
+
if (allMoved.length > moved.length)
|
|
5283
|
+
lines.push(` ${allMoved.length - moved.length} more`);
|
|
5646
5284
|
if (sc.dashboard_url)
|
|
5647
5285
|
lines.push(` ${sc.dashboard_url}`);
|
|
5648
5286
|
}
|
|
@@ -5663,6 +5301,7 @@ Runs uploaded for \`${fallback.branch}\`. The result appears on the dashboard: [
|
|
|
5663
5301
|
`;
|
|
5664
5302
|
}
|
|
5665
5303
|
const v = s.verdicts;
|
|
5304
|
+
const heated = v.scenarios.flatMap((sc) => heatNote(sc) ?? []);
|
|
5666
5305
|
const rows = v.scenarios.map((sc) => {
|
|
5667
5306
|
const c = movedCounts(sc);
|
|
5668
5307
|
return `| ${sc.scenario ?? "-"} | ${sc.device ? sc.device.model : "-"} | ${word(sc.state, sc.word, sc.headline)}${sc.low_data ? ` (${FEW_RUNS})` : ""} | ${noCounts(c) ? "–" : `${c.worse} / ${c.warnings} / ${c.better} / ${c.noChange} / ${c.inconclusive}`} | ${sc.dashboard_url ? `[open](${sc.dashboard_url})` : "-"} |`;
|
|
@@ -5673,9 +5312,10 @@ Runs uploaded for \`${fallback.branch}\`. The result appears on the dashboard: [
|
|
|
5673
5312
|
...setupLine(v.headline) ? [`${setupLine(v.headline)}.`, ""] : [],
|
|
5674
5313
|
`\`${s.branch}\`${s.commit_sha ? ` @ \`${s.commit_sha.slice(0, 7)}\`` : ""}, ${s.runs.valid}/${s.runs.total} runs used.${v.low_data ? ` ${FEW_RUNS}.` : ""}`,
|
|
5675
5314
|
"",
|
|
5676
|
-
`| Journey | Device | Result | ${
|
|
5315
|
+
`| Journey | Device | Result | ${STATE_WORDS.worse} / ${STATE_WORDS.warning} / ${STATE_WORDS.better} / ${STATE_WORDS["no-change"]} / ${STATE_WORDS["cant-tell"]} | Details |`,
|
|
5677
5316
|
"|---|---|---|---|---|",
|
|
5678
5317
|
...rows,
|
|
5318
|
+
...heated.length ? ["", ...heated.map((note) => `${note}.`)] : [],
|
|
5679
5319
|
"",
|
|
5680
5320
|
`[Open the dashboard](${s.dashboard_url})`,
|
|
5681
5321
|
""
|
|
@@ -5683,11 +5323,8 @@ Runs uploaded for \`${fallback.branch}\`. The result appears on the dashboard: [
|
|
|
5683
5323
|
`);
|
|
5684
5324
|
}
|
|
5685
5325
|
|
|
5686
|
-
// apps/agent/src/remote-verdict.ts
|
|
5687
|
-
import { join as join10 } from "node:path";
|
|
5688
|
-
|
|
5689
5326
|
// apps/agent/src/run.ts
|
|
5690
|
-
import { join as
|
|
5327
|
+
import { join as join8 } from "node:path";
|
|
5691
5328
|
// apps/agent/src/extract.ts
|
|
5692
5329
|
import { join as join6 } from "node:path";
|
|
5693
5330
|
|
|
@@ -8560,7 +8197,7 @@ function buildWindowTaskCards(input, windows, options = {}) {
|
|
|
8560
8197
|
if (tasks.length === 0)
|
|
8561
8198
|
return;
|
|
8562
8199
|
const durationMs = Math.max(1, window.t1Ms - window.t0Ms);
|
|
8563
|
-
out.push({ window, name:
|
|
8200
|
+
out.push({ window, name: placeLabel(window.key), set: cardSet(input, folded, classes, tasks, durationMs, offset) });
|
|
8564
8201
|
});
|
|
8565
8202
|
const outsideMs = outside.reduce((sum, task) => sum + task.durationMs, 0);
|
|
8566
8203
|
return { windows: out, outside: cardSet(input, folded, classes, outside, Math.max(1, outsideMs), offset) };
|
|
@@ -9189,53 +8826,6 @@ ${r.stderr}` : ""}`);
|
|
|
9189
8826
|
return { runs, rows };
|
|
9190
8827
|
}
|
|
9191
8828
|
|
|
9192
|
-
// apps/agent/src/verdict.ts
|
|
9193
|
-
import { join as join8 } from "node:path";
|
|
9194
|
-
var g2 = (x, d = 4) => x === undefined || !Number.isFinite(x) ? "-" : Number(x.toPrecision(d)).toString();
|
|
9195
|
-
var pad = (rows) => {
|
|
9196
|
-
const w = rows[0]?.map((_, i) => Math.max(...rows.map((r) => (r[i] ?? "").length))) ?? [];
|
|
9197
|
-
return rows.map((r) => r.map((c, i) => c.padEnd(w[i] ?? 0)).join(" ").trimEnd()).join(`
|
|
9198
|
-
`);
|
|
9199
|
-
};
|
|
9200
|
-
async function load(deps, outDir) {
|
|
9201
|
-
return readRunDataset(await deps.fs.readFile(join8(outDir, "metrics.csv")));
|
|
9202
|
-
}
|
|
9203
|
-
async function verdict(deps, outDir, opts = {}) {
|
|
9204
|
-
const dataset = await load(deps, outDir);
|
|
9205
|
-
const arms = [...new Set([...dataset.metrics.values()].flatMap((m) => [...m.keys()]))].sort();
|
|
9206
|
-
const baseline = opts.baseline ?? arms[0] ?? "A";
|
|
9207
|
-
const candidate = opts.candidate ?? arms.find((a) => a !== baseline) ?? "B";
|
|
9208
|
-
const result = compareRuns({ dataset, baseline, candidate, config: { autoCount: true, countFloor: DEFAULT_COUNT_FLOOR, ...opts.config } });
|
|
9209
|
-
const rows = [["metric", "base", "cand", "ratio", "ci95", "p", "p_adj", "verdict"]];
|
|
9210
|
-
for (const c of result.comparisons) {
|
|
9211
|
-
const r = c.result;
|
|
9212
|
-
if (!r)
|
|
9213
|
-
rows.push([c.metric, "-", "-", "-", "-", "-", "-", `skipped (${c.skipped})`]);
|
|
9214
|
-
else
|
|
9215
|
-
rows.push([c.metric, g2(r.baseline.location), g2(r.candidate.location), g2(r.ratio), `${g2(r.ratioCi95.lo, 3)}..${g2(r.ratioCi95.hi, 3)}`, g2(r.p, 3), g2(c.pAdjusted, 3), r.verdict]);
|
|
9216
|
-
}
|
|
9217
|
-
const table = `${baseline} -> ${candidate}: ${result.nRuns} runs, ${result.nInvalidRuns} invalid
|
|
9218
|
-
${pad(rows)}`;
|
|
9219
|
-
await deps.fs.writeFile(join8(outDir, "verdict.json"), `${JSON.stringify({ kind: "change", baseline, candidate, ...result }, null, 2)}
|
|
9220
|
-
`);
|
|
9221
|
-
return { baseline, candidate, result, table };
|
|
9222
|
-
}
|
|
9223
|
-
async function aa(deps, outDir, opts = {}) {
|
|
9224
|
-
const dataset = await load(deps, outDir);
|
|
9225
|
-
const results = aaRuns({ dataset, autoCount: true, config: { margin: relativeMargin(0.05), ...opts.config } });
|
|
9226
|
-
const rows = [["metric", "n", "median", "robustCV", "MDE@n", "suggested margin"]];
|
|
9227
|
-
for (const r of results) {
|
|
9228
|
-
if (r.skipped)
|
|
9229
|
-
rows.push([r.metric, String(r.pool.n), "-", "-", "-", r.skipped]);
|
|
9230
|
-
else
|
|
9231
|
-
rows.push([r.metric, String(r.pool.n), g2(r.pool.median), g2(r.pool.robustCv, 3), r.rows.map((x) => `${x.n}:${x.mde === null ? ">max" : g2(x.mde, 2)}`).join(" "), r.suggestedMargin.reason]);
|
|
9232
|
-
}
|
|
9233
|
-
const table = pad(rows);
|
|
9234
|
-
await deps.fs.writeFile(join8(outDir, "verdict.json"), `${JSON.stringify({ kind: "aa", results }, null, 2)}
|
|
9235
|
-
`);
|
|
9236
|
-
return { results, table };
|
|
9237
|
-
}
|
|
9238
|
-
|
|
9239
8829
|
// apps/agent/src/run.ts
|
|
9240
8830
|
function renameMetrics(rows, rename) {
|
|
9241
8831
|
const pairs = Object.entries(rename).flatMap(([a, b]) => [[a, b], [`step:${a}`, `step:${b}`]]);
|
|
@@ -9251,49 +8841,6 @@ function renameMetrics(rows, rename) {
|
|
|
9251
8841
|
return r;
|
|
9252
8842
|
});
|
|
9253
8843
|
}
|
|
9254
|
-
var csvCell = (s) => /[",\n]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
|
|
9255
|
-
async function scenariosOf(deps, outDir) {
|
|
9256
|
-
const out = new Map;
|
|
9257
|
-
let text = "";
|
|
9258
|
-
try {
|
|
9259
|
-
text = await deps.fs.readFile(join9(outDir, "runs.ndjson"));
|
|
9260
|
-
} catch {
|
|
9261
|
-
return out;
|
|
9262
|
-
}
|
|
9263
|
-
for (const line of text.split(`
|
|
9264
|
-
`)) {
|
|
9265
|
-
if (!line.trim())
|
|
9266
|
-
continue;
|
|
9267
|
-
const r = JSON.parse(line);
|
|
9268
|
-
if (r.scenario)
|
|
9269
|
-
out.set(r.run_id, r.scenario);
|
|
9270
|
-
}
|
|
9271
|
-
return out;
|
|
9272
|
-
}
|
|
9273
|
-
async function writeComparisonDataset(deps, baselineDir, outDir) {
|
|
9274
|
-
const dir = join9(outDir, "compare");
|
|
9275
|
-
const rows = ["run_id,arm,block,metric,value,valid,invalid_reason"];
|
|
9276
|
-
const maps = [await scenariosOf(deps, baselineDir), await scenariosOf(deps, outDir)];
|
|
9277
|
-
const several = new Set([...maps[0].values(), ...maps[1].values()]).size > 1;
|
|
9278
|
-
for (const [i, [src, arm, tag]] of [[baselineDir, "baseline", "b"], [outDir, "candidate", "c"]].entries()) {
|
|
9279
|
-
let text;
|
|
9280
|
-
try {
|
|
9281
|
-
text = await deps.fs.readFile(join9(src, "metrics.csv"));
|
|
9282
|
-
} catch {
|
|
9283
|
-
throw new Error(`no metrics.csv in ${src}: --baseline must be the out dir of an earlier run`);
|
|
9284
|
-
}
|
|
9285
|
-
for (const { cells: c } of parseCsv(text).rows) {
|
|
9286
|
-
const flow = maps[i].get(c.run_id ?? "");
|
|
9287
|
-
const metric = several && flow ? `${flow}/${c.metric ?? ""}` : c.metric ?? "";
|
|
9288
|
-
rows.push([`${tag}:${c.run_id ?? ""}`, arm, `${tag}${c.block ?? ""}`, metric, c.value ?? "", c.valid ?? "", c.invalid_reason ?? ""].map(csvCell).join(","));
|
|
9289
|
-
}
|
|
9290
|
-
}
|
|
9291
|
-
await deps.fs.mkdir(dir);
|
|
9292
|
-
await deps.fs.writeFile(join9(dir, "metrics.csv"), `${rows.join(`
|
|
9293
|
-
`)}
|
|
9294
|
-
`);
|
|
9295
|
-
return dir;
|
|
9296
|
-
}
|
|
9297
8844
|
async function keepsnappyRun(deps, source, opts) {
|
|
9298
8845
|
const { runs, rows: raw } = await measure(deps, source, undefined, {
|
|
9299
8846
|
arms: [{ name: "A" }],
|
|
@@ -9307,20 +8854,15 @@ async function keepsnappyRun(deps, source, opts) {
|
|
|
9307
8854
|
...opts.clockOffsetMs === undefined ? {} : { clockOffsetMs: opts.clockOffsetMs }
|
|
9308
8855
|
});
|
|
9309
8856
|
const rows = renameMetrics(raw, opts.config.steps.rename);
|
|
9310
|
-
await deps.fs.writeFile(
|
|
9311
|
-
|
|
9312
|
-
|
|
9313
|
-
const dir = await writeComparisonDataset(deps, opts.baselineDir, opts.outDir);
|
|
9314
|
-
v = await verdict(deps, dir, { baseline: "baseline", candidate: "candidate" });
|
|
9315
|
-
}
|
|
9316
|
-
const summary = summarize2({ runs, outDir: opts.outDir, ...opts.baselineDir ? { baselineDir: opts.baselineDir } : {}, ...v ? { comparison: v.result } : {}, pins: opts.config.steps.pin });
|
|
9317
|
-
return { runs, rows, ...v ? { verdict: v } : {}, summary };
|
|
8857
|
+
await deps.fs.writeFile(join8(opts.outDir, "metrics.csv"), toCsv(rows));
|
|
8858
|
+
const summary = summarize({ runs, outDir: opts.outDir });
|
|
8859
|
+
return { runs, rows, summary };
|
|
9318
8860
|
}
|
|
9319
8861
|
async function replayRuns(deps, sources, outDir, scenario) {
|
|
9320
8862
|
await deps.fs.mkdir(outDir);
|
|
9321
8863
|
const runs = [];
|
|
9322
8864
|
for (const src of sources) {
|
|
9323
|
-
const text = await deps.fs.readFile(
|
|
8865
|
+
const text = await deps.fs.readFile(join8(src.dir, "runs.ndjson")).catch(() => {
|
|
9324
8866
|
throw new Error(`--replay ${src.dir}: no runs.ndjson (not a keepsnappy out dir)`);
|
|
9325
8867
|
});
|
|
9326
8868
|
for (const line of text.split(`
|
|
@@ -9331,75 +8873,24 @@ async function replayRuns(deps, sources, outDir, scenario) {
|
|
|
9331
8873
|
if (!r.valid || src.arm && r.arm !== src.arm)
|
|
9332
8874
|
continue;
|
|
9333
8875
|
const id = `${String(runs.length + 1).padStart(3, "0")}-A`;
|
|
9334
|
-
await deps.fs.mkdir(
|
|
8876
|
+
await deps.fs.mkdir(join8(outDir, id));
|
|
9335
8877
|
for (const f of ["keepsnappy.log", "markers.ndjson", "test.log"]) {
|
|
9336
|
-
const body = await deps.fs.readFile(
|
|
8878
|
+
const body = await deps.fs.readFile(join8(src.dir, r.run_id, f)).catch(() => null);
|
|
9337
8879
|
if (body !== null)
|
|
9338
|
-
await deps.fs.writeFile(
|
|
8880
|
+
await deps.fs.writeFile(join8(outDir, id, f), body);
|
|
9339
8881
|
}
|
|
9340
8882
|
runs.push({ ...r, run_id: id, arm: "A", block: runs.length, scenario });
|
|
9341
8883
|
}
|
|
9342
8884
|
}
|
|
9343
8885
|
if (runs.length === 0)
|
|
9344
8886
|
throw new Error(`--replay: no valid runs in ${sources.map((x) => x.dir + (x.arm ? `:${x.arm}` : "")).join(", ")}`);
|
|
9345
|
-
await deps.fs.writeFile(
|
|
8887
|
+
await deps.fs.writeFile(join8(outDir, "runs.ndjson"), runs.map((r) => JSON.stringify(r)).join(`
|
|
9346
8888
|
`) + `
|
|
9347
8889
|
`);
|
|
9348
8890
|
await extract(deps, outDir, undefined);
|
|
9349
8891
|
return runs;
|
|
9350
8892
|
}
|
|
9351
8893
|
|
|
9352
|
-
// apps/agent/src/remote-verdict.ts
|
|
9353
|
-
var RUN_INFO = "keepsnappy-run.json";
|
|
9354
|
-
async function remoteVerdict(deps, outDir, api, opts = {}) {
|
|
9355
|
-
let info = {};
|
|
9356
|
-
try {
|
|
9357
|
-
info = JSON.parse(await deps.fs.readFile(join10(outDir, RUN_INFO)));
|
|
9358
|
-
} catch {}
|
|
9359
|
-
const flows = await scenariosOf(deps, outDir);
|
|
9360
|
-
const local = parseCsv(await deps.fs.readFile(join10(outDir, "metrics.csv"))).rows.map((r) => r.cells);
|
|
9361
|
-
const names = flows.size ? [...new Set(flows.values())].sort() : [opts.scenario];
|
|
9362
|
-
if (names.some((n) => !n))
|
|
9363
|
-
throw new Error("the runs in this out dir have no flow name: pass --scenario <flow file stem>");
|
|
9364
|
-
const results = [];
|
|
9365
|
-
const parts = [];
|
|
9366
|
-
for (const scenario of names) {
|
|
9367
|
-
const base = await fetchBaseline(api, {
|
|
9368
|
-
scenario,
|
|
9369
|
-
...info.device?.model ? { device_model: info.device.model } : {},
|
|
9370
|
-
...info.device?.os ? { os: info.device.os } : {},
|
|
9371
|
-
...opts.branch ? { branch: opts.branch } : {},
|
|
9372
|
-
...opts.limit ? { limit: opts.limit } : {}
|
|
9373
|
-
});
|
|
9374
|
-
if (base.rows.length === 0) {
|
|
9375
|
-
results.push({ scenario, baselineRuns: 0 });
|
|
9376
|
-
parts.push(`${scenario}: no baseline on ${base.branch} yet${info.device?.model ? ` for ${info.device.model}` : ""} (upload runs from ${base.branch} first)`);
|
|
9377
|
-
continue;
|
|
9378
|
-
}
|
|
9379
|
-
const judged = new Set(base.rows.map((r) => r.metric));
|
|
9380
|
-
const lines = ["run_id,arm,block,metric,value,valid,invalid_reason"];
|
|
9381
|
-
for (const r of base.rows)
|
|
9382
|
-
lines.push([`b:${r.run_id}`, "baseline", "b", r.metric, String(r.value), "true", ""].map(csvCell).join(","));
|
|
9383
|
-
for (const c of local) {
|
|
9384
|
-
if (flows.size && flows.get(c.run_id ?? "") !== scenario || !judged.has(c.metric ?? ""))
|
|
9385
|
-
continue;
|
|
9386
|
-
lines.push([`c:${c.run_id ?? ""}`, "candidate", `c${c.block ?? ""}`, c.metric ?? "", c.value ?? "", c.valid ?? "", c.invalid_reason ?? ""].map(csvCell).join(","));
|
|
9387
|
-
}
|
|
9388
|
-
const dir = join10(outDir, "remote-compare", scenario);
|
|
9389
|
-
await deps.fs.mkdir(dir);
|
|
9390
|
-
await deps.fs.writeFile(join10(dir, "metrics.csv"), `${lines.join(`
|
|
9391
|
-
`)}
|
|
9392
|
-
`);
|
|
9393
|
-
const output = await verdict(deps, dir, { baseline: "baseline", candidate: "candidate" });
|
|
9394
|
-
results.push({ scenario, baselineRuns: base.runs.length, output });
|
|
9395
|
-
parts.push(`${scenario}: ${base.runs.length} baseline runs from ${base.branch}${base.device ? ` on ${base.device.model}` : ""}
|
|
9396
|
-
${output.table}`);
|
|
9397
|
-
}
|
|
9398
|
-
return { results, table: parts.join(`
|
|
9399
|
-
|
|
9400
|
-
`) };
|
|
9401
|
-
}
|
|
9402
|
-
|
|
9403
8894
|
// apps/agent/src/defaults.ts
|
|
9404
8895
|
var DEFAULT_API_URL = "https://keepsnappy-api-922918702810.europe-west1.run.app";
|
|
9405
8896
|
|
|
@@ -9485,6 +8976,47 @@ async function pullRequestInfo(deps, env) {
|
|
|
9485
8976
|
const url = typeof pr.html_url === "string" && /^https?:\/\//.test(pr.html_url) ? pr.html_url : undefined;
|
|
9486
8977
|
return number || url ? { ...number ? { number } : {}, ...url ? { url } : {} } : undefined;
|
|
9487
8978
|
}
|
|
8979
|
+
var REMOTE = /^(?:refs\/remotes\/origin\/|refs\/heads\/|origin\/)/;
|
|
8980
|
+
function baseRefs(env) {
|
|
8981
|
+
const forced = env.KEEPSNAPPY_BASE_REF?.trim();
|
|
8982
|
+
if (forced)
|
|
8983
|
+
return [forced];
|
|
8984
|
+
const target = env.GITHUB_BASE_REF?.trim();
|
|
8985
|
+
return target ? [`origin/${target}`] : ["origin/main", "main"];
|
|
8986
|
+
}
|
|
8987
|
+
var SHA = /^[0-9a-f]{40,64}$/;
|
|
8988
|
+
var FIX = "fetch main and the commit, in CI with fetch-depth: 0, or pass --base-commit SHA";
|
|
8989
|
+
async function defaultBaseCommit(deps, env, o) {
|
|
8990
|
+
const refs = baseRefs(env).filter((r) => !r.startsWith("-"));
|
|
8991
|
+
if (o.branch && refs.some((r) => r.replace(REMOTE, "") === o.branch))
|
|
8992
|
+
return {};
|
|
8993
|
+
const tip = o.commit?.trim();
|
|
8994
|
+
if (!tip)
|
|
8995
|
+
return { warning: "base commit not recorded: no commit is recorded for the run set (pass --commit, or run in GitHub Actions)" };
|
|
8996
|
+
const shown = tip.slice(0, 12);
|
|
8997
|
+
if (tip.startsWith("-"))
|
|
8998
|
+
return { warning: `base commit not recorded: "${shown}" is not a commit` };
|
|
8999
|
+
const there = await deps.spawner.run("git", ["rev-parse", "--verify", "--quiet", `${tip}^{commit}`], { timeoutMs: 5000 }).catch(() => null);
|
|
9000
|
+
const tipSha = there && there.code === 0 ? there.stdout.trim() : "";
|
|
9001
|
+
if (!SHA.test(tipSha))
|
|
9002
|
+
return { warning: `base commit not recorded: the commit ${shown} is not in the checkout (${FIX})` };
|
|
9003
|
+
for (const ref of refs) {
|
|
9004
|
+
const r = await deps.spawner.run("git", ["merge-base", ref, tipSha], { timeoutMs: 5000 }).catch(() => null);
|
|
9005
|
+
const sha = r && r.code === 0 ? r.stdout.trim() : "";
|
|
9006
|
+
if (!SHA.test(sha))
|
|
9007
|
+
continue;
|
|
9008
|
+
return sha === tipSha ? {} : { sha };
|
|
9009
|
+
}
|
|
9010
|
+
return { warning: `base commit not recorded: git merge-base ${refs.join(" / ") || "<no ref>"} ${shown} failed (${FIX})` };
|
|
9011
|
+
}
|
|
9012
|
+
async function baseCommitFor(deps, env, o, print) {
|
|
9013
|
+
if (o.explicit)
|
|
9014
|
+
return o.explicit;
|
|
9015
|
+
const d = await defaultBaseCommit(deps, env, o);
|
|
9016
|
+
if (d.warning)
|
|
9017
|
+
print(d.warning);
|
|
9018
|
+
return d.sha;
|
|
9019
|
+
}
|
|
9488
9020
|
var wire = (c) => c.title || c.author || c.committedAt ? { ...c.title ? { title: c.title } : {}, ...c.author ? { author: c.author } : {}, ...c.committedAt ? { committed_at: c.committedAt } : {} } : undefined;
|
|
9489
9021
|
async function commitFields(deps, env, shas) {
|
|
9490
9022
|
const commit = shas.commit ? wire(await commitInfo(deps, shas.commit)) : undefined;
|
|
@@ -9495,6 +9027,12 @@ async function commitFields(deps, env, shas) {
|
|
|
9495
9027
|
|
|
9496
9028
|
// apps/agent/src/run-command.ts
|
|
9497
9029
|
var EXIT = { ok: 0, infra: 1, config: 2, worse: 3 };
|
|
9030
|
+
var RUN_INFO = "keepsnappy-run.json";
|
|
9031
|
+
var LOCAL_VERDICT_REMOVED = "there is no local verdict any more: use --upload; the server judges";
|
|
9032
|
+
function verdictCommand(print = console.error) {
|
|
9033
|
+
print(`keepsnappy: ${LOCAL_VERDICT_REMOVED}`);
|
|
9034
|
+
return EXIT.config;
|
|
9035
|
+
}
|
|
9498
9036
|
function gitContext(flags, env) {
|
|
9499
9037
|
const branch = flags.branch ?? (env.GITHUB_HEAD_REF || undefined) ?? env.GITHUB_REF_NAME;
|
|
9500
9038
|
const commit = flags.commit ?? env.GITHUB_SHA;
|
|
@@ -9549,11 +9087,12 @@ async function runCommand(deps, o, positionals, rt = { env: process.env, print:
|
|
|
9549
9087
|
const replayed = replay ? await replayRuns(deps, replay, out, flows[0].name) : null;
|
|
9550
9088
|
let runSet;
|
|
9551
9089
|
if (api && git.branch) {
|
|
9090
|
+
const base = await baseCommitFor(deps, rt.env, { explicit: o["base-commit"], branch: git.branch, commit: git.commit }, rt.print);
|
|
9552
9091
|
runSet = await createRunSet(api, {
|
|
9553
9092
|
branch: git.branch,
|
|
9554
9093
|
...git.commit ? { commit_sha: git.commit } : {},
|
|
9555
|
-
...
|
|
9556
|
-
...await commitFields(deps, rt.env, { commit: git.commit, base
|
|
9094
|
+
...base ? { base_commit_sha: base } : {},
|
|
9095
|
+
...await commitFields(deps, rt.env, { commit: git.commit, base }),
|
|
9557
9096
|
device: { ...device, provider },
|
|
9558
9097
|
scenarios: flows.map((f) => f.name),
|
|
9559
9098
|
journeys: await journeysFor(flows, { readFile: (p) => deps.fs.readFile(p), root: await repoRoot(deps) }),
|
|
@@ -9584,7 +9123,6 @@ async function runCommand(deps, o, positionals, rt = { env: process.env, print:
|
|
|
9584
9123
|
flows,
|
|
9585
9124
|
outDir: out,
|
|
9586
9125
|
repetitions: repeat,
|
|
9587
|
-
...o.baseline ? { baselineDir: resolve3(o.baseline) } : {},
|
|
9588
9126
|
...markers ? { markers } : {},
|
|
9589
9127
|
...o["clock-offset-ms"] ? { clockOffsetMs: Number(o["clock-offset-ms"]) } : {}
|
|
9590
9128
|
});
|
|
@@ -9594,7 +9132,7 @@ async function runCommand(deps, o, positionals, rt = { env: process.env, print:
|
|
|
9594
9132
|
${outcome.summary}`);
|
|
9595
9133
|
}
|
|
9596
9134
|
const info = { device, ...git.branch ? { branch: git.branch } : {}, ...git.commit ? { commit: git.commit } : {} };
|
|
9597
|
-
await deps.fs.writeFile(
|
|
9135
|
+
await deps.fs.writeFile(join9(out, RUN_INFO), `${JSON.stringify({ ...info, ...runSet ? { run_set_id: runSet.run_set_id } : {} }, null, 2)}
|
|
9598
9136
|
`);
|
|
9599
9137
|
if (!api)
|
|
9600
9138
|
return EXIT.ok;
|
|
@@ -9622,7 +9160,7 @@ ${formatRunSet(summary)}`);
|
|
|
9622
9160
|
}
|
|
9623
9161
|
await writeResult(deps, out, rt, runSet, summary, git.branch);
|
|
9624
9162
|
if (o["fail-on-worse"] && isBlocking(summary)) {
|
|
9625
|
-
rt.print(
|
|
9163
|
+
rt.print(`--fail-on-worse: a journey is ${STATE_WORDS.worse.toLowerCase()} than its baseline and blocks (exit 3)`);
|
|
9626
9164
|
return EXIT.worse;
|
|
9627
9165
|
}
|
|
9628
9166
|
return EXIT.ok;
|
|
@@ -9643,7 +9181,7 @@ var infra = (what) => (e) => {
|
|
|
9643
9181
|
};
|
|
9644
9182
|
async function writeResult(deps, out, rt, runSet, summary, branch) {
|
|
9645
9183
|
const url = summary?.dashboard_url ?? runSet.dashboard_url;
|
|
9646
|
-
await deps.fs.writeFile(
|
|
9184
|
+
await deps.fs.writeFile(join9(out, "keepsnappy-result.json"), `${JSON.stringify({ run_set_id: runSet.run_set_id, dashboard_url: url, state: summary?.verdicts?.state ?? null, summary }, null, 2)}
|
|
9647
9185
|
`);
|
|
9648
9186
|
const step = rt.env.GITHUB_STEP_SUMMARY;
|
|
9649
9187
|
if (step)
|
|
@@ -9764,8 +9302,35 @@ var nodeFs = {
|
|
|
9764
9302
|
var systemClock = { now: () => Date.now() };
|
|
9765
9303
|
var nodeDeps = (log = console.log) => ({ spawner: nodeSpawner, clock: systemClock, fs: nodeFs, log });
|
|
9766
9304
|
|
|
9305
|
+
// apps/agent/src/aa.ts
|
|
9306
|
+
import { join as join10 } from "node:path";
|
|
9307
|
+
var g = (x, d = 4) => x === undefined || !Number.isFinite(x) ? "-" : Number(x.toPrecision(d)).toString();
|
|
9308
|
+
var pad = (rows) => {
|
|
9309
|
+
const w = rows[0]?.map((_, i) => Math.max(...rows.map((r) => (r[i] ?? "").length))) ?? [];
|
|
9310
|
+
return rows.map((r) => r.map((c, i) => c.padEnd(w[i] ?? 0)).join(" ").trimEnd()).join(`
|
|
9311
|
+
`);
|
|
9312
|
+
};
|
|
9313
|
+
async function load(deps, outDir) {
|
|
9314
|
+
return readRunDataset(await deps.fs.readFile(join10(outDir, "metrics.csv")));
|
|
9315
|
+
}
|
|
9316
|
+
async function aa(deps, outDir, opts = {}) {
|
|
9317
|
+
const dataset = await load(deps, outDir);
|
|
9318
|
+
const results = aaRuns({ dataset, autoCount: true, config: { margin: relativeMargin(0.05), ...opts.config } });
|
|
9319
|
+
const rows = [["metric", "n", "median", "robustCV", "MDE@n", "suggested margin"]];
|
|
9320
|
+
for (const r of results) {
|
|
9321
|
+
if (r.skipped)
|
|
9322
|
+
rows.push([r.metric, String(r.pool.n), "-", "-", "-", r.skipped]);
|
|
9323
|
+
else
|
|
9324
|
+
rows.push([r.metric, String(r.pool.n), g(r.pool.median), g(r.pool.robustCv, 3), r.rows.map((x) => `${x.n}:${x.mde === null ? ">max" : g(x.mde, 2)}`).join(" "), r.suggestedMargin.reason]);
|
|
9325
|
+
}
|
|
9326
|
+
const table = pad(rows);
|
|
9327
|
+
await deps.fs.writeFile(join10(outDir, "aa.json"), `${JSON.stringify({ kind: "aa", results }, null, 2)}
|
|
9328
|
+
`);
|
|
9329
|
+
return { results, table };
|
|
9330
|
+
}
|
|
9331
|
+
|
|
9767
9332
|
// apps/agent/src/farm.ts
|
|
9768
|
-
import { dirname as dirname3, join as
|
|
9333
|
+
import { dirname as dirname3, join as join12, resolve as resolve5 } from "node:path";
|
|
9769
9334
|
|
|
9770
9335
|
// packages/appium-driver/dist/src/links.js
|
|
9771
9336
|
var ALERT_POLL_MS = 250;
|
|
@@ -9785,7 +9350,7 @@ function linkArgs(args = "") {
|
|
|
9785
9350
|
return [...fields].map(([k, v]) => `${k}=${v}`).join("&");
|
|
9786
9351
|
}
|
|
9787
9352
|
var activationLink = (scheme, token, args = "") => `${route(scheme)}?activate=1&${linkArgs(args)}&runToken=${token}`;
|
|
9788
|
-
var flushLink = (scheme) => `${route(scheme)}?flush=1&activate=0`;
|
|
9353
|
+
var flushLink = (scheme, ack) => `${route(scheme)}?flush=1&activate=0${ack === undefined ? "" : `&ack=${ack}`}`;
|
|
9789
9354
|
var gcLink = (scheme, n) => `${route(scheme)}?cmd=gc&n=${n}`;
|
|
9790
9355
|
|
|
9791
9356
|
// packages/appium-driver/dist/src/selectors.js
|
|
@@ -9981,9 +9546,9 @@ class Screen {
|
|
|
9981
9546
|
var modelOf2 = (device) => device.replace(/-[^-]*$/, "");
|
|
9982
9547
|
var versionOf = (device) => device.replace(/^.*-/, "");
|
|
9983
9548
|
var FLUSH_ID = "keepsnappy.flushed";
|
|
9984
|
-
function parseBroadcastAnswer(out, n) {
|
|
9549
|
+
function parseBroadcastAnswer(out, n, cmd = "gc") {
|
|
9985
9550
|
const data = /data="([^"]*)"/.exec(String(out ?? ""))?.[1];
|
|
9986
|
-
return data !== undefined && data.startsWith(
|
|
9551
|
+
return data !== undefined && data.startsWith(`${cmd}:${n} `) ? data : null;
|
|
9987
9552
|
}
|
|
9988
9553
|
var NO_WAIT_MS = 0;
|
|
9989
9554
|
var NEW_COMMAND_TIMEOUT_S = 300;
|
|
@@ -9998,6 +9563,7 @@ var ANDROID = {
|
|
|
9998
9563
|
"appium:autoGrantPermissions": true,
|
|
9999
9564
|
"appium:newCommandTimeout": NEW_COMMAND_TIMEOUT_S
|
|
10000
9565
|
}),
|
|
9566
|
+
absentAttributeText: "null",
|
|
10001
9567
|
settings: { waitForIdleTimeout: NO_WAIT_MS, actionAcknowledgmentTimeout: NO_WAIT_MS, waitForSelectorTimeout: NO_WAIT_MS, enableMultiWindows: true, ignoreUnimportantViews: true },
|
|
10002
9568
|
locators: androidLocators,
|
|
10003
9569
|
appArg: (appId) => ({ appId }),
|
|
@@ -10024,6 +9590,7 @@ var ANDROID = {
|
|
|
10024
9590
|
flush: {
|
|
10025
9591
|
lookup: { using: "-android uiautomator", value: `new UiSelector().description("${FLUSH_ID}")` },
|
|
10026
9592
|
statusAttribute: "text",
|
|
9593
|
+
detailAttribute: "hint",
|
|
10027
9594
|
fallback: { lookup: { using: "xpath", value: `//*[@content-desc="${FLUSH_ID}"]` }, setting: "allowInvisibleElements" }
|
|
10028
9595
|
},
|
|
10029
9596
|
async readScreen(driver) {
|
|
@@ -10040,16 +9607,18 @@ var ANDROID = {
|
|
|
10040
9607
|
singleFindFirst: true,
|
|
10041
9608
|
anyOfIds: androidAnyOfIds,
|
|
10042
9609
|
uploadFields: {},
|
|
10043
|
-
|
|
10044
|
-
|
|
10045
|
-
action: `${driver.bundleId}.keepsnappy.COMMAND`,
|
|
10046
|
-
package: driver.bundleId,
|
|
10047
|
-
extras: [["s", "cmd", "gc"], ["i", "n", String(n)]],
|
|
10048
|
-
flags: "0x10000000"
|
|
10049
|
-
});
|
|
10050
|
-
return parseBroadcastAnswer(out, n);
|
|
10051
|
-
}
|
|
9610
|
+
gcCommand: (driver, n) => androidCommand(driver, "gc", n),
|
|
9611
|
+
memReadCommand: (driver, n) => androidCommand(driver, "memread", n)
|
|
10052
9612
|
};
|
|
9613
|
+
async function androidCommand(driver, cmd, n) {
|
|
9614
|
+
const out = await driver.execute("mobile: broadcast", {
|
|
9615
|
+
action: `${driver.bundleId}.keepsnappy.COMMAND`,
|
|
9616
|
+
package: driver.bundleId,
|
|
9617
|
+
extras: [["s", "cmd", cmd], ["i", "n", String(n)]],
|
|
9618
|
+
flags: "0x10000000"
|
|
9619
|
+
});
|
|
9620
|
+
return parseBroadcastAnswer(out, n, cmd);
|
|
9621
|
+
}
|
|
10053
9622
|
|
|
10054
9623
|
// packages/appium-driver/dist/src/webdriver.js
|
|
10055
9624
|
class WdError extends Error {
|
|
@@ -10063,18 +9632,19 @@ class WdError extends Error {
|
|
|
10063
9632
|
var SDK_FLUSH_WORST_CASE_MS = 315000;
|
|
10064
9633
|
var FLUSH_MARGIN_MS = 30000;
|
|
10065
9634
|
var FLUSH_WAIT_MS = SDK_FLUSH_WORST_CASE_MS + FLUSH_MARGIN_MS;
|
|
9635
|
+
var FLUSH_STATUSES = new Set(["ok", "error", "off"]);
|
|
10066
9636
|
// packages/appium-driver/dist/src/version.js
|
|
10067
9637
|
import { createHash } from "node:crypto";
|
|
10068
9638
|
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
10069
|
-
import { dirname as dirname2, join as
|
|
9639
|
+
import { dirname as dirname2, join as join11, relative as relative2, resolve as resolve4 } from "node:path";
|
|
10070
9640
|
import { fileURLToPath } from "node:url";
|
|
10071
9641
|
function driverVersion() {
|
|
10072
|
-
const built = "0.1.0-alpha.
|
|
9642
|
+
const built = "0.1.0-alpha.15";
|
|
10073
9643
|
if (built)
|
|
10074
9644
|
return built;
|
|
10075
9645
|
let dir = dirname2(fileURLToPath(import.meta.url));
|
|
10076
9646
|
for (let i = 0;i < 3; i++, dir = dirname2(dir)) {
|
|
10077
|
-
const file =
|
|
9647
|
+
const file = join11(dir, "package.json");
|
|
10078
9648
|
if (!existsSync(file))
|
|
10079
9649
|
continue;
|
|
10080
9650
|
try {
|
|
@@ -10118,14 +9688,10 @@ function journeyVersion(modulePath) {
|
|
|
10118
9688
|
hash.update(key).update("\x00").update(text).update("\x00");
|
|
10119
9689
|
return hash.digest("hex").slice(0, 12);
|
|
10120
9690
|
}
|
|
10121
|
-
// packages/verdicts/dist/src/heat.js
|
|
10122
|
-
var HEAT_METRIC = "sampler.thermal.heats_up";
|
|
10123
|
-
var HEAT_LABEL = MEASURES[HEAT_METRIC].label;
|
|
10124
|
-
|
|
10125
9691
|
// apps/agent/src/slack.ts
|
|
10126
9692
|
var COLOR = { error: "#E01E5A", warn: "#ECB22E", better: "#2EB67D", none: "#9AA0A6" };
|
|
10127
9693
|
var MAX_ROWS = 3;
|
|
10128
|
-
var NAME_W =
|
|
9694
|
+
var NAME_W = 30;
|
|
10129
9695
|
var COLUMN_GAP = 1;
|
|
10130
9696
|
function metricWord(item) {
|
|
10131
9697
|
if (item.label_text)
|
|
@@ -10135,8 +9701,6 @@ function metricWord(item) {
|
|
|
10135
9701
|
return measureDef(item.metric)?.short ?? item.metric.replace(/_/g, " ");
|
|
10136
9702
|
}
|
|
10137
9703
|
function change2(item) {
|
|
10138
|
-
if (item.metric === HEAT_METRIC && item.baseline !== null && item.candidate !== null)
|
|
10139
|
-
return `+${Math.round((item.candidate - item.baseline) * 100)} pp`;
|
|
10140
9704
|
if (item.change_text)
|
|
10141
9705
|
return item.change_text.replace(/ \(.*\)$/, "");
|
|
10142
9706
|
const { baseline: b, candidate: c } = item;
|
|
@@ -10149,19 +9713,16 @@ function change2(item) {
|
|
|
10149
9713
|
}
|
|
10150
9714
|
var size = (i) => i.baseline === null || i.candidate === null ? 0 : i.baseline === 0 ? i.candidate === 0 ? 0 : Number.POSITIVE_INFINITY : Math.abs(Math.log(Math.max(i.candidate, 0.000000001) / i.baseline));
|
|
10151
9715
|
var bySize = (a, b) => size(a) === size(b) ? 0 : size(a) > size(b) ? -1 : 1;
|
|
10152
|
-
var worseOrder = (a, b) => Number(a.grade === "warn") - Number(b.grade === "warn") ||
|
|
9716
|
+
var worseOrder = (a, b) => Number(a.grade === "warn") - Number(b.grade === "warn") || bySize(a, b);
|
|
10153
9717
|
var isWorse = (i) => i.state === "worse";
|
|
10154
9718
|
var isBetter = (i) => i.state === "better";
|
|
10155
9719
|
var differs = (sc) => sc?.headline?.measured_differently ?? [];
|
|
10156
|
-
var
|
|
9720
|
+
var scenarioBlocks = (sc) => sc?.headline?.blocking !== false;
|
|
9721
|
+
var isRed = (i, sc) => i.grade !== "warn" && scenarioBlocks(sc);
|
|
10157
9722
|
var mark = (i, sc) => isBetter(i) ? "\uD83D\uDFE2" : isRed(i, sc) ? "\uD83D\uDD34" : "\uD83D\uDFE0";
|
|
10158
9723
|
var fit = (s, w) => s.length > w ? `${s.slice(0, w - 1)}…` : s.padEnd(w);
|
|
10159
9724
|
var right = (s, w) => s.padStart(w);
|
|
10160
9725
|
var phone = (sc) => sc?.device ? `${sc.device.model}${sc.device.os ? ` · ${sc.device.os}` : ""}` : "unknown phone";
|
|
10161
|
-
var kindWord = (step) => {
|
|
10162
|
-
const k = placeKindWord(step ?? "");
|
|
10163
|
-
return k === "Window" || k === PLACE_KIND_WORDS.journey ? "" : `${k.toLowerCase()} · `;
|
|
10164
|
-
};
|
|
10165
9726
|
function commitLine(ctx) {
|
|
10166
9727
|
const c = ctx.commit;
|
|
10167
9728
|
if (!c)
|
|
@@ -10177,8 +9738,8 @@ function prTable(items, sc) {
|
|
|
10177
9738
|
const w = [NAME_W, widest("main", (c) => c.main), widest("PR", (c) => c.pr), widest("change", (c) => c.change)];
|
|
10178
9739
|
const head = `${" ".repeat(3)}${fit("", w[0])}${right("main", w[1])}${right("PR", w[2])}${right("change", w[3])}`;
|
|
10179
9740
|
const rows = cells.flatMap((c) => [
|
|
10180
|
-
`${mark(c.i, sc)} ${fit(
|
|
10181
|
-
` ${
|
|
9741
|
+
`${mark(c.i, sc)} ${fit(placeLabel(c.i.step ?? "*"), w[0])}${right(c.main, w[1])}${right(c.pr, w[2])}${right(c.change, w[3])}`,
|
|
9742
|
+
` ${metricWord(c.i)}`
|
|
10182
9743
|
]);
|
|
10183
9744
|
return ["```", head, ...rows, "```"].join(`
|
|
10184
9745
|
`);
|
|
@@ -10187,7 +9748,7 @@ function mainCard(i) {
|
|
|
10187
9748
|
const label = (s) => ` ${s.padEnd(16)}`;
|
|
10188
9749
|
return [
|
|
10189
9750
|
"```",
|
|
10190
|
-
|
|
9751
|
+
placeLabel(i.step ?? "*"),
|
|
10191
9752
|
`${label(metricWord(i))}${right(fieldValueText(i.metric, i.candidate), 9)}`,
|
|
10192
9753
|
`${label("usually")}${right(fieldValueText(i.metric, i.baseline), 9)}`,
|
|
10193
9754
|
`${label("change")}${right(change2(i), 9)}`,
|
|
@@ -10200,7 +9761,7 @@ function blockedMessage(s, ctx) {
|
|
|
10200
9761
|
const blocks = [
|
|
10201
9762
|
section(`*${text}*
|
|
10202
9763
|
${ctx.blocked?.reason ?? "No iteration sent any data."}
|
|
10203
|
-
Nothing was judged, so this is not a "no
|
|
9764
|
+
Nothing was judged, so this is not a "${STATE_WORDS["no-change"].toLowerCase()}".`),
|
|
10204
9765
|
...commitLine(ctx) ? [context(commitLine(ctx))] : [],
|
|
10205
9766
|
...s ? [section(`\uD83D\uDC49 <${s.dashboard_url}|See what happened>`)] : []
|
|
10206
9767
|
];
|
|
@@ -10212,21 +9773,31 @@ function slackMessage(s, ctx) {
|
|
|
10212
9773
|
const v = s?.verdicts;
|
|
10213
9774
|
if (!s || !v)
|
|
10214
9775
|
return null;
|
|
10215
|
-
const items = v.scenarios.flatMap((sc) => movedItems(v, sc));
|
|
9776
|
+
const items = v.scenarios.flatMap((sc) => sc.state === "inconclusive" ? [] : movedItems(v, sc));
|
|
10216
9777
|
const scenarioOf = (i) => v.scenarios.find((sc) => sc.scenario === i.scenario && (!i.device_id || sc.device?.id === i.device_id)) ?? v.scenarios.find((sc) => sc.scenario === i.scenario);
|
|
10217
9778
|
const isPr = v.scenarios.some((sc) => sc.role === "comparison");
|
|
10218
9779
|
const worse = items.filter(isWorse).sort(worseOrder);
|
|
10219
9780
|
const better = items.filter(isBetter).sort(bySize);
|
|
10220
9781
|
const commit = commitLine(ctx);
|
|
9782
|
+
const heated = v.scenarios.filter((sc) => heatNote(sc));
|
|
9783
|
+
const heat = heated.map((sc) => context(heatNote(sc)));
|
|
9784
|
+
const errorRows = worse.filter((i) => i.grade !== "warn");
|
|
9785
|
+
const warningRows = worse.filter((i) => i.grade === "warn");
|
|
9786
|
+
const errors = placeCount(errorRows);
|
|
9787
|
+
const warns = placeCount(worse) - errors;
|
|
10221
9788
|
if (isPr) {
|
|
10222
|
-
if (!worse.length)
|
|
10223
|
-
|
|
10224
|
-
|
|
10225
|
-
|
|
9789
|
+
if (!worse.length) {
|
|
9790
|
+
if (!heat.length)
|
|
9791
|
+
return null;
|
|
9792
|
+
const head = `⚪ ${ctx.branch}: ${INFO_WORDS.heat}`;
|
|
9793
|
+
const blocks = [section(`*${head}*`), ...heat, ...commit ? [context(commit)] : [], section(`\uD83D\uDC49 <${heated[0].dashboard_url ?? s.dashboard_url}|View the comparison>`)];
|
|
9794
|
+
return { text: head, attachments: [{ color: COLOR.none, blocks }] };
|
|
9795
|
+
}
|
|
10226
9796
|
const red = worse.some((i) => isRed(i, scenarioOf(i)));
|
|
10227
9797
|
const graded = worse.some((i) => i.grade);
|
|
10228
9798
|
const notBlocking = errors > 0 && !red;
|
|
10229
|
-
const
|
|
9799
|
+
const places = (word, n) => `${word.toLowerCase()} in ${n} place${n > 1 ? "s" : ""}`;
|
|
9800
|
+
const counts = graded ? [errors ? `${places(STATE_WORDS.worse, errors)}${notBlocking ? ` (${NOT_BLOCKING})` : ""}` : "", warns ? places(STATE_WORDS.warning, warns) : ""].filter(Boolean).join(", ") : places(STATE_WORDS.worse, placeCount(worse));
|
|
10230
9801
|
const head = `${red ? "\uD83D\uDD34" : "\uD83D\uDFE0"} ${ctx.branch}: ${counts}`;
|
|
10231
9802
|
const blocks = [section(`*${head}*`)];
|
|
10232
9803
|
const groups = new Map;
|
|
@@ -10242,27 +9813,29 @@ function slackMessage(s, ctx) {
|
|
|
10242
9813
|
const nC = Math.max(...rows.map((r) => r.n_candidate ?? 0)) || (sc?.runs?.valid ?? 0);
|
|
10243
9814
|
const runs = nB && nC ? ` · ${nC} PR run${nC === 1 ? "" : "s"} vs ${nB} main run${nB === 1 ? "" : "s"}` : "";
|
|
10244
9815
|
const note = differs(sc).length ? `
|
|
10245
|
-
_Measured differently (${differs(sc).join(", ")}) — ${NOT_BLOCKING}_` : "";
|
|
9816
|
+
_Measured differently (${differs(sc).join(", ")})${scenarioBlocks(sc) ? "" : ` — ${NOT_BLOCKING}`}_` : "";
|
|
10246
9817
|
blocks.push(section(`*${sc?.scenario ?? "journey"}* · ${phone(sc)}${runs}${note}
|
|
10247
9818
|
${prTable(rows, sc)}`));
|
|
10248
9819
|
}
|
|
10249
9820
|
const more = worse.length - shown;
|
|
9821
|
+
blocks.push(...heat);
|
|
10250
9822
|
if (commit)
|
|
10251
9823
|
blocks.push(context(commit));
|
|
10252
9824
|
blocks.push(section(`\uD83D\uDC49 <${groups.keys().next().value?.dashboard_url ?? s.dashboard_url}|View the comparison>${more > 0 ? ` · ${more} more` : ""}`));
|
|
10253
9825
|
return { text: head, attachments: [{ color: red ? COLOR.error : COLOR.warn, blocks }] };
|
|
10254
9826
|
}
|
|
10255
|
-
|
|
9827
|
+
const improved = better.length > 0 || v.state === "better";
|
|
9828
|
+
if (!worse.length && !improved && !ctx.always && !heat.length)
|
|
10256
9829
|
return null;
|
|
10257
|
-
const parts = [worse.
|
|
10258
|
-
const quiet = (v.state === null || v.state === "inconclusive") && v.word
|
|
10259
|
-
const status = parts.length ? `${ctx.branch}: ${parts.join(", ")}` : quiet.startsWith("Main: ") && ctx.branch === "main" ? quiet : `${ctx.branch}: ${quiet}`;
|
|
10260
|
-
const head = `${worse.length ? worse.some((i) => i
|
|
9830
|
+
const parts = [errors ? `${errors} ${STATE_WORDS.worse.toLowerCase()}` : "", warns ? `${warns} ${STATE_WORDS.warning.toLowerCase()}` : "", better.length ? `${placeCount(better)} ${STATE_WORDS.better.toLowerCase()}` : ""].filter(Boolean);
|
|
9831
|
+
const quiet = improved ? STATE_WORDS.better.toLowerCase() : (v.state === null || v.state === "inconclusive") && v.word ? v.word : STATE_WORDS["no-change"].toLowerCase();
|
|
9832
|
+
const status = parts.length ? `${ctx.branch}: ${parts.join(", ")}` : !improved && heat.length && !ctx.always ? `${ctx.branch}: ${INFO_WORDS.heat}` : quiet.startsWith("Main: ") && ctx.branch === "main" ? quiet : `${ctx.branch}: ${quiet}`;
|
|
9833
|
+
const head = `${worse.length ? worse.some((i) => isRed(i, scenarioOf(i))) ? "\uD83D\uDD34" : "\uD83D\uDFE0" : improved ? "\uD83D\uDFE2" : "⚪"} ${status}`;
|
|
10261
9834
|
const journeys = new Set(v.scenarios.map((sc) => `${sc.scenario}|${sc.device?.id}`)).size;
|
|
10262
9835
|
const blocks = [section(`*${head}*
|
|
10263
9836
|
${journeys} journey${journeys === 1 ? "" : "s"} checked`)];
|
|
10264
9837
|
let shown = 0;
|
|
10265
|
-
for (const [title, list] of [[
|
|
9838
|
+
for (const [title, list] of [[STATE_WORDS.worse, errorRows], [STATE_WORDS.warning, warningRows], [STATE_WORDS.better, better]]) {
|
|
10266
9839
|
for (const i of list) {
|
|
10267
9840
|
if (shown >= MAX_ROWS)
|
|
10268
9841
|
break;
|
|
@@ -10273,10 +9846,11 @@ ${mainCard(i)}`));
|
|
|
10273
9846
|
}
|
|
10274
9847
|
}
|
|
10275
9848
|
const more = worse.length + better.length - shown;
|
|
9849
|
+
blocks.push(...heat);
|
|
10276
9850
|
if (commit)
|
|
10277
9851
|
blocks.push(context(commit));
|
|
10278
9852
|
blocks.push(section(`\uD83D\uDC49 <${s.dashboard_url}|Open KeepSnappy>${more > 0 ? ` · ${more} more changes` : ""}`));
|
|
10279
|
-
const color = worse.length ? worse.some((i) => i
|
|
9853
|
+
const color = worse.length ? worse.some((i) => isRed(i, scenarioOf(i))) ? COLOR.error : COLOR.warn : improved ? COLOR.better : COLOR.none;
|
|
10280
9854
|
return { text: head, attachments: [{ color, blocks }] };
|
|
10281
9855
|
}
|
|
10282
9856
|
async function postToSlack(url, payload, fetchImpl = fetch) {
|
|
@@ -10459,11 +10033,12 @@ async function farmOpen(deps, f, rt) {
|
|
|
10459
10033
|
const journeyInfo = await farmJourneys(deps, f, rt, journeys);
|
|
10460
10034
|
const withInstrument = { ...f, ...instrumentFlags(f) };
|
|
10461
10035
|
const setup = Object.fromEntries(SETUP_FLAGS.flatMap(([flag, key]) => withInstrument[flag]?.trim() ? [[key, withInstrument[flag].trim().slice(0, 200)]] : []));
|
|
10036
|
+
const base = await baseCommitFor(deps, rt.env, { explicit: f["base-commit"], branch: git.branch, commit: git.commit }, rt.print);
|
|
10462
10037
|
const rs = await createRunSet(api, {
|
|
10463
10038
|
branch: git.branch,
|
|
10464
10039
|
...git.commit ? { commit_sha: git.commit } : {},
|
|
10465
|
-
...
|
|
10466
|
-
...await commitFields(deps, rt.env, { commit: git.commit, base
|
|
10040
|
+
...base ? { base_commit_sha: base } : {},
|
|
10041
|
+
...await commitFields(deps, rt.env, { commit: git.commit, base }),
|
|
10467
10042
|
device: { model: device.model, ...device.os ? { os: device.os } : {}, provider: device.provider, ...f["device-id"]?.trim() ? { id: f["device-id"].trim() } : {} },
|
|
10468
10043
|
scenarios: journeys,
|
|
10469
10044
|
journeys: journeyInfo,
|
|
@@ -10509,7 +10084,7 @@ async function farmOpen(deps, f, rt) {
|
|
|
10509
10084
|
dashboard_url: rs.dashboard_url,
|
|
10510
10085
|
branch: git.branch,
|
|
10511
10086
|
commit: git.commit ?? null,
|
|
10512
|
-
base_commit:
|
|
10087
|
+
base_commit: base ?? null,
|
|
10513
10088
|
device,
|
|
10514
10089
|
journeys,
|
|
10515
10090
|
repeat,
|
|
@@ -10712,7 +10287,7 @@ async function closeRunSet(deps, f, rt, state, statePath) {
|
|
|
10712
10287
|
await deps.fs.writeFile(statePath, `${JSON.stringify(state2, null, 2)}
|
|
10713
10288
|
`);
|
|
10714
10289
|
await deps.fs.mkdir(out);
|
|
10715
|
-
await deps.fs.writeFile(
|
|
10290
|
+
await deps.fs.writeFile(join12(out, "keepsnappy-result.json"), `${JSON.stringify({ run_set_id: state.run_set_id, dashboard_url: summary.dashboard_url, state: state2.result.state, session: { status: session.status, message: session.message }, summary }, null, 2)}
|
|
10716
10291
|
`);
|
|
10717
10292
|
await githubOutputs(deps, rt, { state: state2.result.state ?? "none", dashboard_url: summary.dashboard_url, run_set_id: state.run_set_id });
|
|
10718
10293
|
const step = rt.env.GITHUB_STEP_SUMMARY;
|
|
@@ -10740,7 +10315,7 @@ ${tail.join(" ")}
|
|
|
10740
10315
|
always: envValue(rt.env, "SLACK_ALWAYS") === "1"
|
|
10741
10316
|
});
|
|
10742
10317
|
if (!msg)
|
|
10743
|
-
rt.print(f.wait ?
|
|
10318
|
+
rt.print(f.wait ? `Slack: nothing to report (${STATE_WORDS["no-change"].toLowerCase()})` : "Slack: skipped, the message needs --wait");
|
|
10744
10319
|
else {
|
|
10745
10320
|
const r = await postToSlack(hook, msg, rt.fetch ?? fetch);
|
|
10746
10321
|
rt.print(r.ok ? "Slack: posted" : `Slack: not posted (${r.error}); the job is not affected`);
|
|
@@ -10749,7 +10324,7 @@ ${tail.join(" ")}
|
|
|
10749
10324
|
if (blocked)
|
|
10750
10325
|
return EXIT.infra;
|
|
10751
10326
|
if (f["fail-on-worse"] && isBlocking(summary)) {
|
|
10752
|
-
rt.print(
|
|
10327
|
+
rt.print(`--fail-on-worse: a journey is ${STATE_WORDS.worse.toLowerCase()} than its baseline and blocks (exit 3)`);
|
|
10753
10328
|
return EXIT.worse;
|
|
10754
10329
|
}
|
|
10755
10330
|
return EXIT.ok;
|
|
@@ -10758,11 +10333,11 @@ ${tail.join(" ")}
|
|
|
10758
10333
|
// apps/agent/src/farm-run.ts
|
|
10759
10334
|
import { existsSync as existsSync2 } from "node:fs";
|
|
10760
10335
|
import { createRequire } from "node:module";
|
|
10761
|
-
import { dirname as dirname4, join as
|
|
10336
|
+
import { dirname as dirname4, join as join15, resolve as resolve6 } from "node:path";
|
|
10762
10337
|
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
10763
10338
|
|
|
10764
10339
|
// apps/agent/src/app-source.ts
|
|
10765
|
-
import { join as
|
|
10340
|
+
import { join as join13 } from "node:path";
|
|
10766
10341
|
var EAS_CLI_VERSION = "24.6.0";
|
|
10767
10342
|
var isEas = (spec) => spec.startsWith("eas:");
|
|
10768
10343
|
|
|
@@ -10798,7 +10373,7 @@ class DefaultAppResolver {
|
|
|
10798
10373
|
const url = build.artifacts?.buildUrl ?? build.artifacts?.applicationArchiveUrl;
|
|
10799
10374
|
if (!url)
|
|
10800
10375
|
throw new Error(`EAS build ${id} has no downloadable artifact (is it finished?)`);
|
|
10801
|
-
const file =
|
|
10376
|
+
const file = join13(o.outDir, `app-${id.slice(0, 8)}${build.platform === "ANDROID" ? ".apk" : ".ipa"}`);
|
|
10802
10377
|
await this.deps.fs.mkdir(o.outDir);
|
|
10803
10378
|
const dl = await this.deps.spawner.run("curl", ["-fsSL", "-o", file, url]);
|
|
10804
10379
|
if (dl.code !== 0)
|
|
@@ -10809,7 +10384,7 @@ class DefaultAppResolver {
|
|
|
10809
10384
|
|
|
10810
10385
|
// apps/agent/src/browserstack.ts
|
|
10811
10386
|
import { randomUUID } from "node:crypto";
|
|
10812
|
-
import { basename as basename2, join as
|
|
10387
|
+
import { basename as basename2, join as join14 } from "node:path";
|
|
10813
10388
|
import { readFile as readFile3 } from "node:fs/promises";
|
|
10814
10389
|
var BASE = "https://api-cloud.browserstack.com/app-automate/maestro/v2";
|
|
10815
10390
|
var TERMINAL = new Set(["passed", "failed", "error", "timedout", "skipped", "stopped"]);
|
|
@@ -10893,7 +10468,7 @@ class BrowserStack {
|
|
|
10893
10468
|
}
|
|
10894
10469
|
async runMaestro(b) {
|
|
10895
10470
|
const appUrl = await this.uploadApp(b.app, b.customId);
|
|
10896
|
-
const work =
|
|
10471
|
+
const work = join14(this.o.tmpDir ?? "/tmp", `keepsnappy-suite-${(this.o.uniqueId ?? randomUUID)()}`);
|
|
10897
10472
|
let suiteUrl;
|
|
10898
10473
|
try {
|
|
10899
10474
|
const zip = await this.zipSuite(b, work);
|
|
@@ -10926,17 +10501,17 @@ class BrowserStack {
|
|
|
10926
10501
|
return this.poll(buildId, b);
|
|
10927
10502
|
}
|
|
10928
10503
|
async zipSuite(b, work) {
|
|
10929
|
-
const zipPath =
|
|
10930
|
-
const stage =
|
|
10504
|
+
const zipPath = join14(work, "suite.zip");
|
|
10505
|
+
const stage = join14(work, "stage");
|
|
10931
10506
|
await this.deps.fs.mkdir(stage);
|
|
10932
10507
|
const cp = await this.deps.spawner.run("cp", ["-R", b.suiteDir.replace(/\/$/, ""), stage]);
|
|
10933
10508
|
if (cp.code !== 0)
|
|
10934
10509
|
throw new FarmApiError(`cannot copy the suite ${b.suiteDir}: ${cp.stderr || cp.error}`);
|
|
10935
10510
|
const folder = basename2(b.suiteDir.replace(/\/$/, ""));
|
|
10936
|
-
if (!await this.deps.fs.exists(
|
|
10511
|
+
if (!await this.deps.fs.exists(join14(stage, folder, "config.yaml")))
|
|
10937
10512
|
throw new FarmApiError(`${b.suiteDir} has no config.yaml: is that the Maestro workspace?`);
|
|
10938
|
-
const target =
|
|
10939
|
-
await this.deps.fs.mkdir(
|
|
10513
|
+
const target = join14(stage, folder, b.tokensPath);
|
|
10514
|
+
await this.deps.fs.mkdir(join14(target, ".."));
|
|
10940
10515
|
await this.deps.fs.writeFile(target, await this.deps.fs.readFile(b.tokensJs));
|
|
10941
10516
|
const include = SUITE_EXTENSIONS.flatMap((e) => ["-i", `${folder}/*${e}`, "-i", `${folder}/**/*${e}`]);
|
|
10942
10517
|
const z = await this.deps.spawner.run("sh", ["-c", 'cd "$1" && shift && zip -qr "$@"', "sh", stage, zipPath, folder, ...include, "-x", `${folder}/runs/*`, "-x", "*/node_modules/*"]);
|
|
@@ -10995,9 +10570,9 @@ class BrowserStackLocal {
|
|
|
10995
10570
|
const url = TUNNEL_ZIP[this.o.platform ?? process.platform];
|
|
10996
10571
|
if (!url)
|
|
10997
10572
|
throw new FarmApiError(`no BrowserStackLocal binary known for ${this.o.platform ?? process.platform}`);
|
|
10998
|
-
const dir =
|
|
10999
|
-
const bin =
|
|
11000
|
-
const zip =
|
|
10573
|
+
const dir = join14(this.o.dir, identifier);
|
|
10574
|
+
const bin = join14(dir, "BrowserStackLocal");
|
|
10575
|
+
const zip = join14(dir, "bs-local.zip");
|
|
11001
10576
|
await this.deps.fs.mkdir(dir);
|
|
11002
10577
|
for (const [cmd, args] of [
|
|
11003
10578
|
["curl", ["-fsSL", "-o", zip, url]],
|
|
@@ -11031,12 +10606,12 @@ ${r.stderr}`;
|
|
|
11031
10606
|
// apps/agent/src/farm-run.ts
|
|
11032
10607
|
function appiumDriverEntry(o = {}) {
|
|
11033
10608
|
const exists = o.exists ?? existsSync2;
|
|
11034
|
-
const bundled =
|
|
10609
|
+
const bundled = join15(o.selfDir ?? dirname4(fileURLToPath2(import.meta.url)), "appium-driver.js");
|
|
11035
10610
|
if (exists(bundled))
|
|
11036
10611
|
return bundled;
|
|
11037
10612
|
const pkgDir = dirname4(o.packageJson ? o.packageJson() : createRequire(import.meta.url).resolve("@keepsnappy/appium-driver/package.json"));
|
|
11038
|
-
const built =
|
|
11039
|
-
return exists(built) ? built :
|
|
10613
|
+
const built = join15(pkgDir, "dist", "src", "cli.js");
|
|
10614
|
+
return exists(built) ? built : join15(pkgDir, "src", "cli.ts");
|
|
11040
10615
|
}
|
|
11041
10616
|
var driverRuntime = (entry) => entry.endsWith(".ts") ? process.execPath : "node";
|
|
11042
10617
|
function nodeFarmRunPorts(deps, rt) {
|
|
@@ -11045,7 +10620,7 @@ function nodeFarmRunPorts(deps, rt) {
|
|
|
11045
10620
|
if (!user || !key)
|
|
11046
10621
|
throw new ConfigError(["BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY must be set"]);
|
|
11047
10622
|
const tmp = rt.env.RUNNER_TEMP ?? rt.env.TMPDIR ?? "/tmp";
|
|
11048
|
-
const dir =
|
|
10623
|
+
const dir = join15(tmp, "keepsnappy-bs-local");
|
|
11049
10624
|
return {
|
|
11050
10625
|
apps: new DefaultAppResolver(deps, rt.env),
|
|
11051
10626
|
provider: new BrowserStack(deps, { credentials: { user, key }, tmpDir: tmp, ...rt.fetch ? { fetch: rt.fetch } : {}, ...rt.sleep ? { sleep: rt.sleep } : {}, ...rt.now ? { now: rt.now } : {} }),
|
|
@@ -11158,7 +10733,7 @@ function printPlan(p, f, rt) {
|
|
|
11158
10733
|
line("keepsnappy farm run: dry run, no request is made and nothing is written");
|
|
11159
10734
|
line(` 1. validate: ${p.journeys.join(", ")} x ${p.repeat} on ${p.device} (${p.platform}), driver ${p.driver}`);
|
|
11160
10735
|
line(` 2. resolve the app: ${p.app.startsWith("eas:") ? `eas-cli build:view ${p.app.slice(4)}, download into ${p.out}` : p.app.startsWith("bs://") ? "bs:// passes through" : `${p.app} (local file)`}`);
|
|
11161
|
-
line(` 3. farm open -> ${
|
|
10736
|
+
line(` 3. farm open -> ${join15(p.out, "keepsnappy-tokens.js")}, ${join15(p.out, "keepsnappy-farm.json")} (POST /v1/run-sets, then POST /v1/runs x ${p.journeys.length * p.repeat})`);
|
|
11162
10737
|
line(` 4. tunnel: ${p.ports.length ? `BrowserStackLocal for bs-local.com ports ${p.ports.join(", ")}, identifier = run id` : "none (no --tunnel-for)"}`);
|
|
11163
10738
|
if (p.driver === "maestro") {
|
|
11164
10739
|
line(` 5. maestro on BrowserStack: upload app, zip ${f["suite-dir"]} with the tokens at ${f["tokens-path"] ?? "flows/keepsnappy/tokens.js"}, upload suite, POST build (execute ${f.execute}), poll`);
|
|
@@ -11167,10 +10742,11 @@ function printPlan(p, f, rt) {
|
|
|
11167
10742
|
}
|
|
11168
10743
|
for (const [k, v] of p.env)
|
|
11169
10744
|
line(` env ${k}=${substitute(v, {})}`);
|
|
11170
|
-
line(` 6. farm close${f.wait ? " --wait" : ""}${f["fail-on-worse"] ? " --fail-on-worse" : ""} with ${
|
|
10745
|
+
line(` 6. farm close${f.wait ? " --wait" : ""}${f["fail-on-worse"] ? " --fail-on-worse" : ""} with ${join15(p.out, `browserstack.${p.platform}.json`)}`);
|
|
11171
10746
|
line(" 7. always: stop the tunnel, put the empty stub back over the tokens file");
|
|
11172
10747
|
line(" exit: 0 any verdict, 1 infrastructure / driver failed / Blocked, 2 configuration, 3 --fail-on-worse");
|
|
11173
10748
|
}
|
|
10749
|
+
var ANNOTATION = /^::(error|warning|notice)( [^:]*)?::/;
|
|
11174
10750
|
async function runDriver2(deps, rt, f, p, ports, ctx) {
|
|
11175
10751
|
const env = Object.fromEntries(p.env.map(([k, v]) => [k, substitute(v, ctx)]));
|
|
11176
10752
|
if (p.driver === "maestro") {
|
|
@@ -11190,13 +10766,13 @@ async function runDriver2(deps, rt, f, p, ports, ctx) {
|
|
|
11190
10766
|
timeoutMinutes: p.timeoutMinutes,
|
|
11191
10767
|
pollSeconds: p.pollSeconds,
|
|
11192
10768
|
onStarted: async (buildId) => {
|
|
11193
|
-
await deps.fs.writeFile(
|
|
10769
|
+
await deps.fs.writeFile(join15(ctx.out, `browserstack.${p.platform}.json`), `${JSON.stringify({ buildId, dashboard: `https://app-automate.browserstack.com/dashboard/v2/builds/${buildId}`, note: "written at submit; replaced by the full detail once the build ends", status: "unknown" }, null, 2)}
|
|
11194
10770
|
`);
|
|
11195
10771
|
rt.print(`build ${buildId} started: https://app-automate.browserstack.com/dashboard/v2/builds/${buildId}`);
|
|
11196
10772
|
},
|
|
11197
10773
|
onStatus: (buildId, status) => rt.print(`build ${buildId}: ${status}`)
|
|
11198
10774
|
});
|
|
11199
|
-
await deps.fs.writeFile(
|
|
10775
|
+
await deps.fs.writeFile(join15(ctx.out, `browserstack.${p.platform}.json`), `${redact(JSON.stringify(detail.detail, null, 2), secretsOf(p, f, rt.env))}
|
|
11200
10776
|
`);
|
|
11201
10777
|
return detail.status === "passed" ? { ok: true, message: null } : { ok: false, message: `the device build ${detail.buildId} ended ${detail.status}` };
|
|
11202
10778
|
}
|
|
@@ -11229,7 +10805,7 @@ async function runDriver2(deps, rt, f, p, ports, ctx) {
|
|
|
11229
10805
|
const secrets = secretsOf(p, f, rt.env);
|
|
11230
10806
|
let streamed = false;
|
|
11231
10807
|
let pending = "";
|
|
11232
|
-
const emit = (l) => l.trim() && rt.print(` driver: ${redact(l, secrets)}`);
|
|
10808
|
+
const emit = (l) => l.trim() && rt.print(ANNOTATION.test(l) ? redact(l, secrets) : ` driver: ${redact(l, secrets)}`);
|
|
11233
10809
|
const r = await deps.spawner.run(f["driver-script"] ? "node" : driverRuntime(entry), args, {
|
|
11234
10810
|
env,
|
|
11235
10811
|
timeoutMs: p.timeoutMinutes * 60000,
|
|
@@ -11251,7 +10827,7 @@ async function runDriver2(deps, rt, f, p, ports, ctx) {
|
|
|
11251
10827
|
}
|
|
11252
10828
|
async function farmRun(deps, f, rt, ports) {
|
|
11253
10829
|
const p = planFarmRun(f);
|
|
11254
|
-
const statePath =
|
|
10830
|
+
const statePath = join15(p.out, "keepsnappy-farm.json");
|
|
11255
10831
|
if (await deps.fs.exists(statePath))
|
|
11256
10832
|
throw new ConfigError([`${statePath} exists: a run set was opened here before; use another --out or remove ${p.out} (close it first with keepsnappy farm close)`]);
|
|
11257
10833
|
if (f["dry-run"]) {
|
|
@@ -11259,7 +10835,7 @@ async function farmRun(deps, f, rt, ports) {
|
|
|
11259
10835
|
return EXIT.ok;
|
|
11260
10836
|
}
|
|
11261
10837
|
const farm = ports ?? nodeFarmRunPorts(deps, rt);
|
|
11262
|
-
const tokensJs =
|
|
10838
|
+
const tokensJs = join15(p.out, "keepsnappy-tokens.js");
|
|
11263
10839
|
await deps.fs.mkdir(p.out);
|
|
11264
10840
|
const resolved = await farm.apps.resolve(p.app, { outDir: p.out });
|
|
11265
10841
|
const commit = f.commit ?? resolved.commit ?? undefined;
|
|
@@ -11290,9 +10866,9 @@ async function farmRun(deps, f, rt, ports) {
|
|
|
11290
10866
|
const runId = f["run-id"] ?? `keepsnappy-${p.platform}-${state.run_set_id.slice(0, 8)}`;
|
|
11291
10867
|
const ctx = { runId, out: p.out, tokensJs, runSetId: state.run_set_id, app: resolved.app };
|
|
11292
10868
|
try {
|
|
11293
|
-
await deps.spawner.run("rm", ["-f",
|
|
10869
|
+
await deps.spawner.run("rm", ["-f", join15(p.out, `browserstack.${p.platform}.json`)]);
|
|
11294
10870
|
if (p.ports.length) {
|
|
11295
|
-
tunnel = await farm.tunnel.open({ identifier: runId, ports: p.ports, logFile:
|
|
10871
|
+
tunnel = await farm.tunnel.open({ identifier: runId, ports: p.ports, logFile: join15(p.out, "browserstack-local.log") });
|
|
11296
10872
|
rt.print(`tunnel ${runId}: bs-local.com ports ${p.ports.join(", ")}`);
|
|
11297
10873
|
}
|
|
11298
10874
|
driver = await runDriver2(deps, rt, f, p, farm, { ...ctx, tunnelId: tunnel?.identifier ?? null });
|
|
@@ -11303,7 +10879,7 @@ async function farmRun(deps, f, rt, ports) {
|
|
|
11303
10879
|
rt.print(`device run failed: ${driver.message}; closing the run set with what arrived`);
|
|
11304
10880
|
closeCode = await farmClose(deps, {
|
|
11305
10881
|
state: statePath,
|
|
11306
|
-
"bs-build":
|
|
10882
|
+
"bs-build": join15(p.out, `browserstack.${p.platform}.json`),
|
|
11307
10883
|
out: p.out,
|
|
11308
10884
|
...f.wait ? { wait: true } : {},
|
|
11309
10885
|
...f["fail-on-worse"] ? { "fail-on-worse": true } : {},
|
|
@@ -11482,10 +11058,10 @@ function formatFinding(f, opts = {}) {
|
|
|
11482
11058
|
line("Layer", `${f.layer.id} · ${f.layer.name} (${f.certificate ? "certified" : "uncertified"})${f.limits.length ? `; limits: ${f.limits.join("; ")}` : ""}`),
|
|
11483
11059
|
line("Id", f.id)
|
|
11484
11060
|
];
|
|
11485
|
-
if (f.why.components.length
|
|
11061
|
+
if (f.why.components.length) {
|
|
11486
11062
|
out.push(line("Components", ""));
|
|
11487
11063
|
for (const c of f.why.components) {
|
|
11488
|
-
out.push(` ${c.component}: ${c.self_ms_before} → ${c.self_ms_after} ms own time, renders ${c.renders_before} → ${c.renders_after}${c.cause ? `; ${c.cause}` : ""}${c.flagged ? "" : " (not certain)"}`);
|
|
11064
|
+
out.push(` ${c.component}: ${c.self_ms_before ?? "–"} → ${c.self_ms_after} ms own time, renders ${c.renders_before ?? "–"} → ${c.renders_after ?? "–"}${c.cause ? `; ${c.cause}` : ""}${c.flagged ? "" : " (not certain)"}`);
|
|
11489
11065
|
}
|
|
11490
11066
|
}
|
|
11491
11067
|
if (opts.prompt)
|
|
@@ -11523,18 +11099,16 @@ function measuredLine(l) {
|
|
|
11523
11099
|
function countsLine2(l) {
|
|
11524
11100
|
const c = l.counts;
|
|
11525
11101
|
const parts = [
|
|
11526
|
-
c.worse ? `${c.worse} worse${c.not_blocking ? ` (${c.not_blocking} not blocking)` : ""}` : "",
|
|
11527
|
-
c.warn ?
|
|
11528
|
-
c.better ? `${c.better} better` : ""
|
|
11102
|
+
c.worse ? `${c.worse} ${STATE_WORDS.worse.toLowerCase()}${c.not_blocking ? ` (${c.not_blocking} not blocking)` : ""}` : "",
|
|
11103
|
+
c.warn ? `${c.warn} ${STATE_WORDS.warning.toLowerCase()}` : "",
|
|
11104
|
+
c.better ? `${c.better} ${STATE_WORDS.better.toLowerCase()}` : ""
|
|
11529
11105
|
].filter(Boolean);
|
|
11530
11106
|
const state = l.filters.state;
|
|
11531
11107
|
const unjudged = l.measured.lanes.filter((x) => !x.judged);
|
|
11532
|
-
const head = parts.length ? `${parts.join(", ")}${state === "all" ? "" : `, ${STATE_WORDS2[state] ?? state}`}` : state !== "present" && state !== "all" ? `No finding ${STATE_WORDS2[state] ?? state}` : unjudged.length === l.measured.lanes.length ? `No finding, but nothing was compared yet (${[...new Set(unjudged.map((x) => x.word))].join(", ")}): measure again before calling it clean` :
|
|
11108
|
+
const head = parts.length ? `${parts.join(", ")}${state === "all" ? "" : `, ${STATE_WORDS2[state] ?? state}`}` : state !== "present" && state !== "all" ? `No finding ${STATE_WORDS2[state] ?? state}` : unjudged.length === l.measured.lanes.length ? `No finding, but nothing was compared yet (${[...new Set(unjudged.map((x) => x.word))].join(", ")}): measure again before calling it clean` : `Nothing ${STATE_WORDS.worse.toLowerCase()}, nothing ${STATE_WORDS.warning.toLowerCase()}, nothing ${STATE_WORDS.better.toLowerCase()}`;
|
|
11533
11109
|
const heated = c.heat_runs ?? 0;
|
|
11534
|
-
const never = c.unsettled ?? 0;
|
|
11535
11110
|
const more = [
|
|
11536
|
-
|
|
11537
|
-
heated ? `${plural(heated, "run")} heated up the phone (run health, layer 3)` : "",
|
|
11111
|
+
heated ? `${INFO_WORDS.heat.toLowerCase()} in ${plural(heated, "run")} (run health, layer 3)` : "",
|
|
11538
11112
|
state === "present" && l.states.not_judged ? `${l.states.not_judged} more not judged in the newest run set (state not-judged)` : ""
|
|
11539
11113
|
].filter(Boolean);
|
|
11540
11114
|
const line = `${head}${more.length ? `; ${more.join("; ")}` : ""}.`;
|
|
@@ -11638,7 +11212,7 @@ async function get(t, path, query = {}) {
|
|
|
11638
11212
|
}
|
|
11639
11213
|
var listFindings = (t, f = {}) => get(t, "v1/findings", { branch: f.branch, main: f.main ? "true" : undefined, journey: f.journey, phone: f.phone, state: f.state, limit: f.limit, offset: f.offset || undefined });
|
|
11640
11214
|
var getFinding = (t, id) => get(t, `v1/findings/${encodeURIComponent(id)}`);
|
|
11641
|
-
var
|
|
11215
|
+
var compareRuns = (t, a, b) => get(t, "v1/compare", { a, b });
|
|
11642
11216
|
var windowHistory = (t, q) => get(t, "v1/history", q);
|
|
11643
11217
|
var commitsBetween = (t, q) => get(t, "v1/commits", q);
|
|
11644
11218
|
var layers = (t) => get(t, "v1/layers");
|
|
@@ -11647,7 +11221,7 @@ var layers = (t) => get(t, "v1/layers");
|
|
|
11647
11221
|
var FINDINGS_USAGE = `keepsnappy findings [--branch NAME | --main] [--journey KEY] [--phone MODEL] [--state present|not-judged|resolved|all] [--limit N] [--offset N] [--json]
|
|
11648
11222
|
finding <id> [--prompt] [--json] (--prompt: only the hand-off for a coding agent; with --json the whole finding)
|
|
11649
11223
|
compare <runA> <runB> [--json] (run ids or external ids; one run against one, no statistics)
|
|
11650
|
-
history <journey> <window> [--phone MODEL] [--json] (window id or its name, e.g. "Tap
|
|
11224
|
+
history <journey> <window> [--phone MODEL] [--json] (window id or its name, e.g. "Tap · “live chat send”")
|
|
11651
11225
|
commits <from> [<to>] [--branch NAME] [--json] (measured commits between two run sets or commit hashes)
|
|
11652
11226
|
layers [--json]
|
|
11653
11227
|
(key: KEEPSNAPPY_KEY or --key-env NAME; API: --api URL or KEEPSNAPPY_API)`;
|
|
@@ -11734,7 +11308,7 @@ async function findingsCommand(cmd, argv, rt) {
|
|
|
11734
11308
|
}
|
|
11735
11309
|
case "compare":
|
|
11736
11310
|
need(2, "<runA> <runB>");
|
|
11737
|
-
return out(await
|
|
11311
|
+
return out(await compareRuns(t, p[0], p[1]), formatCompare);
|
|
11738
11312
|
case "history":
|
|
11739
11313
|
need(2, "<journey> <window>");
|
|
11740
11314
|
return out(await windowHistory(t, { journey: p[0], window: p.slice(1).join(" "), ...o.phone ? { phone: o.phone } : {}, ...limit ? { limit } : {} }), formatHistory);
|
|
@@ -11750,7 +11324,7 @@ async function findingsCommand(cmd, argv, rt) {
|
|
|
11750
11324
|
|
|
11751
11325
|
// apps/agent/src/cli.ts
|
|
11752
11326
|
var USAGE = `keepsnappy <command>
|
|
11753
|
-
run [--config keepsnappy.yml] [--
|
|
11327
|
+
run [--config keepsnappy.yml] [--repeat N] [--out DIR]
|
|
11754
11328
|
[--upload [--wait] [--fail-on-worse] [--ingest-url URL] [--commit SHA] [--branch NAME] [--base-commit SHA]]
|
|
11755
11329
|
[--device-model NAME --device-os VERSION] [--scenario NAME] [--replay <out dir>[:ARM] ...]
|
|
11756
11330
|
[--no-markers] [-- <e2e command...>]
|
|
@@ -11771,8 +11345,6 @@ var USAGE = `keepsnappy <command>
|
|
|
11771
11345
|
appium: --journey-module FILE | --driver-script FILE [--driver-arg ARG ...]
|
|
11772
11346
|
[--branch NAME --commit SHA] [--wait [--fail-on-worse]] [--out DIR] [--dry-run] (open + tunnel + device + close; docs/plans/54a)
|
|
11773
11347
|
extract <out-dir> --ri-capture observations.ndjson [--mem-window NAME]
|
|
11774
|
-
verdict <out-dir> [--baseline A --candidate B]
|
|
11775
|
-
verdict <out-dir> --baseline remote [--config keepsnappy.yml] [--branch main] [--limit N] [--scenario NAME]
|
|
11776
11348
|
aa <out-dir> [--reps N] [--boot N]
|
|
11777
11349
|
doctor
|
|
11778
11350
|
${SOURCEMAPS_USAGE.replace("keepsnappy ", "")}
|
|
@@ -11796,6 +11368,11 @@ async function main2(argv) {
|
|
|
11796
11368
|
console.log(USAGE);
|
|
11797
11369
|
return 0;
|
|
11798
11370
|
}
|
|
11371
|
+
if (cmd === "verdict")
|
|
11372
|
+
return verdictCommand();
|
|
11373
|
+
if (cmd === "run" && rest.slice(0, rest.indexOf("--") < 0 ? rest.length : rest.indexOf("--")).some((a) => a === "--baseline" || a.startsWith("--baseline="))) {
|
|
11374
|
+
throw new ConfigError([`--baseline: ${LOCAL_VERDICT_REMOVED}`]);
|
|
11375
|
+
}
|
|
11799
11376
|
if (cmd === "sourcemaps")
|
|
11800
11377
|
return sourcemapsCommand(rest, { env: process.env, print: console.log });
|
|
11801
11378
|
if (cmd && FINDINGS_COMMANDS.has(cmd))
|
|
@@ -11815,8 +11392,6 @@ async function main2(argv) {
|
|
|
11815
11392
|
out: { type: "string" },
|
|
11816
11393
|
"bundle-id": { type: "string" },
|
|
11817
11394
|
timeout: { type: "string" },
|
|
11818
|
-
baseline: { type: "string" },
|
|
11819
|
-
candidate: { type: "string" },
|
|
11820
11395
|
reps: { type: "string" },
|
|
11821
11396
|
boot: { type: "string" },
|
|
11822
11397
|
"ri-capture": { type: "string" },
|
|
@@ -11836,7 +11411,6 @@ async function main2(argv) {
|
|
|
11836
11411
|
"device-os": { type: "string" },
|
|
11837
11412
|
"device-id": { type: "string" },
|
|
11838
11413
|
replay: { type: "string", multiple: true },
|
|
11839
|
-
limit: { type: "string" },
|
|
11840
11414
|
api: { type: "string" },
|
|
11841
11415
|
"key-env": { type: "string" },
|
|
11842
11416
|
"device-provider": { type: "string" },
|
|
@@ -11937,37 +11511,12 @@ async function main2(argv) {
|
|
|
11937
11511
|
console.log(`${rows.length} rows -> ${resolve7(dir, "metrics.csv")}`);
|
|
11938
11512
|
return 0;
|
|
11939
11513
|
}
|
|
11940
|
-
if (cmd === "
|
|
11514
|
+
if (cmd === "aa") {
|
|
11941
11515
|
const dir = positionals[0];
|
|
11942
11516
|
if (!dir)
|
|
11943
|
-
throw new Error(
|
|
11944
|
-
|
|
11945
|
-
|
|
11946
|
-
const text = await readProjectConfig((p) => readFile5(p, "utf8"), o.config).catch(() => {
|
|
11947
|
-
throw new ConfigError([`cannot read ${configPath}: --baseline remote takes the API URL and key from keepsnappy.yml (use --config <file>)`]);
|
|
11948
|
-
});
|
|
11949
|
-
const config = parseConfig(text, { commandGiven: true });
|
|
11950
|
-
const url = o["ingest-url"] ?? config.project.ingestUrl ?? envValue(process.env, "API") ?? DEFAULT_API_URL;
|
|
11951
|
-
const key = projectKey(process.env, config.project.keyEnv);
|
|
11952
|
-
if (!url)
|
|
11953
|
-
throw new ConfigError(["--baseline remote needs project.ingest_url in keepsnappy.yml (or --ingest-url)"]);
|
|
11954
|
-
if (!key)
|
|
11955
|
-
throw new ConfigError([`--baseline remote needs the project key in the environment variable ${config.project.keyEnv}`]);
|
|
11956
|
-
const limit = o.limit ? Number(o.limit) : undefined;
|
|
11957
|
-
if (limit !== undefined && (!Number.isInteger(limit) || limit < 1))
|
|
11958
|
-
throw new ConfigError(["--limit must be a whole number >= 1"]);
|
|
11959
|
-
const out = await remoteVerdict(deps, resolve7(dir), { url, key }, {
|
|
11960
|
-
branch: o.branch ?? config.baselineBranch,
|
|
11961
|
-
...limit ? { limit } : {},
|
|
11962
|
-
...o.scenario ? { scenario: o.scenario } : {}
|
|
11963
|
-
});
|
|
11964
|
-
console.log(out.table);
|
|
11965
|
-
} else if (cmd === "verdict") {
|
|
11966
|
-
console.log((await verdict(deps, resolve7(dir), { ...o.baseline ? { baseline: o.baseline } : {}, ...o.candidate ? { candidate: o.candidate } : {} })).table);
|
|
11967
|
-
} else {
|
|
11968
|
-
const config = { ...o.reps ? { reps: Number(o.reps) } : {}, ...o.boot ? { nBoot: Number(o.boot) } : {} };
|
|
11969
|
-
console.log((await aa(deps, resolve7(dir), { config })).table);
|
|
11970
|
-
}
|
|
11517
|
+
throw new Error("aa needs <out-dir>");
|
|
11518
|
+
const config = { ...o.reps ? { reps: Number(o.reps) } : {}, ...o.boot ? { nBoot: Number(o.boot) } : {} };
|
|
11519
|
+
console.log((await aa(deps, resolve7(dir), { config })).table);
|
|
11971
11520
|
return 0;
|
|
11972
11521
|
}
|
|
11973
11522
|
console.error(USAGE);
|