@codyswann/lisa 3.70.5 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/all/copy-overwrite/scripts/lisa-gates.mjs +6 -8
- package/dist/cli/ui-ci-quality-jobs-compute.d.ts +0 -1
- package/dist/cli/ui-ci-quality-jobs-compute.d.ts.map +1 -1
- package/dist/cli/ui-ci-quality-jobs-compute.js +0 -9
- package/dist/cli/ui-ci-quality-jobs-compute.js.map +1 -1
- package/dist/cli/ui-ci-quality-jobs-parse.d.ts.map +1 -1
- package/dist/cli/ui-ci-quality-jobs-parse.js +0 -1
- package/dist/cli/ui-ci-quality-jobs-parse.js.map +1 -1
- package/dist/core/lisa-owned-hash-ledger.d.ts.map +1 -1
- package/dist/core/lisa-owned-hash-ledger.js +3 -0
- package/dist/core/lisa-owned-hash-ledger.js.map +1 -1
- package/dist/core/upstream-evidence-manifest.d.ts.map +1 -1
- package/dist/core/upstream-evidence-manifest.js +8 -7
- package/dist/core/upstream-evidence-manifest.js.map +1 -1
- package/expo/create-only/.github/required-checks.json +0 -7
- package/expo/create-only/.github/workflows/ci.yml +1 -1
- package/nestjs/create-only/.github/required-checks.json +0 -7
- package/nestjs/create-only/.github/workflows/ci.yml +1 -1
- package/package.json +2 -2
- package/plugins/lisa/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-agy/plugin.json +1 -1
- package/plugins/lisa-cdk/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-cdk/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-cdk-agy/plugin.json +1 -1
- package/plugins/lisa-cdk-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-cdk-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-expo/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-expo/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-expo-agy/plugin.json +1 -1
- package/plugins/lisa-expo-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-expo-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-harper-fabric/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-harper-fabric/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-harper-fabric-agy/plugin.json +1 -1
- package/plugins/lisa-harper-fabric-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-harper-fabric-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-nestjs/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-nestjs/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-nestjs-agy/plugin.json +1 -1
- package/plugins/lisa-nestjs-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-nestjs-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-openclaw/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-openclaw/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-openclaw-agy/plugin.json +1 -1
- package/plugins/lisa-openclaw-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-openclaw-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-phaser/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-phaser/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-phaser-agy/plugin.json +1 -1
- package/plugins/lisa-phaser-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-phaser-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-rails/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-rails/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-rails-agy/plugin.json +1 -1
- package/plugins/lisa-rails-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-rails-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-typescript/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-typescript/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-typescript-agy/plugin.json +1 -1
- package/plugins/lisa-typescript-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-typescript-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-wiki/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-wiki/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-wiki-agy/plugin.json +1 -1
- package/plugins/lisa-wiki-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-wiki-cursor/.claude-plugin/plugin.json +1 -1
- package/typescript/copy-overwrite/scripts/lisa-mutation.mjs +457 -17
- package/ui/index.html +0 -33
|
@@ -67,13 +67,40 @@
|
|
|
67
67
|
* config declares none — a budget nobody chose is unactionable until somebody
|
|
68
68
|
* is told that is what it was.
|
|
69
69
|
*
|
|
70
|
+
* ## A timeout is not a kill either
|
|
71
|
+
*
|
|
72
|
+
* The other half of the same problem, and the one that survives a run
|
|
73
|
+
* COMPLETING. **Stryker scores a timed-out mutant as KILLED**, so every score it
|
|
74
|
+
* reports credits a bucket nothing demonstrably caught, and which mutants land
|
|
75
|
+
* in that bucket depends on how busy the machine was. The consequence is
|
|
76
|
+
* perverse in a way worth stating plainly: **a slower box yields a better
|
|
77
|
+
* score.** A genuine regression can hide there too — a mutation that makes the
|
|
78
|
+
* covering tests hang is scored identically to one they catch.
|
|
79
|
+
*
|
|
80
|
+
* One whole-list run measured 117 timeouts against 3,455 detected: 3.39% of
|
|
81
|
+
* everything counted as detection, worth up to 2.00 score points.
|
|
82
|
+
*
|
|
83
|
+
* So every completed run now prints the count, the score as reported, and the
|
|
84
|
+
* score recomputed with timeouts NOT credited — and the recomputed one is
|
|
85
|
+
* judged against `thresholds.break`. That can only ever tighten: it is applied
|
|
86
|
+
* on top of Stryker's own verdict, and nothing here can turn a red run green.
|
|
87
|
+
* Raising `timeoutMS` is NOT the fix and is refused as one — it converts a
|
|
88
|
+
* timeout into a slow pass and hides the identical gap.
|
|
89
|
+
*
|
|
70
90
|
* ## Configuration
|
|
71
91
|
*
|
|
72
92
|
* `mutation.gate.json` (project-owned / create-only):
|
|
73
93
|
* `{ "enabled": false, "since": "main" }`.
|
|
74
94
|
* Overridable via env: `MUTATION_ENABLED=true|false`, `MUTATION_SINCE=<ref>`.
|
|
75
95
|
* `MUTATION_CAPTURE=0` turns the output capture off, trading the diagnosis
|
|
76
|
-
* above for Stryker's TTY progress bar
|
|
96
|
+
* above for Stryker's TTY progress bar — and with it the timeout accounting,
|
|
97
|
+
* which is read from the same transcript.
|
|
98
|
+
* `MUTATION_TIMEOUT_SHARE_MAX=<percent>` moves the share ceiling for one run.
|
|
99
|
+
*
|
|
100
|
+
* `--all` mutates the project's whole `mutate` list instead of a diff. It is
|
|
101
|
+
* the same gate, so the accounting above applies to it — which is the point:
|
|
102
|
+
* the whole-list run is where the timeout bucket is large enough to matter, and
|
|
103
|
+
* it used to be `stryker run` invoked directly, outside this script entirely.
|
|
77
104
|
* @module scripts/lisa-mutation
|
|
78
105
|
*/
|
|
79
106
|
import { execFileSync, spawnSync } from "node:child_process";
|
|
@@ -100,9 +127,14 @@ export const OUTCOMES = Object.freeze({
|
|
|
100
127
|
inertConfig: "mutation-gate: inert-mutate-config",
|
|
101
128
|
unrepresentablePath: "mutation-gate: unrepresentable-path",
|
|
102
129
|
scoped: "mutation-gate: scoped-run",
|
|
130
|
+
wholeList: "mutation-gate: whole-list-run",
|
|
103
131
|
dryRunTimeout: "mutation-gate: dry-run-timeout",
|
|
104
132
|
scoreBelowBreak: "mutation-gate: score-below-break",
|
|
105
133
|
runFailed: "mutation-gate: run-failed",
|
|
134
|
+
timeoutAccounting: "mutation-gate: timeout-accounting",
|
|
135
|
+
timeoutUnmeasured: "mutation-gate: timeout-share-unmeasured",
|
|
136
|
+
timeoutShareExceeded: "mutation-gate: timeout-share-exceeded",
|
|
137
|
+
inflatedByTimeouts: "mutation-gate: score-below-break-without-timeouts",
|
|
106
138
|
});
|
|
107
139
|
|
|
108
140
|
/**
|
|
@@ -387,6 +419,344 @@ const BREAK_THRESHOLD_PATTERN =
|
|
|
387
419
|
*/
|
|
388
420
|
const TIMED_OUT_MUTANTS_PATTERN = /\(\d+ survived, (\d+) timed out\)/gu;
|
|
389
421
|
|
|
422
|
+
/**
|
|
423
|
+
* One escape character, kept out of the pattern literals below.
|
|
424
|
+
*
|
|
425
|
+
* A regular-expression literal containing a control character is refused by the
|
|
426
|
+
* shipped ruleset (`no-control-regex`) and it is right to: an unexplained
|
|
427
|
+
* control byte in a pattern is nearly always a mistake. This one is not — the
|
|
428
|
+
* clear-text reporter colours its score cells with chalk, and `FORCE_COLOR` in
|
|
429
|
+
* CI turns that on even under a pipe — so the escape is named once, here, and
|
|
430
|
+
* the pattern is assembled from it.
|
|
431
|
+
* @type {string}
|
|
432
|
+
*/
|
|
433
|
+
const ESCAPE = String.fromCharCode(27);
|
|
434
|
+
|
|
435
|
+
/** Chalk's SGR sequences, so a coloured table row can still be read. */
|
|
436
|
+
const ANSI_PATTERN = new RegExp(`${ESCAPE}\\[[0-9;]*m`, "gu");
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* The clear-text reporter's `All files` row.
|
|
440
|
+
*
|
|
441
|
+
* ```
|
|
442
|
+
* All files | 59.03 | 77.09 | 3338 | 117 | 1027 | 1371 | 191 |
|
|
443
|
+
* ```
|
|
444
|
+
*
|
|
445
|
+
* Seven cells after the name: total score, covered score, then the five counts.
|
|
446
|
+
* Only the counts are read — the two scores are RECOMPUTED here rather than
|
|
447
|
+
* taken, because the whole point is that the printed one credits a bucket
|
|
448
|
+
* nothing demonstrably caught.
|
|
449
|
+
*
|
|
450
|
+
* The row is absent from a run whose reporters do not include `clear-text`, and
|
|
451
|
+
* from one where `skipFull` is set and every file scored 100. Both are reported
|
|
452
|
+
* as "not measured" rather than guessed at; see {@link parseMutantTally}.
|
|
453
|
+
*
|
|
454
|
+
* It is found by a prefix test over already-split lines rather than by an
|
|
455
|
+
* anchored pattern over the whole transcript. That is the shipped ruleset's
|
|
456
|
+
* rule and it is the right one here twice over: this is a parser reading a
|
|
457
|
+
* multi-megabyte transcript inside a git hook, so a pattern that can backtrack
|
|
458
|
+
* is a hazard, and a single expression covering seven cells was over the
|
|
459
|
+
* complexity ceiling anyway.
|
|
460
|
+
* @type {string}
|
|
461
|
+
*/
|
|
462
|
+
const ALL_FILES_ROW_PREFIX = "All files";
|
|
463
|
+
|
|
464
|
+
/** One count cell, which is a bare non-negative integer or nothing useful. */
|
|
465
|
+
const COUNT_CELL_PATTERN = /^\d+$/u;
|
|
466
|
+
|
|
467
|
+
/**
|
|
468
|
+
* Where each count sits once the row is split on its separators.
|
|
469
|
+
*
|
|
470
|
+
* `name | total | covered | killed | timeout | survived | no cov | errors |`,
|
|
471
|
+
* so the counts start at index 3. Named rather than inlined because reading
|
|
472
|
+
* the wrong column would produce a plausible number for the wrong quantity,
|
|
473
|
+
* which is the failure this whole file is about.
|
|
474
|
+
* @type {Readonly<Record<string, number>>}
|
|
475
|
+
*/
|
|
476
|
+
const COUNT_COLUMNS = Object.freeze({
|
|
477
|
+
killed: 3,
|
|
478
|
+
timedOut: 4,
|
|
479
|
+
survived: 5,
|
|
480
|
+
noCoverage: 6,
|
|
481
|
+
errors: 7,
|
|
482
|
+
});
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* The mutant counts a completed run reported, or null when it reported none.
|
|
486
|
+
*
|
|
487
|
+
* Null is a real answer and is treated as one everywhere below. A gate that
|
|
488
|
+
* could not read a tally must say the timeout share was NOT measured, never
|
|
489
|
+
* assume it was zero — "I do not know" turned into "it was fine" is the one
|
|
490
|
+
* mistake the rest of this file is organised around not making.
|
|
491
|
+
* @param {string|null|undefined} output - Stryker's combined output.
|
|
492
|
+
* @returns {{killed: number, timedOut: number, survived: number,
|
|
493
|
+
* noCoverage: number, errors: number}|null} The counts, or null.
|
|
494
|
+
*/
|
|
495
|
+
export const parseMutantTally = output => {
|
|
496
|
+
if (typeof output !== "string" || output.length === 0) return null;
|
|
497
|
+
const row = output
|
|
498
|
+
.replaceAll(ANSI_PATTERN, "")
|
|
499
|
+
.split("\n")
|
|
500
|
+
.find(line => line.startsWith(ALL_FILES_ROW_PREFIX));
|
|
501
|
+
if (row === undefined) return null;
|
|
502
|
+
const cells = row.split("|").map(cell => cell.trim());
|
|
503
|
+
const counts = Object.entries(COUNT_COLUMNS).map(([name, column]) => [
|
|
504
|
+
name,
|
|
505
|
+
cells[column] ?? "",
|
|
506
|
+
]);
|
|
507
|
+
// Every count cell has to be a count. A short row, a reformatted table or a
|
|
508
|
+
// line that merely starts with the same words all land here, and reading
|
|
509
|
+
// `Number("")` as a zero would turn any of them into a clean tally.
|
|
510
|
+
if (!counts.every(([, cell]) => COUNT_CELL_PATTERN.test(cell))) return null;
|
|
511
|
+
return Object.freeze(
|
|
512
|
+
Object.fromEntries(counts.map(([name, cell]) => [name, Number(cell)]))
|
|
513
|
+
);
|
|
514
|
+
};
|
|
515
|
+
|
|
516
|
+
/**
|
|
517
|
+
* A percentage, or NaN when there is nothing to take a percentage of.
|
|
518
|
+
* @param {number} part - Numerator.
|
|
519
|
+
* @param {number} whole - Denominator.
|
|
520
|
+
* @returns {number} `part/whole` as a percentage.
|
|
521
|
+
*/
|
|
522
|
+
const percent = (part, whole) =>
|
|
523
|
+
whole === 0 ? Number.NaN : (part / whole) * 100;
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* The score as reported, and the score with timeouts NOT credited.
|
|
527
|
+
*
|
|
528
|
+
* ## Why the second number exists
|
|
529
|
+
*
|
|
530
|
+
* Stryker scores a timed-out mutant as KILLED. A mutant whose covering tests
|
|
531
|
+
* exceed the per-mutant budget is therefore counted as detected, identically to
|
|
532
|
+
* one an assertion caught — and which bucket a mutant lands in depends on how
|
|
533
|
+
* busy the machine was. The perverse consequence is that **a slower box yields
|
|
534
|
+
* a better score**, and a genuine regression can hide inside the timeout
|
|
535
|
+
* bucket: a mutation that makes the covering tests hang scores exactly like one
|
|
536
|
+
* they catch.
|
|
537
|
+
*
|
|
538
|
+
* `withoutTimeouts` reclassifies every timeout as survived. That is the worst
|
|
539
|
+
* case rather than the truth — some timeouts are genuine infinite loops the
|
|
540
|
+
* mutation correctly introduced — and it is deliberately the worst case,
|
|
541
|
+
* because it is the only one of the two that cannot be inflated by the clock.
|
|
542
|
+
* A gate may be pessimistic about what it proved; it may not be optimistic.
|
|
543
|
+
*
|
|
544
|
+
* Errors are excluded from every denominator, which is Stryker's own
|
|
545
|
+
* arithmetic: a compile or runtime error produced no verdict about the mutant.
|
|
546
|
+
* @param {{killed: number, timedOut: number, survived: number,
|
|
547
|
+
* noCoverage: number}} tally - From {@link parseMutantTally}.
|
|
548
|
+
* @returns {{detected: number, total: number, reported: number,
|
|
549
|
+
* withoutTimeouts: number, reportedCovered: number,
|
|
550
|
+
* coveredWithoutTimeouts: number, timedOutShare: number}} The accounting.
|
|
551
|
+
*/
|
|
552
|
+
export const timeoutAccounting = tally => {
|
|
553
|
+
const detected = tally.killed + tally.timedOut;
|
|
554
|
+
const total = detected + tally.survived + tally.noCoverage;
|
|
555
|
+
const covered = detected + tally.survived;
|
|
556
|
+
return Object.freeze({
|
|
557
|
+
detected,
|
|
558
|
+
total,
|
|
559
|
+
reported: percent(detected, total),
|
|
560
|
+
withoutTimeouts: percent(tally.killed, total),
|
|
561
|
+
reportedCovered: percent(detected, covered),
|
|
562
|
+
coveredWithoutTimeouts: percent(tally.killed, covered),
|
|
563
|
+
timedOutShare: percent(tally.timedOut, detected),
|
|
564
|
+
});
|
|
565
|
+
};
|
|
566
|
+
|
|
567
|
+
/**
|
|
568
|
+
* Largest share of DETECTED mutants that may have been decided by the clock.
|
|
569
|
+
*
|
|
570
|
+
* A bound on how load-dependent the score is allowed to be, not a performance
|
|
571
|
+
* budget and not an aspiration. One whole-list run measured 117 timeouts
|
|
572
|
+
* against 3,455 detected — **3.39%**, worth up to 2.00 score points — on a box
|
|
573
|
+
* at one-minute load 40-77. 5% leaves room above that measurement and still
|
|
574
|
+
* refuses a run where a twentieth of the evidence is a stopwatch reading.
|
|
575
|
+
*
|
|
576
|
+
* It lives here rather than in a config file on purpose. `stryker.conf.json`'s
|
|
577
|
+
* `thresholds` are governed by the threshold ratchet; this is not, so putting
|
|
578
|
+
* it beside them would create a number that looks governed and is not. As a
|
|
579
|
+
* constant, raising it is a code change that gets reviewed.
|
|
580
|
+
* `MUTATION_TIMEOUT_SHARE_MAX` exists for a one-off run, not for a project's
|
|
581
|
+
* standing configuration.
|
|
582
|
+
* @type {number}
|
|
583
|
+
*/
|
|
584
|
+
export const DEFAULT_TIMEOUT_SHARE_CEILING_PCT = 5;
|
|
585
|
+
|
|
586
|
+
/**
|
|
587
|
+
* Detected mutants needed before the share above means anything.
|
|
588
|
+
*
|
|
589
|
+
* A share is a ratio, and a ratio over a handful of mutants is noise. The
|
|
590
|
+
* diff-only gate routinely runs over a single changed guard: at 8 detected
|
|
591
|
+
* mutants one timeout is 12.5%, which would fail a push for a reason that is
|
|
592
|
+
* entirely about the machine — the exact false red this gate's own "a timeout
|
|
593
|
+
* is not a score" section exists to prevent. Below this many detected mutants
|
|
594
|
+
* the share is REPORTED and not enforced, and the report says which.
|
|
595
|
+
*
|
|
596
|
+
* The score recomputed without timeouts is enforced at every size, because it
|
|
597
|
+
* is not a ratio over a small sample — it is the same score, computed honestly.
|
|
598
|
+
* @type {number}
|
|
599
|
+
*/
|
|
600
|
+
export const MIN_DETECTED_FOR_SHARE = 50;
|
|
601
|
+
|
|
602
|
+
/**
|
|
603
|
+
* The share ceiling in force.
|
|
604
|
+
* @returns {number} The ceiling, as a percentage.
|
|
605
|
+
*/
|
|
606
|
+
export const resolveTimeoutShareCeiling = () => {
|
|
607
|
+
const raw = process.env.MUTATION_TIMEOUT_SHARE_MAX;
|
|
608
|
+
if (raw === undefined) return DEFAULT_TIMEOUT_SHARE_CEILING_PCT;
|
|
609
|
+
const parsed = Number(raw);
|
|
610
|
+
return Number.isFinite(parsed) && parsed >= 0
|
|
611
|
+
? parsed
|
|
612
|
+
: DEFAULT_TIMEOUT_SHARE_CEILING_PCT;
|
|
613
|
+
};
|
|
614
|
+
|
|
615
|
+
/**
|
|
616
|
+
* The break threshold the project declared, or null when it declared none.
|
|
617
|
+
*
|
|
618
|
+
* Null is not zero. Stryker's own default is to have no breaking threshold at
|
|
619
|
+
* all, and a project with none has not asked for a floor — inventing one here
|
|
620
|
+
* would fail a gate against a number nobody chose.
|
|
621
|
+
* @param {string} cwd - Project root.
|
|
622
|
+
* @returns {number|null} `thresholds.break`, or null.
|
|
623
|
+
*/
|
|
624
|
+
export const resolveBreakThreshold = cwd => {
|
|
625
|
+
const value = readJsonConfig(cwd)?.thresholds?.break;
|
|
626
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
627
|
+
};
|
|
628
|
+
|
|
629
|
+
/** Decimal places every score in the accounting block is printed to. */
|
|
630
|
+
const SCORE_PRECISION = 2;
|
|
631
|
+
|
|
632
|
+
/**
|
|
633
|
+
* One score, printed the way Stryker prints one.
|
|
634
|
+
* @param {number} value - A percentage.
|
|
635
|
+
* @returns {string} Two decimal places, or `n/a`.
|
|
636
|
+
*/
|
|
637
|
+
const score = value =>
|
|
638
|
+
Number.isFinite(value) ? value.toFixed(SCORE_PRECISION) : "n/a";
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* The block every completed run prints, whatever its verdict.
|
|
642
|
+
*
|
|
643
|
+
* This is the reporting half of the fix. Before it, the timeout bucket was
|
|
644
|
+
* credited and unmeasured: a reader could not tell how much of the number in
|
|
645
|
+
* front of them came from an assertion and how much from a stopwatch.
|
|
646
|
+
* @param {{killed: number, timedOut: number}} tally - The counts.
|
|
647
|
+
* @param {{detected: number, reported: number, withoutTimeouts: number,
|
|
648
|
+
* reportedCovered: number, coveredWithoutTimeouts: number,
|
|
649
|
+
* timedOutShare: number}} accounting - From {@link timeoutAccounting}.
|
|
650
|
+
* @param {number} ceiling - The share ceiling in force.
|
|
651
|
+
* @returns {string} The block.
|
|
652
|
+
*/
|
|
653
|
+
const accountingBlock = (tally, accounting, ceiling) => {
|
|
654
|
+
const sampleNote =
|
|
655
|
+
accounting.detected < MIN_DETECTED_FOR_SHARE
|
|
656
|
+
? `, NOT enforced below ${MIN_DETECTED_FOR_SHARE} detected mutants — too small a sample to be a ratio`
|
|
657
|
+
: "";
|
|
658
|
+
return (
|
|
659
|
+
`🕒 ${OUTCOMES.timeoutAccounting}\n` +
|
|
660
|
+
` ${tally.timedOut} of ${accounting.detected} detected mutant(s) were decided by the per-mutant\n` +
|
|
661
|
+
" clock rather than by an assertion. Stryker scores those as KILLED.\n" +
|
|
662
|
+
` score as reported ${score(accounting.reported)} | without crediting timeouts ${score(accounting.withoutTimeouts)}\n` +
|
|
663
|
+
` covered as reported ${score(accounting.reportedCovered)} | without crediting timeouts ${score(accounting.coveredWithoutTimeouts)}\n` +
|
|
664
|
+
` timed-out share of detected: ${score(accounting.timedOutShare)}% (ceiling ${ceiling}%${sampleNote})`
|
|
665
|
+
);
|
|
666
|
+
};
|
|
667
|
+
|
|
668
|
+
/**
|
|
669
|
+
* The block printed when a run could not be accounted for at all.
|
|
670
|
+
* @returns {string} The block.
|
|
671
|
+
*/
|
|
672
|
+
const unmeasuredBlock = () =>
|
|
673
|
+
`⚠️ ${OUTCOMES.timeoutUnmeasured}\n` +
|
|
674
|
+
" No `All files` row was found in Stryker's output, so the timed-out share of\n" +
|
|
675
|
+
" this score was NOT measured. That is not a claim it was zero: Stryker scores\n" +
|
|
676
|
+
" a timed-out mutant as KILLED, so an unmeasured share is an unknown amount of\n" +
|
|
677
|
+
" this score decided by the clock.\n" +
|
|
678
|
+
' Add "clear-text" to `reporters` in your Stryker config to measure it, or set\n' +
|
|
679
|
+
" MUTATION_CAPTURE=0 to say out loud that this run is not being accounted for.";
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Judge a completed run on what it can prove, rather than on what it counted.
|
|
683
|
+
*
|
|
684
|
+
* Two verdicts, and neither can turn a red run green — both are checks the gate
|
|
685
|
+
* applies IN ADDITION to Stryker's own, on a run Stryker already judged:
|
|
686
|
+
*
|
|
687
|
+
* - the score recomputed without crediting timeouts is under the project's
|
|
688
|
+
* `thresholds.break`, so the run cleared the floor only because the clock
|
|
689
|
+
* helped it;
|
|
690
|
+
* - the timed-out share of detected mutants is over the ceiling, so the score is
|
|
691
|
+
* more a property of the machine than of the tests, whatever its value.
|
|
692
|
+
* @param {{killed: number, timedOut: number}} tally - The counts.
|
|
693
|
+
* @param {number|null} breakThreshold - `thresholds.break`, or null.
|
|
694
|
+
* @param {number} ceiling - The share ceiling in force.
|
|
695
|
+
* @returns {{failed: boolean, measured: boolean, message: string}} The block,
|
|
696
|
+
* whether a tally was read, and whether it fails.
|
|
697
|
+
*/
|
|
698
|
+
export const judgeTimeoutAccounting = (tally, breakThreshold, ceiling) => {
|
|
699
|
+
const accounting = timeoutAccounting(tally);
|
|
700
|
+
const report = accountingBlock(tally, accounting, ceiling);
|
|
701
|
+
|
|
702
|
+
if (
|
|
703
|
+
breakThreshold !== null &&
|
|
704
|
+
Number.isFinite(accounting.withoutTimeouts) &&
|
|
705
|
+
accounting.withoutTimeouts < breakThreshold
|
|
706
|
+
) {
|
|
707
|
+
return {
|
|
708
|
+
failed: true,
|
|
709
|
+
measured: true,
|
|
710
|
+
message:
|
|
711
|
+
`${report}\n❌ ${OUTCOMES.inflatedByTimeouts}\n` +
|
|
712
|
+
` Without crediting the ${tally.timedOut} timed-out mutant(s), this run scores\n` +
|
|
713
|
+
` ${score(accounting.withoutTimeouts)} against a break threshold of ${breakThreshold}. It cleared the floor\n` +
|
|
714
|
+
" ONLY because Stryker counts a timeout as a kill, and which mutants time out\n" +
|
|
715
|
+
" is a property of how busy this machine was — so a slower box would have\n" +
|
|
716
|
+
" scored HIGHER. Nothing here demonstrably caught those mutants.\n" +
|
|
717
|
+
" Strengthen the tests covering them, or find out why they hang. Do NOT raise\n" +
|
|
718
|
+
' "timeoutMS": that converts a timeout into a slow pass and hides the same gap.',
|
|
719
|
+
};
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
if (
|
|
723
|
+
accounting.detected >= MIN_DETECTED_FOR_SHARE &&
|
|
724
|
+
Number.isFinite(accounting.timedOutShare) &&
|
|
725
|
+
accounting.timedOutShare > ceiling
|
|
726
|
+
) {
|
|
727
|
+
return {
|
|
728
|
+
failed: true,
|
|
729
|
+
measured: true,
|
|
730
|
+
message:
|
|
731
|
+
`${report}\n❌ ${OUTCOMES.timeoutShareExceeded}\n` +
|
|
732
|
+
` ${score(accounting.timedOutShare)}% of what this run counted as DETECTED was decided by the\n` +
|
|
733
|
+
` per-mutant clock, over a ceiling of ${ceiling}%. The score above is reported\n` +
|
|
734
|
+
" rather than relied on: too much of it is a fact about this machine.\n" +
|
|
735
|
+
" Investigate the mutants that hang, or run this where it is not contended.",
|
|
736
|
+
};
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
return { failed: false, measured: true, message: report };
|
|
740
|
+
};
|
|
741
|
+
|
|
742
|
+
/**
|
|
743
|
+
* The whole accounting step, from a captured transcript to a verdict.
|
|
744
|
+
* @param {string|null} output - Stryker's combined output, or null.
|
|
745
|
+
* @param {string} cwd - Project root.
|
|
746
|
+
* @returns {{failed: boolean, measured: boolean, message: string}} The block,
|
|
747
|
+
* whether a tally was read, and whether it fails.
|
|
748
|
+
*/
|
|
749
|
+
export const accountForTimeouts = (output, cwd) => {
|
|
750
|
+
const tally = parseMutantTally(output);
|
|
751
|
+
if (tally === null)
|
|
752
|
+
return { failed: false, measured: false, message: unmeasuredBlock() };
|
|
753
|
+
return judgeTimeoutAccounting(
|
|
754
|
+
tally,
|
|
755
|
+
resolveBreakThreshold(cwd),
|
|
756
|
+
resolveTimeoutShareCeiling()
|
|
757
|
+
);
|
|
758
|
+
};
|
|
759
|
+
|
|
390
760
|
/**
|
|
391
761
|
* How many of Stryker's last lines an unrecognised failure quotes back.
|
|
392
762
|
* @type {number}
|
|
@@ -415,11 +785,20 @@ const tailOf = output => {
|
|
|
415
785
|
};
|
|
416
786
|
|
|
417
787
|
/**
|
|
418
|
-
* How many mutants the clock decided
|
|
788
|
+
* How many mutants the clock decided.
|
|
789
|
+
*
|
|
790
|
+
* The clear-text table is preferred and the progress line is the fallback,
|
|
791
|
+
* because they are not the same quantity: the table is the run's FINAL count
|
|
792
|
+
* and the progress line is a running tally printed every ten seconds, which
|
|
793
|
+
* stops wherever the last tick landed. Reading the running one when the final
|
|
794
|
+
* one is available under-reports, and this number goes into a message about how
|
|
795
|
+
* much of a score the clock decided.
|
|
419
796
|
* @param {string} output - Stryker's combined output.
|
|
420
|
-
* @returns {number} The count, or 0 when
|
|
797
|
+
* @returns {number} The count, or 0 when neither surface reported one.
|
|
421
798
|
*/
|
|
422
799
|
const timedOutMutants = output => {
|
|
800
|
+
const tally = parseMutantTally(output);
|
|
801
|
+
if (tally !== null) return tally.timedOut;
|
|
423
802
|
const tallies = [...output.matchAll(TIMED_OUT_MUTANTS_PATTERN)];
|
|
424
803
|
return tallies.length === 0 ? 0 : Number(tallies[tallies.length - 1][1]);
|
|
425
804
|
};
|
|
@@ -804,15 +1183,24 @@ const runStrykerCaptured = (cwd, entry, env) => {
|
|
|
804
1183
|
|
|
805
1184
|
/**
|
|
806
1185
|
* Hand the selected files to Stryker.
|
|
1186
|
+
*
|
|
1187
|
+
* `selected` empty means the WHOLE LIST: `--mutate` is omitted entirely so the
|
|
1188
|
+
* project's committed patterns stand, which is what `--all` asks for. It is not
|
|
1189
|
+
* reachable by accident — an empty diff selection is reported as
|
|
1190
|
+
* `nothing-to-mutate` and returns long before here.
|
|
807
1191
|
* @param {string} cwd - Project root.
|
|
808
|
-
* @param {readonly string[]} selected - Repository-relative paths
|
|
1192
|
+
* @param {readonly string[]} selected - Repository-relative paths, or empty for
|
|
1193
|
+
* the project's own `mutate` patterns.
|
|
809
1194
|
* @returns {{code: number, output: string|null}} Stryker's status, and its
|
|
810
1195
|
* output when this machine could keep a copy.
|
|
811
1196
|
*/
|
|
812
1197
|
const runStryker = (cwd, selected) => {
|
|
813
1198
|
const scope = selected.join(",");
|
|
814
1199
|
const base = strykerEntry(cwd);
|
|
815
|
-
const entry = {
|
|
1200
|
+
const entry = {
|
|
1201
|
+
file: base.file,
|
|
1202
|
+
args: scope === "" ? base.args : [...base.args, "--mutate", scope],
|
|
1203
|
+
};
|
|
816
1204
|
const env = {
|
|
817
1205
|
...process.env,
|
|
818
1206
|
// What the run was scoped to, for a test-runner config that wants to
|
|
@@ -827,12 +1215,65 @@ const runStryker = (cwd, selected) => {
|
|
|
827
1215
|
);
|
|
828
1216
|
};
|
|
829
1217
|
|
|
1218
|
+
/**
|
|
1219
|
+
* The argument that mutates the project's whole `mutate` list, not a diff.
|
|
1220
|
+
*
|
|
1221
|
+
* The whole-list run used to be `stryker run` invoked directly, which meant it
|
|
1222
|
+
* bypassed this gate entirely — and with it the timeout accounting, on the one
|
|
1223
|
+
* run big enough for the timeout bucket to be worth anything. It is the same
|
|
1224
|
+
* gate either way now; only the scope differs.
|
|
1225
|
+
* @type {string}
|
|
1226
|
+
*/
|
|
1227
|
+
export const WHOLE_LIST_FLAG = "--all";
|
|
1228
|
+
|
|
1229
|
+
/**
|
|
1230
|
+
* Report a completed Stryker run, and decide what its status should be.
|
|
1231
|
+
*
|
|
1232
|
+
* ## The two things a completed run is judged on
|
|
1233
|
+
*
|
|
1234
|
+
* Stryker's own verdict stands and is never overturned — nothing here can make
|
|
1235
|
+
* a failing run pass. What is added is the accounting that Stryker does not do:
|
|
1236
|
+
* it scores a timed-out mutant as KILLED, so part of every score it reports was
|
|
1237
|
+
* decided by how busy the machine was rather than by an assertion. That part is
|
|
1238
|
+
* measured, printed, and — when it is what carried the run over the floor —
|
|
1239
|
+
* failed on. See {@link timeoutAccounting}.
|
|
1240
|
+
* @param {string} cwd - Project root.
|
|
1241
|
+
* @param {{code: number, output: string|null}} result - From `runStryker`.
|
|
1242
|
+
* @returns {number} The exit code the caller should use.
|
|
1243
|
+
*/
|
|
1244
|
+
const reportRun = (cwd, result) => {
|
|
1245
|
+
const accounting = accountForTimeouts(result.output, cwd);
|
|
1246
|
+
if (result.code !== 0) {
|
|
1247
|
+
// Stryker's own verdict stands; what is added is WHICH failure it was. The
|
|
1248
|
+
// gate used to end here on a bare status, and the hook above it then had to
|
|
1249
|
+
// guess — which it did, out loud, as "mutation score below threshold", for
|
|
1250
|
+
// dry runs that never computed a score at all.
|
|
1251
|
+
console.error(
|
|
1252
|
+
classifyStrykerFailure(result.output, resolveTimeoutBudgets(cwd)).message
|
|
1253
|
+
);
|
|
1254
|
+
// A failure that produced no table produced no score either — a dry run
|
|
1255
|
+
// killed by the clock is the common case — so the unmeasured warning would
|
|
1256
|
+
// be noise on top of a failure that has already explained itself. A failure
|
|
1257
|
+
// that DID produce one still gets the honest recomputation: a run under the
|
|
1258
|
+
// floor is under it by more than Stryker said.
|
|
1259
|
+
if (accounting.measured) console.error(accounting.message);
|
|
1260
|
+
return result.code;
|
|
1261
|
+
}
|
|
1262
|
+
if (accounting.failed) {
|
|
1263
|
+
console.error(accounting.message);
|
|
1264
|
+
return 1;
|
|
1265
|
+
}
|
|
1266
|
+
console.log(accounting.message);
|
|
1267
|
+
return 0;
|
|
1268
|
+
};
|
|
1269
|
+
|
|
830
1270
|
/**
|
|
831
1271
|
* The whole gate, as one function so it can be driven from a test.
|
|
832
1272
|
* @param {string} [cwd] - Project root; defaults to the process working dir.
|
|
1273
|
+
* @param {readonly string[]} [argv] - Arguments after the script name.
|
|
833
1274
|
* @returns {number} The exit code the caller should use.
|
|
834
1275
|
*/
|
|
835
|
-
export const runGate = (cwd = process.cwd()) => {
|
|
1276
|
+
export const runGate = (cwd = process.cwd(), argv = []) => {
|
|
836
1277
|
const gate = readGate(cwd);
|
|
837
1278
|
const enabled = envFlag("MUTATION_ENABLED") ?? gate.enabled === true;
|
|
838
1279
|
const since = process.env.MUTATION_SINCE || gate.since || "main";
|
|
@@ -860,6 +1301,14 @@ export const runGate = (cwd = process.cwd()) => {
|
|
|
860
1301
|
return 1;
|
|
861
1302
|
}
|
|
862
1303
|
|
|
1304
|
+
if (argv.includes(WHOLE_LIST_FLAG)) {
|
|
1305
|
+
console.log(
|
|
1306
|
+
`🧬 ${OUTCOMES.wholeList} — Stryker over every pattern in ` +
|
|
1307
|
+
`${declaration.source}, with no diff scoping.`
|
|
1308
|
+
);
|
|
1309
|
+
return reportRun(cwd, runStryker(cwd, []));
|
|
1310
|
+
}
|
|
1311
|
+
|
|
863
1312
|
const base = resolveDiffBase(cwd, since);
|
|
864
1313
|
if (!base) {
|
|
865
1314
|
console.log(
|
|
@@ -913,16 +1362,7 @@ export const runGate = (cwd = process.cwd()) => {
|
|
|
913
1362
|
);
|
|
914
1363
|
for (const file of scope.selected) console.log(` • ${file}`);
|
|
915
1364
|
|
|
916
|
-
|
|
917
|
-
if (result.code === 0) return 0;
|
|
918
|
-
// Stryker's own verdict stands; what is added is WHICH failure it was. The
|
|
919
|
-
// gate used to end here on a bare status, and the hook above it then had to
|
|
920
|
-
// guess — which it did, out loud, as "mutation score below threshold", for
|
|
921
|
-
// dry runs that never computed a score at all.
|
|
922
|
-
console.error(
|
|
923
|
-
classifyStrykerFailure(result.output, resolveTimeoutBudgets(cwd)).message
|
|
924
|
-
);
|
|
925
|
-
return result.code;
|
|
1365
|
+
return reportRun(cwd, runStryker(cwd, scope.selected));
|
|
926
1366
|
};
|
|
927
1367
|
|
|
928
1368
|
/**
|
|
@@ -930,7 +1370,7 @@ export const runGate = (cwd = process.cwd()) => {
|
|
|
930
1370
|
* @returns {void}
|
|
931
1371
|
*/
|
|
932
1372
|
export const runCli = () => {
|
|
933
|
-
process.exit(runGate());
|
|
1373
|
+
process.exit(runGate(process.cwd(), process.argv.slice(2)));
|
|
934
1374
|
};
|
|
935
1375
|
|
|
936
1376
|
if (invokedAsScript(import.meta.url)) runCli();
|
package/ui/index.html
CHANGED
|
@@ -9835,19 +9835,6 @@
|
|
|
9835
9835
|
reason: "checking…",
|
|
9836
9836
|
},
|
|
9837
9837
|
],
|
|
9838
|
-
[
|
|
9839
|
-
{ t: "mono", v: "🕷️ OWASP ZAP Baseline" },
|
|
9840
|
-
{
|
|
9841
|
-
t: "text",
|
|
9842
|
-
v: "DAST baseline scan against zap_target_url with per-rule IGNORE/WARN/FAIL decisions",
|
|
9843
|
-
},
|
|
9844
|
-
{
|
|
9845
|
-
t: "status",
|
|
9846
|
-
jobId: "zap_baseline",
|
|
9847
|
-
v: null,
|
|
9848
|
-
reason: "checking…",
|
|
9849
|
-
},
|
|
9850
|
-
],
|
|
9851
9838
|
[
|
|
9852
9839
|
{ t: "mono", v: "🧾 Compliance Validation" },
|
|
9853
9840
|
{
|
|
@@ -9968,26 +9955,6 @@
|
|
|
9968
9955
|
},
|
|
9969
9956
|
],
|
|
9970
9957
|
},
|
|
9971
|
-
{
|
|
9972
|
-
card: "DAST (OWASP ZAP)",
|
|
9973
|
-
rows: [
|
|
9974
|
-
{
|
|
9975
|
-
key: "zap_target_url",
|
|
9976
|
-
label: "ZAP target URL",
|
|
9977
|
-
desc: "",
|
|
9978
|
-
control: txt("https://staging.acme-app.io", {
|
|
9979
|
-
mono: true,
|
|
9980
|
-
wide: true,
|
|
9981
|
-
}),
|
|
9982
|
-
},
|
|
9983
|
-
{
|
|
9984
|
-
key: "zap_rules_file",
|
|
9985
|
-
label: "Rules file",
|
|
9986
|
-
desc: "Per-rule IGNORE / WARN / FAIL decisions.",
|
|
9987
|
-
control: txt(".zap/baseline.conf", { mono: true }),
|
|
9988
|
-
},
|
|
9989
|
-
],
|
|
9990
|
-
},
|
|
9991
9958
|
],
|
|
9992
9959
|
};
|
|
9993
9960
|
|