mutineer 0.11.3 → 0.11.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b671af802d8c032a7b9fa3e48eac7d63478930f6077ce58793007c4e8aa98de6
-  data.tar.gz: ef1ef88c9b6af5a0c634d0e9f7bf68ce9644e8c5d5a48002a5b3de1c1a4b7ffe
+  metadata.gz: 85591c1b61bcdc3c8a1d39b5b657f667bf7de9039d61e05ab3e5691692f555f0
+  data.tar.gz: 76acdc054e82d9ea10dee4a63bcbb5f0f86db37dd585323f734991f413f21e61
 SHA512:
-  metadata.gz: 69ab5b4e78163653f838afba995273f156ad51d15b4613d8bd40f1d4f0571423018e5b1a8ffd59160405e81e9d2aca26fce08deba802ee896be3fd37254ff887
-  data.tar.gz: 98e011da927ed5108fec748628a9bb0a69d74e03cdf3b1f2d395b5082e38f3478ad45bbb5caa58fa7099ecb7d787086cdaebef20828fe5b670591c4fa8e0b5bf
+  metadata.gz: b5855d2e0a9d4e226fe0ad2034cc4d08f4b5ac5e2d34df9e92f97b9d344ea5a50c4df01bbdcb0973dc66e8ccde717baab8842935fa07a4cad1238f77b9bebf42
+  data.tar.gz: 3a5c57795de651128a3d3597b2f1776fd32ff76aeaea4edf14037a1ec32e13e72e54c38a56ef8c209f3ee1d6afc0ffbb5fd47153967803c841e64d989d80a047

data/CHANGELOG.md

@@ -6,6 +6,27 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [0.11.4] - 2026-07-29
+
+### Fixed
+- **`--threshold` now gates on the run being complete, not just its score** — a
+  score is `killed / (killed + survived)`, so mutants that error, time out or come
+  back uncapturable are excluded from the denominator: a broken harness *raised*
+  the score instead of lowering it. Ninety errored mutants and ten that ran (nine
+  killed) reported 90% and exited 0, so CI could not tell a complete run from a
+  mostly-broken one. Past 10% of attempted mutants producing no verdict, a positive
+  `--threshold` now exits 1 whatever the score, and the report says which states
+  broke, on every `--format`. A single bad mutant never trips it, however small the
+  run, so a `--since` PR with a handful of mutants keeps its flake tolerance (#78).
+
+### Added
+- **`no_verdict[]` in the JSON report** (`schema_version` 1.2) — every attempted
+  mutant that produced no verdict, with its `status` and, where there is one, the
+  `details` explaining the cause. `Result#details` was built and rendered in no
+  format at all, so a daemon crash reached the user as nothing but a larger errored
+  count. `summary` gains `attempted` and `no_verdict`, the two figures the
+  completeness gate is computed from (#78).
+
 ## [0.11.3] - 2026-07-29
 
 ### Fixed
@@ -297,6 +318,7 @@ Rails hardening + CI batch (issues #8–#13), all verified Rails-free.
 - `.mutineer.yml` configuration (CLI > config > default precedence).
 - Byte-correct source handling for multibyte (UTF-8) sources.
 
+[0.11.4]: https://github.com/davidteren/mutineer/releases/tag/v0.11.4
 [0.11.3]: https://github.com/davidteren/mutineer/releases/tag/v0.11.3
 [0.11.2]: https://github.com/davidteren/mutineer/releases/tag/v0.11.2
 [0.11.1]: https://github.com/davidteren/mutineer/releases/tag/v0.11.1

data/README.md

@@ -47,7 +47,7 @@ mutineer run lib/calculator.rb --test test/calculator_test.rb --threshold 90
 |------|---------|
 | `--test FILE` | Test file covering the sources (repeatable) |
 | `--operators LIST` | Comma-separated operator names (default: the Tier-1 set) |
-| `--threshold FLOAT` | Exit 1 when the score is below FLOAT (default: 0 = off) |
+| `--threshold FLOAT` | Exit 1 when the score is below FLOAT, or when nothing could be scored and something broke, or more than one mutant produced no verdict and they exceed 10% of those attempted (default: 0 = off) |
 | `--only NAME` | Restrict to one fully-qualified subject, e.g. `Calculator#add` |
 | `--framework NAME` | `minitest` (default) or `rspec`; auto-detected as rspec when most `--test` files end in `_spec.rb` |
 | `--since REF` | Only mutate lines changed since git `REF` (e.g. `origin/main`) — ideal for PR CI |
@@ -70,7 +70,7 @@ mutineer run lib/calculator.rb --test test/calculator_test.rb --threshold 90
 | Code | Meaning |
 |------|---------|
 | `0` | Score ≥ threshold (or no threshold set) |
-| `1` | Survivors below threshold, or a runtime error |
+| `1` | Score below threshold, nothing could be scored and something broke, or more than one mutant produced no verdict and they exceed 10% of those attempted, a `--baseline` regression, or a runtime error |
 | `2` | Usage / invalid-flag error |
 
 ### Operators

data/lib/mutineer/reporter.rb

@@ -14,6 +14,17 @@ module Mutineer
   # `source_map` is { file_path => raw source string }, used to extract the
   # containing source line for each survivor diff.
   class Reporter
+    # Share of attempted mutants that may produce no verdict before `--threshold`
+    # stops trusting the score. A handful of flaky mutants in a large run is noise;
+    # a mostly-broken run is not a score. Deliberately not a flag: no one has needed
+    # a different number yet.
+    BROKEN_SHARE_LIMIT = 0.10
+
+    # A share alone gives a small run no tolerance at all: on a `--since` PR that
+    # yields 8 mutants, one timeout is 12.5%. README recommends exactly that
+    # workflow, so one bad mutant never fails the gate on its own, at any size.
+    BROKEN_FLOOR = 1
+
     def initialize(aggregate, source_map)
       @agg = aggregate
       @source_map = source_map
@@ -42,6 +53,20 @@ module Mutineer
       else
         out.print rendered
       end
+
+      # Both ways a run can fail the gate on completeness, said here rather than in
+      # the human renderer: --format json is the documented CI path, and a run that
+      # exits 1 must say why on every format, not only the one a person reads.
+      return unless threshold&.positive?
+
+      if @agg.mutation_score.nil? && broken_nil_score?
+        err.puts "[mutineer] nothing could be scored (#{broken_counts_detail}), so the " \
+                 "--threshold gate fails. See no_verdict[] in --format json for the cause of each."
+      elsif broken_share_exceeded?
+        err.puts "[mutineer] #{no_verdict_ratio}: #{broken_counts_detail}. The score covers " \
+                 "only part of the run, so the --threshold gate fails. See no_verdict[] in " \
+                 "--format json for the cause of each."
+      end
     end
 
     # Renders the human report.
@@ -85,6 +110,11 @@ module Mutineer
         return 0 # pure no_coverage / ignored / empty — gate skipped
       end
 
+      # A score computed over a small slice of what was attempted is not this
+      # suite's score. Without this, 90 errored mutants and 10 that ran (9 killed)
+      # reports 90% and exits 0, so CI cannot tell a complete run from a broken one.
+      return 1 if broken_share_exceeded?
+
       score >= threshold ? 0 : 1
     end
 
@@ -106,7 +136,7 @@ module Mutineer
       score = @agg.mutation_score
 
       doc = {
-        schema_version: "1.1",
+        schema_version: "1.2",
         summary: {
           total: @agg.total, killed: killed, survived: survived,
           no_coverage: @agg.no_coverage_count,
@@ -114,6 +144,8 @@ module Mutineer
           skipped_invalid: @agg.skipped_invalid_count,
           errored: @agg.errored_count, timeout: @agg.timeout_count,
           ignored: @agg.ignored_count,
+          # The gate is computed from these two, so a consumer never re-derives them.
+          attempted: attempted_count, no_verdict: no_verdict_count,
           score: score
         },
         survivors: @agg.surviving_mutants.map { |r| survivor_json(r) }
@@ -123,6 +155,21 @@ module Mutineer
         # Same shape as no_coverage; additive key.
         uncapturable: @agg.results.select(&:uncapturable?).map { |r| no_coverage_json(r) }
                           .sort_by { |h| [h[:file], h[:line]] },
+        # Every mutant that was attempted and produced no verdict, whatever the
+        # reason — the set the --threshold completeness gate counts. Named for the
+        # condition rather than one status, because summary.errored means :error
+        # alone and a key that reconciled with neither would be worse. `details`
+        # carries the cause where there is one. Uncapturable mutants also appear in
+        # uncapturable[]; that key keeps its lean shape for existing consumers.
+        # to_s/to_i because a pre-fork failure has no subject, so its file and line
+        # are null and would not compare against a real entry. id and status extend
+        # the key to a total order: these entries collide on (file, line) far more
+        # than survivors do — several mutants on one crashy line, every pre-fork
+        # entry on ("", 0) — and sort_by is not stable, so equal keys would leave
+        # worker finish order in the output and break the byte-stability promise.
+        no_verdict: @agg.results.select { |r| r.error? || r.timeout? || r.uncapturable? }
+                        .map { |r| no_verdict_json(r) }
+                        .sort_by { |h| [h[:file].to_s, h[:line].to_i, h[:id].to_s, h[:status].to_s, h[:details].to_s] },
         # Equivalent mutants the user suppressed: emitted with their stable id so
         # the user can audit what is silenced (and copy ids for survivors they
         # want to add). Excluded from the score; never in `survivors`.
@@ -134,7 +181,7 @@ module Mutineer
                         .sort_by { |h| h[:file] }
       }
       # Additive baseline-delta block, present only with --baseline. Existing
-      # consumers ignore the extra key; schema_version stays 1.1.
+      # consumers ignore the extra key; it does not move schema_version on its own.
       doc[:baseline] = baseline_json(baseline) if baseline
       "#{JSON.generate(doc)}\n"
     end
@@ -358,6 +405,31 @@ module Mutineer
       }
     end
 
+    # An entry under the JSON `no_verdict:` key: an attempted mutant with no verdict.
+    # A pre-fork failure has no subject or mutation attached, so those degrade to
+    # nulls rather than dropping the entry — the count must still reconcile with
+    # `summary.no_verdict`.
+    #
+    # @api private
+    # @param result [Mutineer::Result] an errored or timed-out result.
+    # @return [Hash] no-verdict JSON object.
+    def no_verdict_json(result)
+      file = result.subject&.file
+      line =
+        if result.mutation && file
+          source = @source_map[file] || File.read(file)
+          source.byteslice(0, result.mutation.start_offset).count("\n") + 1
+        end
+      {
+        subject: result.subject&.qualified_name,
+        file: file,
+        line: line,
+        id: result.id,
+        status: result.status.to_s,
+        details: result.details
+      }
+    end
+
     # Writes the summary block.
     #
     # @param out [IO] output stream.
@@ -388,10 +460,10 @@ module Mutineer
                  "#{@agg.ignored_count} ignored excluded"
       if score.nil?
         out.puts "Mutation score: N/A  (no covered mutants)"
-        if broken_nil_score?
-          err.puts "[mutineer] no covered mutations (#{broken_nil_score_detail}); " \
-                   "threshold gate fails under a positive --threshold (broken harness)."
-        else
+        # Only the benign case here: the gate-failure explanation is emitted once
+        # from {report}, for every format, so it cannot be said twice or only to
+        # the reader of the human report.
+        unless broken_nil_score?
           err.puts "[mutineer] no covered mutations; mutation score is N/A and the threshold check is skipped."
         end
       else
@@ -408,11 +480,58 @@ module Mutineer
         (@agg.errored_count + @agg.timeout_count + @agg.uncapturable_count).positive?
     end
 
-    # Human-readable counts for a broken nil-score run.
+    # Mutants that were attempted but produced no verdict, over everything
+    # attempted. Above the limit the score describes too small a slice of the run
+    # to gate on. A few flaky mutants in a large run stay under it.
+    #
+    # @api private
+    # @return [Boolean] true when too much of the run failed to produce a verdict.
+    def broken_share_exceeded?
+      attempted = attempted_count
+      attempted.positive? && no_verdict_count > BROKEN_FLOOR &&
+        no_verdict_count > attempted * BROKEN_SHARE_LIMIT
+    end
+
+    # Mutants that were attempted and produced no verdict, whatever the reason.
+    #
+    # @api private
+    # @return [Integer] errored + timed out + uncapturable.
+    def no_verdict_count
+      @agg.errored_count + @agg.timeout_count + @agg.uncapturable_count
+    end
+
+    # Mutants that were actually run. Deliberately not `total`: no_coverage,
+    # skipped-invalid and ignored mutants were never attempted, so counting them
+    # would dilute the share and let a broken run slip under the limit.
+    #
+    # skipped-invalid is excluded from both sides: it means a mutant did not
+    # re-parse and was correctly never run, which is a validity outcome rather
+    # than a broken harness. Cost: an overwhelmingly-skipped run still scores on
+    # what little ran; that is our operator misbehaving and wants its own signal.
+    #
+    # @api private
+    # @return [Integer] killed + survived + no-verdict.
+    def attempted_count
+      @agg.killed_count + @agg.survived_count + no_verdict_count
+    end
+
+    # The sentence both the verdict line and the stderr note are built from, so a
+    # user cannot read one number in the report and a different one beside it.
+    #
+    # @api private
+    # @return [String] e.g. "90 of 100 attempted mutants produced no verdict (90.0%, limit 10%)".
+    def no_verdict_ratio
+      pct = (no_verdict_count * 100.0 / attempted_count).round(1)
+      "#{no_verdict_count} of #{attempted_count} attempted mutants produced no verdict " \
+        "(#{pct}%, limit #{(BROKEN_SHARE_LIMIT * 100).round}%)"
+    end
+
+    # Human-readable counts of the states that produced no verdict. Used by both
+    # the nil-score message and the completeness gate, so they agree.
     #
     # @api private
     # @return [String]
-    def broken_nil_score_detail
+    def broken_counts_detail
       parts = []
       parts << "#{@agg.errored_count} errored" if @agg.errored_count.positive?
       parts << "#{@agg.timeout_count} timeout" if @agg.timeout_count.positive?
@@ -505,13 +624,17 @@ module Mutineer
       score = @agg.mutation_score
       if score.nil?
         if broken_nil_score?
-          out.puts "FAILED: no covered mutants (#{broken_nil_score_detail}); " \
+          out.puts "FAILED: no covered mutants (#{broken_counts_detail}); " \
                    "threshold #{threshold}% cannot pass with a broken harness"
         end
         return
       end
 
-      if score >= threshold
+      # Same rule as exit_code, or the report says PASSED on a run that exits 1 —
+      # and with --output that wrong verdict is what gets archived.
+      if broken_share_exceeded?
+        out.puts "FAILED: #{no_verdict_ratio}; #{score}% covers only part of the run"
+      elsif score >= threshold
         out.puts "PASSED: #{score}% >= threshold #{threshold}%"
       else
         out.puts "FAILED: #{score}% < threshold #{threshold}%"

data/lib/mutineer/version.rb

@@ -2,5 +2,5 @@
 
 module Mutineer
   # Current Mutineer release version.
-  VERSION = "0.11.3"
+  VERSION = "0.11.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mutineer
 version: !ruby/object:Gem::Version
-  version: 0.11.3
+  version: 0.11.4
 platform: ruby
 authors:
 - David Teren