featureparity 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b98c13a1f3911211fc2d07f23ad5917f6d6326563abda2b31bdb52840750f26
-  data.tar.gz: ec5cb53697d89763e41735deea90ba9342905b5622d5d3c43187782bef3205b5
+  metadata.gz: e3dba0cc6ba18239f6952822d96bd4526d26d207e8b7e5448554de745e6517a8
+  data.tar.gz: 1521147f4b1574062b3e04f22cef0ce1cb7ef6b822f767c3f75ed66a200b6a5c
 SHA512:
-  metadata.gz: 404d64805e0a04b939f920ca3ecd5fbc6765ad4f0e11e62f396ad3794eea8b4c500168a474f06f6df914d088a517a735b0cc801b3bfb73b2733722ac0e6eae0e
-  data.tar.gz: a80c7661ab55355e9d9ab13ab2430ea2b83d52c41f3185793cf4a1b39671507f5d4f3227d1540d5be305194ad42ea99c0292e651283562b7ac71487d0baf5768
+  metadata.gz: a65ac140de45599a9a5a27f4705e179fd816bad77fe563477fd6b92efbc0d592cea80f8c90c2e519d559fc604927d6b321c3a3af99bb10c2b99b9e06b39467f5
+  data.tar.gz: c9ba4966e9a2d036f266b3646eb0c7569399d606d196175df189047de7693c55badce253c94205045449ca7b20117a6f496eeee486067c03ca2291f2caed026f

data/lib/fp/client.rb

@@ -151,7 +151,8 @@ module Fp
     # server can fall back to resolving markers itself for older clients — but it can
     # only do that by fetching the test files from GitHub, which fails silently on a
     # private repo, so resolved results are always sent when we have them.
-    def ingest_ci_results(project_id, ci_url:, surface:, junit_xml:, pr_url: nil, results: nil)
+    def ingest_ci_results(project_id, ci_url:, surface:, junit_xml:, pr_url: nil, results: nil,
+                          unmarked_failures: nil)
       body = {
         ci_url: ci_url,
         surface: surface,
@@ -160,6 +161,9 @@ module Fp
       # Only send pr_url when set, so a merged-branch run stays a plain ingest.
       body[:pr_url] = pr_url if pr_url && !pr_url.empty?
       body[:results] = results if results && !results.empty?
+      # Loose ends: failing tests with no fp:<slug> marker. The server stores them so the
+      # matrix can surface a red suite whose failures aren't bound to any requirement.
+      body[:unmarked_failures] = unmarked_failures if unmarked_failures && !unmarked_failures.empty?
 
       post("/api/projects/#{project_id}/ci_results", body)
     end

data/lib/fp/commands/ci_report.rb

@@ -82,7 +82,7 @@ module Fp
         known = known_slugs(project_id)
 
         totals = { created: 0, updated: 0, missing_failing: 0, reports: [], resolved: 0,
-                   unmatched: 0, unknown: [] }
+                   unmatched: 0, unknown: [], unmarked_failures: [] }
 
         junit_paths.each do |path|
           xml = File.read(path)
@@ -92,6 +92,7 @@ module Fp
           results, unknown = partition_known(resolved[:results], known)
           totals[:resolved] += results.length
           totals[:unknown].concat(unknown)
+          totals[:unmarked_failures].concat(resolved[:unmarked_failures])
 
           result = client.ingest_ci_results(
             project_id,
@@ -99,7 +100,8 @@ module Fp
             surface: opts[:surface],
             junit_xml: xml,
             pr_url: opts[:pr],
-            results: results
+            results: results,
+            unmarked_failures: resolved[:unmarked_failures]
           )
 
           unless result[:ok]
@@ -154,6 +156,19 @@ module Fp
                    "#{orphans.join(', ')}. Propose the requirement (fp propose) or fix the slug.")
         end
 
+        # Loose ends: tests that failed but carry no fp:<slug> marker. Not a gate failure
+        # (they bind to no requirement, so nothing can go red), but a red test nobody is
+        # tracking is worth a human's eyes — surfaced the same way orphan markers are, and
+        # persisted so it shows on the matrix's "loose ends" panel.
+        unless totals[:unmarked_failures].empty?
+          count = totals[:unmarked_failures].length
+          names = totals[:unmarked_failures].map { |f| f[:name] }.compact.take(10).join(', ')
+          annotate("#{count} failing test(s) on surface '#{opts[:surface]}' carry no fp:<slug> " \
+                   "marker, so no requirement tracks them: #{names}#{'…' if count > 10}. " \
+                   'Mark them (fp:<slug>) to bind them to a requirement, or leave them — ' \
+                   'they show on the matrix as loose ends either way.')
+        end
+
         emit_summary(opts, totals)
       end
 
@@ -214,7 +229,18 @@ module Fp
               failure_count: r.failure_count
             }
           end,
-          unmatched: resolved[:unmatched].length
+          unmatched: resolved[:unmatched].length,
+          # Loose ends: failing tests with no fp:<slug> marker. Shipped to the server so a
+          # human can see them on the matrix; a red test nobody tracks against a
+          # requirement is a real gap, even though it isn't a requirement cell.
+          unmarked_failures: Fp::JUnit.unmarked_failures(resolved[:unmatched]).map do |f|
+            {
+              name: f.name,
+              classname: f.classname,
+              source_file: f.file,
+              source_line: f.line
+            }
+          end
         }
       rescue ArgumentError => e
         output.error(e.message)
@@ -234,6 +260,7 @@ module Fp
           missing_marked_failing: totals[:missing_failing],
           markers_resolved: totals[:resolved],
           unmatched_testcases: totals[:unmatched],
+          unmarked_failures: totals[:unmarked_failures],
           skipped_unknown: totals[:unknown].uniq,
           reports: totals[:reports]
         }
@@ -253,6 +280,10 @@ module Fp
             puts "  skipped #{totals[:unknown].uniq.length} marker(s) with no matching " \
                  "requirement: #{totals[:unknown].uniq.join(', ')}"
           end
+          unless totals[:unmarked_failures].empty?
+            puts "  loose ends: #{totals[:unmarked_failures].length} failing test(s) with no " \
+                 'fp:<slug> marker (shown on the matrix, not a gate failure)'
+          end
           puts "  created: #{totals[:created]}   updated: #{totals[:updated]}   marked failing (missing): #{totals[:missing_failing]}"
         end
       end

data/lib/fp/junit.rb

@@ -35,6 +35,14 @@ module Fp
     CiResult = Struct.new(:slug, :surface, :status, :file, :line, :test_count, :failure_count,
                           keyword_init: true)
 
+    # A failing testcase that carries no fp:<slug> marker at all — a "loose end". Not a
+    # cell on the parity grid (nothing binds it to a requirement) and not an orphan
+    # marker (there is no marker); it's a red test nobody is tracking against a
+    # requirement. CI ships these to the server so a human can see, on the matrix, that
+    # the suite has failures no requirement claims. Green unmarked tests are ignored on
+    # purpose — not everything needs a requirement; only a *failing* one is a signal.
+    UnmarkedFailure = Struct.new(:name, :classname, :file, :line, keyword_init: true)
+
     # A testcase paired with the marker that annotates it. `precise` records whether we
     # know *which* test the marker sits above (matched by line or by test name) or only
     # that the marker is somewhere in the same file — a title is only worth reporting in
@@ -249,6 +257,18 @@ module Fp
       { results: results, unmatched: unmatched, missing_surface: missing_surface.uniq }
     end
 
+    # The loose ends in a set of unmatched testcases: the ones that *failed* but carry no
+    # fp:<slug> marker. Pass the `unmatched` list from `ci_results` (or `bind_markers`).
+    #
+    # Only failures are returned. A green unmarked test is fine — not everything needs a
+    # requirement — but a red one is a failure nobody is tracking, which is exactly the
+    # signal a human should see. Skipped tests are not failures, so they're excluded too.
+    def unmarked_failures(unmatched)
+      unmatched.select(&:failed?).map do |tc|
+        UnmarkedFailure.new(name: tc.name, classname: tc.classname, file: tc.file, line: tc.line)
+      end
+    end
+
     # Walk testcases and yield a Match for every one we can tie to a marker.
     # Shared by bind_markers (agent evidence) and ci_results (CI verdicts) so the two
     # can never disagree about which test a marker annotates.

data/lib/fp/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Fp
-  VERSION = '0.0.12'
+  VERSION = '0.0.13'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: featureparity
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Stowzilla