featureparity 0.0.12 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b98c13a1f3911211fc2d07f23ad5917f6d6326563abda2b31bdb52840750f26
-  data.tar.gz: ec5cb53697d89763e41735deea90ba9342905b5622d5d3c43187782bef3205b5
+  metadata.gz: 7fc65a33008ccbd43f9b8ec10f7e3c013ab3ed502980f97b64c55e14ff1d2eb5
+  data.tar.gz: a841a639236c27ea6bf7c30ba1a7f4a5710176bc0fd4bb153daac009d89e90db
 SHA512:
-  metadata.gz: 404d64805e0a04b939f920ca3ecd5fbc6765ad4f0e11e62f396ad3794eea8b4c500168a474f06f6df914d088a517a735b0cc801b3bfb73b2733722ac0e6eae0e
-  data.tar.gz: a80c7661ab55355e9d9ab13ab2430ea2b83d52c41f3185793cf4a1b39671507f5d4f3227d1540d5be305194ad42ea99c0292e651283562b7ac71487d0baf5768
+  metadata.gz: df0c381690d9fa55e189e3a057eddd35c979bb970fac56b9294ad426d758613374c483c5f4317829cedd5a85d3ff3443a543e8dbcfdff99a146a82c7de831c53
+  data.tar.gz: f2d8dd3a182c74e6e7ef5e4395803eaf86a62b510532227c2b7ec4c3bf4a3fe06db62c8866280d016c7cb4d1cb6a786b2b5c2dcb731f5f0ada518de288eb3714

data/lib/fp/client.rb

@@ -151,7 +151,8 @@ module Fp
     # server can fall back to resolving markers itself for older clients — but it can
     # only do that by fetching the test files from GitHub, which fails silently on a
     # private repo, so resolved results are always sent when we have them.
-    def ingest_ci_results(project_id, ci_url:, surface:, junit_xml:, pr_url: nil, results: nil)
+    def ingest_ci_results(project_id, ci_url:, surface:, junit_xml:, pr_url: nil, results: nil,
+                          unmarked_failures: nil)
       body = {
         ci_url: ci_url,
         surface: surface,
@@ -160,6 +161,9 @@ module Fp
       # Only send pr_url when set, so a merged-branch run stays a plain ingest.
       body[:pr_url] = pr_url if pr_url && !pr_url.empty?
       body[:results] = results if results && !results.empty?
+      # Loose ends: failing tests with no fp:<slug> marker. The server stores them so the
+      # matrix can surface a red suite whose failures aren't bound to any requirement.
+      body[:unmarked_failures] = unmarked_failures if unmarked_failures && !unmarked_failures.empty?
 
       post("/api/projects/#{project_id}/ci_results", body)
     end

data/lib/fp/commands/ci_report.rb

@@ -82,7 +82,7 @@ module Fp
         known = known_slugs(project_id)
 
         totals = { created: 0, updated: 0, missing_failing: 0, reports: [], resolved: 0,
-                   unmatched: 0, unknown: [] }
+                   unmatched: 0, unknown: [], unmarked_failures: [] }
 
         junit_paths.each do |path|
           xml = File.read(path)
@@ -92,6 +92,7 @@ module Fp
           results, unknown = partition_known(resolved[:results], known)
           totals[:resolved] += results.length
           totals[:unknown].concat(unknown)
+          totals[:unmarked_failures].concat(resolved[:unmarked_failures])
 
           result = client.ingest_ci_results(
             project_id,
@@ -99,7 +100,8 @@ module Fp
             surface: opts[:surface],
             junit_xml: xml,
             pr_url: opts[:pr],
-            results: results
+            results: results,
+            unmarked_failures: resolved[:unmarked_failures]
           )
 
           unless result[:ok]
@@ -154,6 +156,19 @@ module Fp
                    "#{orphans.join(', ')}. Propose the requirement (fp propose) or fix the slug.")
         end
 
+        # Loose ends: tests that failed but carry no fp:<slug> marker. Not a gate failure
+        # (they bind to no requirement, so nothing can go red), but a red test nobody is
+        # tracking is worth a human's eyes — surfaced the same way orphan markers are, and
+        # persisted so it shows on the matrix's "loose ends" panel.
+        unless totals[:unmarked_failures].empty?
+          count = totals[:unmarked_failures].length
+          names = totals[:unmarked_failures].map { |f| f[:name] }.compact.take(10).join(', ')
+          annotate("#{count} failing test(s) on surface '#{opts[:surface]}' carry no fp:<slug> " \
+                   "marker, so no requirement tracks them: #{names}#{'…' if count > 10}. " \
+                   'Mark them (fp:<slug>) to bind them to a requirement, or leave them — ' \
+                   'they show on the matrix as loose ends either way.')
+        end
+
         emit_summary(opts, totals)
       end
 
@@ -214,7 +229,18 @@ module Fp
               failure_count: r.failure_count
             }
           end,
-          unmatched: resolved[:unmatched].length
+          unmatched: resolved[:unmatched].length,
+          # Loose ends: failing tests with no fp:<slug> marker. Shipped to the server so a
+          # human can see them on the matrix; a red test nobody tracks against a
+          # requirement is a real gap, even though it isn't a requirement cell.
+          unmarked_failures: Fp::JUnit.unmarked_failures(resolved[:unmatched]).map do |f|
+            {
+              name: f.name,
+              classname: f.classname,
+              source_file: f.file,
+              source_line: f.line
+            }
+          end
         }
       rescue ArgumentError => e
         output.error(e.message)
@@ -234,6 +260,7 @@ module Fp
           missing_marked_failing: totals[:missing_failing],
           markers_resolved: totals[:resolved],
           unmatched_testcases: totals[:unmatched],
+          unmarked_failures: totals[:unmarked_failures],
           skipped_unknown: totals[:unknown].uniq,
           reports: totals[:reports]
         }
@@ -253,6 +280,10 @@ module Fp
             puts "  skipped #{totals[:unknown].uniq.length} marker(s) with no matching " \
                  "requirement: #{totals[:unknown].uniq.join(', ')}"
           end
+          unless totals[:unmarked_failures].empty?
+            puts "  loose ends: #{totals[:unmarked_failures].length} failing test(s) with no " \
+                 'fp:<slug> marker (shown on the matrix, not a gate failure)'
+          end
           puts "  created: #{totals[:created]}   updated: #{totals[:updated]}   marked failing (missing): #{totals[:missing_failing]}"
         end
       end

data/lib/fp/junit.rb

@@ -35,11 +35,21 @@ module Fp
     CiResult = Struct.new(:slug, :surface, :status, :file, :line, :test_count, :failure_count,
                           keyword_init: true)
 
+    # A failing testcase that carries no fp:<slug> marker at all — a "loose end". Not a
+    # cell on the parity grid (nothing binds it to a requirement) and not an orphan
+    # marker (there is no marker); it's a red test nobody is tracking against a
+    # requirement. CI ships these to the server so a human can see, on the matrix, that
+    # the suite has failures no requirement claims. Green unmarked tests are ignored on
+    # purpose — not everything needs a requirement; only a *failing* one is a signal.
+    UnmarkedFailure = Struct.new(:name, :classname, :file, :line, keyword_init: true)
+
     # A testcase paired with the marker that annotates it. `precise` records whether we
     # know *which* test the marker sits above (matched by line or by test name) or only
     # that the marker is somewhere in the same file — a title is only worth reporting in
-    # the first case.
-    Match = Struct.new(:testcase, :marker, :precise, keyword_init: true)
+    # the first case. `repo_file` is the testcase's source path made relative to the git
+    # repo root (see repo_relative_file), so evidence links resolve regardless of the
+    # base_dir the runner reported paths against.
+    Match = Struct.new(:testcase, :marker, :precise, :repo_file, keyword_init: true)
 
     # Evidence discovered by binding a testcase to a marker.
     # `line` is the 1-based line number of the test in its source file: the
@@ -183,7 +193,7 @@ module Fp
           if surface.nil? || surface.empty?
             missing_surface << { slug: marker[:slug], file: tc.file }
           else
-            record_binding(bindings, marker[:slug], surface, tc, marker, titles_reliable: match.precise)
+            record_binding(bindings, marker[:slug], surface, tc, marker, match.repo_file, titles_reliable: match.precise)
           end
         end
       end
@@ -224,7 +234,7 @@ module Fp
           end
 
           key = [marker[:slug], surface]
-          agg = aggregates[key] ||= { file: tc.file, line: tc.line || marker[:line], ran: 0, failed: 0, skipped: 0 }
+          agg = aggregates[key] ||= { file: match.repo_file || tc.file, line: tc.line || marker[:line], ran: 0, failed: 0, skipped: 0 }
           if tc.stub?
             agg[:skipped] += 1
           else
@@ -249,22 +259,40 @@ module Fp
       { results: results, unmatched: unmatched, missing_surface: missing_surface.uniq }
     end
 
+    # The loose ends in a set of unmatched testcases: the ones that *failed* but carry no
+    # fp:<slug> marker. Pass the `unmatched` list from `ci_results` (or `bind_markers`).
+    #
+    # Only failures are returned. A green unmarked test is fine — not everything needs a
+    # requirement — but a red one is a failure nobody is tracking, which is exactly the
+    # signal a human should see. Skipped tests are not failures, so they're excluded too.
+    def unmarked_failures(unmatched)
+      unmatched.select(&:failed?).map do |tc|
+        UnmarkedFailure.new(name: tc.name, classname: tc.classname, file: tc.file, line: tc.line)
+      end
+    end
+
     # Walk testcases and yield a Match for every one we can tie to a marker.
     # Shared by bind_markers (agent evidence) and ci_results (CI verdicts) so the two
     # can never disagree about which test a marker annotates.
     def walk_matches(testcases, base_dir)
       marker_cache = {}
+      prefix_cache = {}
 
       # Group testcases by their source file so we can reason about ordering.
-      testcases.group_by(&:file).each do |_file, cases|
+      testcases.group_by(&:file).each do |file, cases|
         markers = markers_for(cases.first, base_dir, marker_cache)
         next if markers.empty?
 
+        # The path we'll store as evidence: relative to the git root, not to base_dir,
+        # so a GitHub blob link resolves. base_dir's offset below the root is cached, so
+        # this is one shell-out per run, not per file.
+        repo_file = repo_relative_file(file, base_dir, prefix_cache)
+
         cases.each do |tc|
           marker, precise = marker_for(tc, cases, markers)
           next unless marker
 
-          yield Match.new(testcase: tc, marker: marker, precise: precise)
+          yield Match.new(testcase: tc, marker: marker, precise: precise, repo_file: repo_file)
         end
       end
     end
@@ -343,8 +371,8 @@ module Fp
       candidate
     end
 
-    def record_binding(bindings, slug, surface, testcase, marker, titles_reliable: true)
-      rel_file = testcase.file
+    def record_binding(bindings, slug, surface, testcase, marker, repo_file, titles_reliable: true)
+      rel_file = repo_file || testcase.file
       key = [slug, surface, rel_file]
       state = testcase.stub? ? 'stub' : 'present'
 
@@ -380,6 +408,70 @@ module Fp
       cache[file] = markers
     end
 
+    # Rewrite a report-relative test-file path so it is relative to the git repo root.
+    #
+    # Runners report paths relative to wherever they ran: RSpec/minitest from the repo
+    # root, but vitest from its own package (frontend/), which is why the web job passes
+    # `--base-dir frontend`. That base_dir is enough to *find* the markers, but the path
+    # we then store as `source_file` was the raw report path (`src/pages/Matrix.test.jsx`),
+    # not the repo-root path (`frontend/src/pages/Matrix.test.jsx`). The web app builds a
+    # GitHub link as `blob/<sha>/<source_file>`, and a blob path is always relative to the
+    # git root — so the missing `frontend/` prefix produced a 404 on every web test link.
+    #
+    # The fix is purely additive: find how far base_dir sits below the git root and
+    # prepend that. When base_dir *is* the git root (the Ruby suites, run from the repo
+    # root), the prefix is empty and the original path is returned byte-for-byte — so this
+    # only ever corrects a subdirectory base_dir and never rewrites a path that was
+    # already right. Falls back to the raw path when there's no git checkout (an agent
+    # running from a tarball) or base_dir sits outside one.
+    def repo_relative_file(file, base_dir, cache = {})
+      return file if file.nil? || file.empty?
+
+      prefix = base_dir_prefix(base_dir, cache)
+      return file if prefix.nil? || prefix.empty?
+
+      File.join(prefix, file)
+    end
+
+    # base_dir's path relative to the git root that contains it: '' when base_dir is the
+    # root, 'frontend' when it's a package below it, nil when base_dir isn't in a checkout
+    # (or sits above the root). Cached per base_dir so a report shells out once.
+    def base_dir_prefix(base_dir, cache)
+      absolute = File.expand_path(base_dir)
+      return cache[absolute] if cache.key?(absolute)
+
+      root = git_toplevel_for(absolute)
+      prefix =
+        if root.nil?
+          nil
+        else
+          rel = relative_path(absolute, root)
+          rel == '.' || rel.start_with?('..') ? '' : rel
+        end
+      cache[absolute] = prefix
+    end
+
+    # The git repo root containing `dir`, or nil if `dir` isn't inside a checkout.
+    def git_toplevel_for(dir)
+      out = `git -C #{quote(dir)} rev-parse --show-toplevel 2>/dev/null`
+      root = out.strip
+      ($?&.success? && !root.empty?) ? File.expand_path(root) : nil
+    end
+
+    def relative_path(absolute, root)
+      require 'pathname'
+      Pathname.new(absolute).relative_path_from(Pathname.new(root)).to_s
+    rescue ArgumentError
+      # Different roots on Windows, or otherwise not relative — treat as outside.
+      '..'
+    end
+
+    # Minimal shell-quote for a path passed to `git -C`. The directory comes from
+    # --base-dir, not arbitrary report content, but quoting keeps spaces safe.
+    def quote(str)
+      "'#{str.gsub("'", "'\\\\''")}'"
+    end
+
     # Scan file contents for fp:<slug>[@surface[,surface...]] markers, returning
     # [{ slug:, surfaces: [..], line:, title:, title_kind: }, ...] (1-based line numbers).
     # `surfaces` is [] when the marker pins none (use the default surface).

data/lib/fp/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Fp
-  VERSION = '0.0.12'
+  VERSION = '0.0.14'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: featureparity
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.14
 platform: ruby
 authors:
 - Stowzilla