featureparity 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20c34d118adff460a3de1eaaa15f66718593100ae0261f9dc064f63b65d8b3f2
-  data.tar.gz: e14ab1bf7bbf6c646f42e0b9a9291653e072f47055b8dff8ccf2294d10effa7a
+  metadata.gz: e4c131ba204a5f2e3b872551963af559ee1cc2ba941eb192228cd261a12c6f48
+  data.tar.gz: 4919cdca3946beec35921271be6998938150085879a3b6c9ed4d8cd4c84aa622
 SHA512:
-  metadata.gz: 248bf8d80901ed3a4075e44e501169bd37803dc4fad78b3b342cfd79307a4649b824664a0b7ce5013f3fd323b2f64ed7deaa2318813389604e21d8c212cd2c36
-  data.tar.gz: 1e22a8d5b38f77581693dd4cb0d912aedb0d09afa8cdffaa96332eca3afc1d9c62893ff2c0a09e9ca7f4d44d46fd6126d71b915cb3bfa357099c2a3726ce2006
+  metadata.gz: 8977bbdb59e48dd3d19878858ebc0e5363ec9be1313b8352054f8227e9a84a99c8b3e7343ae74cb034bc96f2f45fc5700656a5218f46db61abcaa5a32f9fa146
+  data.tar.gz: e37abf6853447467b7b91937f26447d4e2c7b4de5d52bcc3b17ac6c348ce01fbd996e806e0a10add88f04b8c4823af292302a81b6cb81b7eae5b3cb306846caf

data/lib/fp/client.rb

@@ -144,7 +144,14 @@ module Fp
     # writes passing/failing evidence per surface. It also marks previously
     # passing slugs that have vanished from the results as failing. This is the
     # half of the loop only CI is allowed to close (#1222).
-    def ingest_ci_results(project_id, ci_url:, surface:, junit_xml:, pr_url: nil)
+    # POST /projects/:project_id/ci_results — CI's real pass/fail.
+    #
+    # `results` are the run's outcomes already bound to their fp:<slug> markers by the
+    # CLI, in the checkout CI is holding. The raw `junit_xml` still goes along so the
+    # server can fall back to resolving markers itself for older clients — but it can
+    # only do that by fetching the test files from GitHub, which fails silently on a
+    # private repo, so resolved results are always sent when we have them.
+    def ingest_ci_results(project_id, ci_url:, surface:, junit_xml:, pr_url: nil, results: nil)
       body = {
         ci_url: ci_url,
         surface: surface,
@@ -152,6 +159,7 @@ module Fp
       }
       # Only send pr_url when set, so a merged-branch run stays a plain ingest.
       body[:pr_url] = pr_url if pr_url && !pr_url.empty?
+      body[:results] = results if results && !results.empty?
 
       post("/api/projects/#{project_id}/ci_results", body)
     end
@@ -180,8 +188,13 @@ module Fp
     # what has already been recorded, which on a PR is the default branch's state; this
     # is how a branch gets gated on its own results without writing evidence that would
     # churn the live matrix.
-    def preview_parity(project_id, surface:, junit_xmls:, level: nil)
+    #
+    # `results` are the run's outcomes already bound to their markers locally (see
+    # ingest_ci_results). Sending them is what lets a surface whose repo isn't registered
+    # — or a private one the server can't read — be previewed at all.
+    def preview_parity(project_id, surface:, junit_xmls:, level: nil, results: nil)
       body = { surface: surface, junit_xmls: Array(junit_xmls) }
+      body[:results] = results if results && !results.empty?
       body[:level] = level if level
       post("/api/projects/#{project_id}/parity/preview", body)
     end

data/lib/fp/commands/check.rb

@@ -41,6 +41,7 @@ module Fp
         surface: :string,
         level: :string,
         junit: :string,
+        base_dir: :string,
         warn_only: :boolean
       }.freeze
 
@@ -62,7 +63,7 @@ module Fp
           if junit_paths.empty?
             client.get_parity(project_id, level: level, surface: surface)
           else
-            preview(project_id, junit_paths, surface, level)
+            preview(project_id, junit_paths, surface, level, opts)
           end
 
         unless result[:ok]
@@ -86,7 +87,14 @@ module Fp
       # Preview mode needs to know which surface the results belong to, because the
       # server uses it to decide which cells the run had a chance to cover — and
       # therefore which vanished tests it may legitimately flag.
-      def preview(project_id, junit_paths, surface, level)
+      #
+      # The markers are resolved here, in the checkout, and sent with the request. The
+      # server can also resolve them itself by fetching the test files from GitHub, but
+      # that only works for a surface with a registered, readable repo — and it reads the
+      # *default branch*, which by definition can't see the tests the branch under review
+      # just added. Resolving locally is both more accurate and the only option for a
+      # private repo.
+      def preview(project_id, junit_paths, surface, level, opts)
         unless surface
           output.error(
             '--surface is required with --junit (the surface these results are for). ' \
@@ -105,10 +113,46 @@ module Fp
           project_id,
           surface: surface,
           junit_xmls: junit_paths.map { |p| File.read(p) },
-          level: level
+          level: level,
+          results: resolve_results(junit_paths, surface, opts)
         )
       end
 
+      # Bind every report's testcases to their fp markers locally. Returns nil (rather
+      # than an empty array) when nothing resolved, so the server falls back to its own
+      # resolution instead of reading "no results" as "every test vanished".
+      def resolve_results(junit_paths, surface, opts)
+        base_dir = opts[:base_dir] || Dir.pwd
+
+        results = junit_paths.flat_map do |path|
+          testcases = Fp::JUnit.parse_file(path)
+          Fp::JUnit.ci_results(testcases, base_dir: base_dir, default_surface: surface)[:results]
+        end
+
+        if results.empty?
+          # Stderr, not output.error: this is advisory and must not corrupt --json's stdout.
+          warn "Warning: no fp:<slug> markers resolved from #{junit_paths.join(', ')} " \
+               "(base dir: #{base_dir}). Falling back to the server's own marker resolution, " \
+               'which can only read a registered public repo. Check --base-dir.'
+          return nil
+        end
+
+        results.map do |r|
+          {
+            slug: r.slug,
+            surface: r.surface,
+            status: r.status,
+            source_file: r.file,
+            source_line: r.line,
+            test_count: r.test_count,
+            failure_count: r.failure_count
+          }
+        end
+      rescue ArgumentError => e
+        output.error(e.message)
+        exit 1
+      end
+
       def render(report, violations, passed, project_slug, opts)
         payload = report.merge('project' => project_slug, 'warn_only' => !!opts[:warn_only])
 

data/lib/fp/commands/ci_report.rb

@@ -1,31 +1,46 @@
 # frozen_string_literal: true
 
+require 'set'
+
 module Fp
   module Commands
     # CI-only evidence reporting from a JUnit XML report.
     #
     #   fp ci-report --junit <path> --project <slug> --surface X --ci-url URL
     #
-    # This is the CI counterpart to `fp report --junit`. Where `fp report`
-    # writes agent evidence (present/stub) and never claims a test passed, this
-    # command hands the raw JUnit XML to the backend's CI ingest endpoint, which:
+    # This is the CI counterpart to `fp report --junit`. Where `fp report` writes agent
+    # evidence (present/stub) and never claims a test passed, this command resolves the
+    # run's results against the fp:<slug> markers in the checked-out test files and
+    # uploads the real verdict per requirement:
+    #
+    #   - each testcase is bound to the marker that annotates it (same convention and
+    #     the same code path as `fp report --junit`),
+    #   - passing / failing evidence is written per requirement for the surface,
+    #   - and any slug that was passing but has now vanished from the report is marked
+    #     failing.
     #
-    #   - maps each testcase to its fp:<slug> marker (server-side, same
-    #     file-scanning convention as the CLI),
-    #   - writes passing / failing evidence per requirement for the surface,
-    #   - and marks any slug that was passing but has now vanished from the
-    #     report as failing.
+    # Resolution happens here rather than server-side on purpose. The backend used to
+    # re-fetch the test files from GitHub to find the markers itself, which cannot work
+    # for a private repo (unauthenticated, the API answers 404): every run resolved zero
+    # markers and reported nothing at all, with no error. CI already has the branch
+    # checked out — including tests the branch just added, which the default branch's
+    # tree would not have shown. The raw XML still goes up alongside the results so the
+    # server can fall back to its own resolution for older clients.
     #
-    # That last part is why the whole report goes up as one document rather than
-    # per-slug: the server can only detect a disappeared test by diffing the full
-    # result set against what it had. Reporting passing/failing is a privilege of
-    # CI (#1222) — agents can't reach this path, they only ever report evidence
-    # via `fp report`.
+    # That last part of the ingest is why the whole report goes up as one document rather
+    # than per-slug: the server can only detect a disappeared test by diffing the full
+    # result set against what it had. Reporting passing/failing is a privilege of CI
+    # (#1222) — agents can't reach this path, they only ever report evidence via
+    # `fp report`.
     #
     # Multiple --junit flags may be passed to ingest several reports for the same
     # surface in one invocation (e.g. a suite split across shards). Each is
     # uploaded in turn.
     #
+    # --base-dir tells the CLI where the report's relative test-file paths resolve from
+    # (vitest, for instance, reports paths relative to the frontend package, not the repo
+    # root). Defaults to the working directory.
+    #
     # --pr <url> marks the run as a pull request's CI rather than the merged branch's.
     # Passing tests are then recorded as `pr_passing` ("would pass once merged") instead
     # of `passing`, and no merged evidence is demoted. Run it without --pr on pushes to
@@ -37,7 +52,9 @@ module Fp
         surface: :string,
         ci_url: :string,
         pr: :string,
-        junit: :string
+        junit: :string,
+        base_dir: :string,
+        allow_empty: :boolean
       }.freeze
 
       def run(args)
@@ -62,17 +79,27 @@ module Fp
         end
 
         project_id = resolve_project_id(opts[:project])
+        known = known_slugs(project_id)
 
-        totals = { created: 0, updated: 0, missing_failing: 0, reports: [] }
+        totals = { created: 0, updated: 0, missing_failing: 0, reports: [], resolved: 0,
+                   unmatched: 0, unknown: [] }
 
         junit_paths.each do |path|
           xml = File.read(path)
+          resolved = resolve_results(path, opts)
+          totals[:unmatched] += resolved[:unmatched]
+
+          results, unknown = partition_known(resolved[:results], known)
+          totals[:resolved] += results.length
+          totals[:unknown].concat(unknown)
+
           result = client.ingest_ci_results(
             project_id,
             ci_url: opts[:ci_url],
             surface: opts[:surface],
             junit_xml: xml,
-            pr_url: opts[:pr]
+            pr_url: opts[:pr],
+            results: results
           )
 
           unless result[:ok]
@@ -87,14 +114,101 @@ module Fp
           totals[:reports] << { path: path, results: data['results'] || [] }
         end
 
+        # Reporting nothing is the failure this command exists to make visible: a run that
+        # resolves no markers has told the matrix nothing, and used to do so silently.
+        if totals[:resolved].zero? && !opts[:allow_empty]
+          if totals[:unknown].empty?
+            output.error('No fp:<slug> markers resolved from the report(s), so no evidence was uploaded.')
+            output.error("Check that --base-dir (#{opts[:base_dir] || Dir.pwd}) is where the report's " \
+                         'test-file paths resolve from, and that the tests carry fp:<slug> markers.')
+          else
+            output.error('None of the fp:<slug> markers in the report(s) match a requirement in ' \
+                         "this project, so no evidence was uploaded: #{totals[:unknown].uniq.join(', ')}")
+            output.error('Propose the requirement (fp propose) or fix the slug in the marker.')
+          end
+          output.error('Pass --allow-empty if a suite with no markers is expected here.')
+          exit 1
+        end
+
+        # Resolved markers but nothing recorded: the API accepted the report and stored
+        # none of it — a server that predates resolved results, or slugs/surfaces it
+        # dropped. Not the author's fault, so don't fail their build, but say it loudly
+        # rather than printing "created: 0" as though that were a result.
+        if totals[:resolved].positive? && (totals[:created] + totals[:updated]).zero?
+          annotate("fp ci-report resolved #{totals[:resolved]} marker(s) but the API recorded " \
+                   "none of them for surface '#{opts[:surface]}'. The parity matrix has not " \
+                   'been updated by this run.')
+        end
+
         emit_summary(opts, totals)
       end
 
       private
 
-      # Pull every occurrence of a repeatable flag (and its value) out of args,
-      # returning [collected_values, remaining_args]. Leaves all other flags for
-      # the normal parser.
+      # Every slug this project has a requirement for, in one call, so a marker that
+      # matches nothing can be skipped with a warning instead of filing evidence against a
+      # requirement that doesn't exist. (A few test suites legitimately contain
+      # marker-shaped *fixtures* — the tests for the marker feature itself.) Drafts count:
+      # a requirement awaiting review still collects evidence.
+      #
+      # Returns nil when the list can't be fetched, which means "don't filter" — losing a
+      # run's evidence to a hiccup on this call would be worse than the pollution.
+      def known_slugs(project_id)
+        result = client.list_requirements(project_id: project_id)
+        unless result[:ok]
+          warn "Warning: could not list requirements (#{result[:error]}); reporting every " \
+               'resolved marker without checking it against a requirement.'
+          return nil
+        end
+
+        (result[:data]['requirements'] || []).map { |r| r['slug'] }.compact.to_set
+      end
+
+      def partition_known(results, known)
+        return [results, []] if known.nil?
+
+        kept, unknown = results.partition { |r| known.include?(r[:slug]) }
+        [kept, unknown.map { |r| r[:slug] }.uniq]
+      end
+
+      # A warning that survives a CI log. Stderr always; a GitHub Actions annotation too
+      # when we're running in one, so it shows up in the run summary rather than scrolling
+      # past in a collapsed step.
+      def annotate(message)
+        warn "Warning: #{message}"
+        puts "::warning::#{message}" if ENV['GITHUB_ACTIONS'] == 'true'
+      end
+
+      # Bind this report's testcases to their fp markers, locally, in the checkout.
+      def resolve_results(path, opts)
+        testcases = Fp::JUnit.parse_file(path)
+        resolved = Fp::JUnit.ci_results(
+          testcases,
+          base_dir: opts[:base_dir] || Dir.pwd,
+          default_surface: opts[:surface]
+        )
+
+        {
+          results: resolved[:results].map do |r|
+            {
+              slug: r.slug,
+              surface: r.surface,
+              status: r.status,
+              source_file: r.file,
+              source_line: r.line,
+              test_count: r.test_count,
+              failure_count: r.failure_count
+            }
+          end,
+          unmatched: resolved[:unmatched].length
+        }
+      rescue ArgumentError => e
+        output.error(e.message)
+        exit 1
+      end
+
+      # Pull the human-readable summary together. `markers_resolved` is worth printing
+      # even when everything worked: it's the number that used to silently be zero.
       def emit_summary(opts, totals)
         data = {
           project: opts[:project],
@@ -104,6 +218,9 @@ module Fp
           evidence_created: totals[:created],
           evidence_updated: totals[:updated],
           missing_marked_failing: totals[:missing_failing],
+          markers_resolved: totals[:resolved],
+          unmatched_testcases: totals[:unmatched],
+          skipped_unknown: totals[:unknown].uniq,
           reports: totals[:reports]
         }
 
@@ -116,6 +233,12 @@ module Fp
             end
           end
           puts
+          puts "  resolved: #{totals[:resolved]} marker(s) from the report(s); " \
+               "#{totals[:unmatched]} testcase(s) carried none."
+          unless totals[:unknown].empty?
+            puts "  skipped #{totals[:unknown].uniq.length} marker(s) with no matching " \
+                 "requirement: #{totals[:unknown].uniq.join(', ')}"
+          end
           puts "  created: #{totals[:created]}   updated: #{totals[:updated]}   marked failing (missing): #{totals[:missing_failing]}"
         end
       end

data/lib/fp/commands/help.rb

@@ -108,6 +108,10 @@ module Fp
             fp check --warn-only                            # report violations but exit 0
             fp check --json                                 # machine-readable violations
 
+            # Gate a branch on its own results (nothing is saved). One call per surface.
+            fp check --surface api --junit junit.xml
+            fp check --surface web --junit web-junit.xml --base-dir frontend
+
             # Show the resolved configuration (which profile/env/API URL/key is in effect)
             fp whoami
             fp whoami --json
@@ -223,6 +227,11 @@ module Fp
               --surface api \\
               --ci-url "$CI_RUN_URL"
             # Pass --junit more than once to ingest several reports for one surface.
+            # Markers are resolved from the test files in the checkout, so run this from
+            # the repo (or pass --base-dir DIR if the report's paths are relative to a
+            # subdirectory, e.g. --base-dir frontend for vitest).
+            # A report that resolves no markers is an error, not a silent no-op; pass
+            # --allow-empty when a suite genuinely has none.
 
             # Show the parity matrix
             fp matrix --project stowzilla

data/lib/fp/commands/init.rb

@@ -246,6 +246,10 @@ module Fp
             # real passing/failing evidence — without it, the matrix only ever shows the
             # `present` markers agents filed, and the gate has nothing green to see.
             #
+            # It reads the fp:<slug> markers out of the checkout, so run it in the repo
+            # and pass --base-dir when the report's test-file paths are relative to a
+            # subdirectory (vitest, for instance, reports them relative to its package).
+            #
             # report:
             #   name: Report evidence
             #   if: github.event_name == 'push'

data/lib/fp/junit.rb

@@ -12,14 +12,35 @@ module Fp
   # evidence for each requirement the suite covered.
   module JUnit
     # A single parsed testcase.
-    TestCase = Struct.new(:name, :classname, :file, :line, :status, keyword_init: true) do
+    #
+    # `status` is the *agent* view (present/skipped) — what `fp report` files. `failed`
+    # is the runner's own verdict, which only CI is allowed to act on (`fp ci-report`).
+    # Both are kept because they answer different questions: a red test still proves the
+    # test exists (present), it just doesn't prove the requirement is met.
+    TestCase = Struct.new(:name, :classname, :file, :line, :status, :failed, keyword_init: true) do
       # Agents only ever report present/stub — never passing/failing (that's CI).
       # A skipped test is treated as a stub; anything else counts as present.
       def stub?
         status == :skipped
       end
+
+      def failed?
+        !!failed
+      end
     end
 
+    # One requirement's aggregated CI outcome for one surface, ready to hand to the
+    # backend's ingest endpoint. This is what lets CI upload *resolved* results rather
+    # than raw XML the server would have to re-scrape the repo to understand.
+    CiResult = Struct.new(:slug, :surface, :status, :file, :line, :test_count, :failure_count,
+                          keyword_init: true)
+
+    # A testcase paired with the marker that annotates it. `precise` records whether we
+    # know *which* test the marker sits above (matched by line or by test name) or only
+    # that the marker is somewhere in the same file — a title is only worth reporting in
+    # the first case.
+    Match = Struct.new(:testcase, :marker, :precise, keyword_init: true)
+
     # Evidence discovered by binding a testcase to a marker.
     # `line` is the 1-based line number of the test in its source file: the
     # testcase's own line when the runner reported one, otherwise the line of
@@ -75,7 +96,8 @@ module Fp
           classname: classname,
           file: resolve_file(el.attributes['file'], classname, suite_name),
           line: (el.attributes['line'] && el.attributes['line'].to_i),
-          status: testcase_status(el)
+          status: testcase_status(el),
+          failed: testcase_failed?(el)
         )
       end
 
@@ -111,6 +133,12 @@ module Fp
       :present
     end
 
+    # Did the runner report this testcase as red? Only CI acts on this (`fp ci-report`);
+    # agent evidence deliberately ignores it.
+    def testcase_failed?(el)
+      el.get_elements('failure').any? || el.get_elements('error').any?
+    end
+
     # Bind parsed testcases to fp:<slug> markers found in their source files.
     #
     # A marker binds to the test it annotates — the testcase whose line is the
@@ -139,57 +167,165 @@ module Fp
     #     missing_surface: [Binding-ish], # markers with no surface and no default
     #   }
     def bind_markers(testcases, base_dir: Dir.pwd, default_surface: nil)
-      marker_cache = {}
       bindings = {}
       matched = {}
       missing_surface = []
 
-      # Group testcases by their source file so we can reason about ordering.
-      by_file = testcases.group_by(&:file)
+      walk_matches(testcases, base_dir) do |match|
+        tc = match.testcase
+        marker = match.marker
+        matched[tc.object_id] = true
 
-      by_file.each do |file, cases|
-        markers = markers_for(cases.first, base_dir, marker_cache)
-        next if markers.empty?
+        surfaces = marker[:surfaces]
+        surfaces = [default_surface] if surfaces.empty?
 
-        # A title is only trustworthy when we know which test the marker annotates: either
-        # the report carried line numbers, or the file holds a single test. Otherwise the
-        # marker was matched file-wide and naming any one test would be a guess — vitest
-        # and jest emit no line numbers, so this is the common case, not an edge one.
-        titles_reliable = cases.any?(&:line) || cases.length == 1
+        surfaces.each do |surface|
+          if surface.nil? || surface.empty?
+            missing_surface << { slug: marker[:slug], file: tc.file }
+          else
+            record_binding(bindings, marker[:slug], surface, tc, marker, titles_reliable: match.precise)
+          end
+        end
+      end
 
-        cases.each do |tc|
-          marker = marker_for(tc, cases, markers)
-          next unless marker
+      unmatched = testcases.reject { |tc| matched[tc.object_id] }
+      { bindings: bindings.values, unmatched: unmatched, missing_surface: missing_surface.uniq }
+    end
+
+    # Aggregate a run's results per (slug, surface) into the pass/fail verdict CI files.
+    #
+    # This is `fp ci-report`'s half of the marker convention. The backend used to
+    # re-resolve markers itself by fetching the test files from GitHub, which cannot work
+    # for a private repo (unauthenticated, the API answers 404) and silently resolved
+    # nothing — the run reported no evidence at all. CI already has the branch checked
+    # out, so the resolution happens here and the result is uploaded.
+    #
+    # A slug is `failing` if any test claiming it went red, `stub` if every test claiming
+    # it was skipped (a skipped test is not proof of anything), and `passing` otherwise.
+    #
+    # Returns { results: [CiResult, ...], unmatched: [TestCase], missing_surface: [...] }.
+    def ci_results(testcases, base_dir: Dir.pwd, default_surface: nil)
+      aggregates = {}
+      matched = {}
+      missing_surface = []
+
+      walk_matches(testcases, base_dir) do |match|
+        tc = match.testcase
+        marker = match.marker
+        matched[tc.object_id] = true
 
-          matched[tc.object_id] = true
+        surfaces = marker[:surfaces]
+        surfaces = [default_surface] if surfaces.empty?
 
-          surfaces = marker[:surfaces]
-          surfaces = [default_surface] if surfaces.empty?
+        surfaces.each do |surface|
+          if surface.nil? || surface.empty?
+            missing_surface << { slug: marker[:slug], file: tc.file }
+            next
+          end
 
-          surfaces.each do |surface|
-            if surface.nil? || surface.empty?
-              missing_surface << { slug: marker[:slug], file: tc.file }
-            else
-              record_binding(bindings, marker[:slug], surface, tc, marker, titles_reliable: titles_reliable)
-            end
+          key = [marker[:slug], surface]
+          agg = aggregates[key] ||= { file: tc.file, line: tc.line || marker[:line], ran: 0, failed: 0, skipped: 0 }
+          if tc.stub?
+            agg[:skipped] += 1
+          else
+            agg[:ran] += 1
+            agg[:failed] += 1 if tc.failed?
           end
         end
       end
 
+      results = aggregates.map do |(slug, surface), agg|
+        status =
+          if agg[:failed].positive? then 'failing'
+          elsif agg[:ran].zero? then 'stub'
+          else 'passing'
+          end
+
+        CiResult.new(slug: slug, surface: surface, status: status, file: agg[:file], line: agg[:line],
+                     test_count: agg[:ran], failure_count: agg[:failed])
+      end
+
       unmatched = testcases.reject { |tc| matched[tc.object_id] }
-      { bindings: bindings.values, unmatched: unmatched, missing_surface: missing_surface.uniq }
+      { results: results, unmatched: unmatched, missing_surface: missing_surface.uniq }
     end
 
-    # Determine the marker (slug + surfaces) that annotates a testcase, or nil.
+    # Walk testcases and yield a Match for every one we can tie to a marker.
+    # Shared by bind_markers (agent evidence) and ci_results (CI verdicts) so the two
+    # can never disagree about which test a marker annotates.
+    def walk_matches(testcases, base_dir)
+      marker_cache = {}
+
+      # Group testcases by their source file so we can reason about ordering.
+      testcases.group_by(&:file).each do |_file, cases|
+        markers = markers_for(cases.first, base_dir, marker_cache)
+        next if markers.empty?
+
+        cases.each do |tc|
+          marker, precise = marker_for(tc, cases, markers)
+          next unless marker
+
+          yield Match.new(testcase: tc, marker: marker, precise: precise)
+        end
+      end
+    end
+
+    # Determine the marker (slug + surfaces) that annotates a testcase.
+    # Returns [marker, precise] — or [nil, false] when nothing can be tied to it.
     def marker_for(testcase, sibling_cases, markers)
       if testcase.line
-        marker_for_line(testcase.line, sibling_cases, markers)
+        [marker_for_line(testcase.line, sibling_cases, markers), true]
+      elsif (by_title = marker_for_title(testcase, markers))
+        # No line numbers (vitest, jest): match the marker to the test whose declared
+        # name follows it in the file. This is the common case for the web surface, and
+        # without it any file with more than one marker resolves to nothing at all.
+        [by_title, true]
       elsif markers.length == 1 && sibling_cases.none?(&:line)
-        # No line info anywhere and a single marker: unambiguous.
-        markers.first
+        # No line info anywhere and a single marker: unambiguous, but we can't say which
+        # test it annotates, so a title would be a guess.
+        [markers.first, sibling_cases.length == 1]
+      else
+        [nil, false]
       end
     end
 
+    # Match a testcase to a marker by the name of the test declared beneath the marker.
+    #
+    # Runners that omit line numbers still report the test's name, and vitest/jest qualify
+    # it with the enclosing describe blocks ("Group > Subgroup > test name"). So a marker
+    # above a test matches the last segment, and one above a `describe` matches any
+    # segment before it. A test-level match wins over a group-level one, so a marker on a
+    # single test inside an already-marked group still binds to just that test.
+    def marker_for_title(testcase, markers)
+      name = testcase.name.to_s.strip
+      return nil if name.empty?
+
+      segments = name.split(NAME_SEPARATOR).map(&:strip)
+
+      by_test = markers.select { |m| m[:title_kind] == :test && title_matches_test?(m[:title], name, segments) }
+      return best_by_title(by_test) unless by_test.empty?
+
+      by_group = markers.select do |m|
+        m[:title_kind] == :group && !m[:title].to_s.empty? && segments[0..-2].include?(m[:title])
+      end
+      best_by_title(by_group)
+    end
+
+    # The most specific match wins (longest title), and among equally good matches the
+    # lowest marker in the file — the same "nearest marker above the test" rule the
+    # line-based path applies when markers are stacked.
+    def best_by_title(markers)
+      markers.max_by { |m| [m[:title].length, m[:line]] }
+    end
+
+    # vitest/jest join describe names and the test name with " > ".
+    NAME_SEPARATOR = / > /.freeze
+
+    def title_matches_test?(title, name, segments)
+      return false if title.nil? || title.empty?
+
+      segments.last == title || name == title || name.end_with?(title)
+    end
+
     # A marker annotates the testcase that is the first test below it. Given a
     # testcase line, find the marker that sits directly above it with no other
     # testcase in between.
@@ -245,21 +381,69 @@ module Fp
     end
 
     # Scan file contents for fp:<slug>[@surface[,surface...]] markers, returning
-    # [{ slug:, surfaces: [..], line: }, ...] (1-based line numbers).
+    # [{ slug:, surfaces: [..], line:, title:, title_kind: }, ...] (1-based line numbers).
     # `surfaces` is [] when the marker pins none (use the default surface).
     #
+    # `title` is the name of the test (or describe block) declared below the marker, used
+    # to bind markers to results from runners that report no line numbers. It is nil when
+    # no declaration we recognise follows the marker.
+    #
     # Only comment lines are considered — see COMMENT_LINE_RE.
     def scan_markers(contents)
+      lines = contents.lines
       markers = []
-      contents.each_line.with_index(1) do |line, num|
+      lines.each_with_index do |line, idx|
         next unless line.match?(COMMENT_LINE_RE)
 
         line.scan(MARKER_RE) do |(slug, surface_list)|
           surfaces = surface_list ? surface_list.split(',').map(&:strip).reject(&:empty?) : []
-          markers << { slug: slug, surfaces: surfaces, line: num }
+          title, kind = declared_test_after(lines, idx)
+          markers << { slug: slug, surfaces: surfaces, line: idx + 1, title: title, title_kind: kind }
         end
       end
       markers
     end
+
+    # How a test (or group) declaration announces its name, per language. Each entry is
+    # [regex, kind]; the captured group is the name. `:group` entries are describe-style
+    # blocks, whose name prefixes the names of the tests inside them.
+    TEST_DECL_PATTERNS = [
+      [/\b(?:it|test|bench)(?:\.\w+)*\s*\(\s*(['"`])(.+?)\1/, :test],   # vitest/jest/mocha
+      [/\bdescribe(?:\.\w+)*\s*\(\s*(['"`])(.+?)\1/, :group],           # vitest/jest/mocha group
+      [/\b(?:it|specify|scenario)\s+(['"])(.+?)\1/, :test],             # rspec
+      [/\b(?:def|fun)\s+`(.+?)`/, :test],                               # kotlin backticked
+      [/\b(?:def|fun|func)\s+(test_?\w+)/i, :test]                      # minitest / xctest / pytest
+    ].freeze
+
+    # The name of the first test declaration at or below `idx`, as [title, kind].
+    # Intervening comment and blank lines are skipped (a marker is often followed by more
+    # comments), but the search gives up at the first line of real code that declares
+    # nothing recognisable, so a marker can't borrow the name of an unrelated test
+    # further down the file.
+    def declared_test_after(lines, idx)
+      ((idx + 1)...[idx + 1 + DECL_SEARCH_WINDOW, lines.length].min).each do |i|
+        line = lines[i]
+        next if line.strip.empty? || line.match?(COMMENT_LINE_RE)
+
+        TEST_DECL_PATTERNS.each do |re, kind|
+          match = line.match(re)
+          next unless match
+
+          # Two-quote patterns capture the quote first, the name second.
+          return [match.captures.last, kind]
+        end
+
+        # A line of real code that declares nothing recognisable: the marker doesn't
+        # annotate a test we can name.
+        return [nil, nil]
+      end
+
+      [nil, nil]
+    end
+
+    # How far below a marker to look for the declaration it annotates. The search stops at
+    # the first line of real code regardless, so this only bounds how long a comment block
+    # between the marker and its test may be.
+    DECL_SEARCH_WINDOW = 40
   end
 end

data/lib/fp/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Fp
-  VERSION = '0.0.9'
+  VERSION = '0.0.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: featureparity
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors:
 - Stowzilla