featureparity 0.0.27 → 0.0.29

data/lib/fp/commands/question.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Fp
+  module Commands
+    # Shared by `fp question` and `fp questions`: find the requirement, print a question.
+    module QuestionSupport
+      private
+
+      def find_requirement!(project_id, slug)
+        result = client.find_requirement_by_slug(project_id, slug)
+        unless result[:ok]
+          output.error(result[:error], status: result[:status])
+          exit 1
+        end
+        result[:data]['requirement']
+      end
+
+      def print_question(question)
+        asker = question['asked_by'] || 'unknown'
+        kind = question['asked_by_kind'] == 'agent' ? 'agent' : 'human'
+        puts "  [#{question['status']}] #{question['text']}"
+        puts "    asked by #{asker} (#{kind}) at #{question['asked_at']}"
+        case question['status']
+        when 'answered'
+          puts "    answer: #{question['answer']} (#{question['resolved_by']})"
+        when 'dismissed'
+          puts "    dismissed by #{question['resolved_by']}"
+        end
+      end
+    end
+
+    # `fp question <slug> "<text>"` — attach an open question for a human to the draft it
+    # concerns (Fizzy #1786).
+    #
+    # An agent writing a batch of drafts usually has questions only a human can answer:
+    # a limit the source never set, a role it didn't name. Left in the chat, they're
+    # detached from the drafts. Asked here, the human reviewing the draft in Quick Review
+    # sees them beside it, and answers or dismisses each one there. Agents can ask but
+    # not answer; the web app is where a human resolves them.
+    class Question < Base
+      include QuestionSupport
+
+      KNOWN_FLAGS = { project: :string, body: :string }.freeze
+
+      help_summary 'Attach an open question for a human to a draft requirement'
+      help_usage 'fp question <slug> "<question>" [--project <slug>]'
+      help_flag :project, 'Project slug (falls back to .featureparity.yml)'
+      help_flag :body, 'The question text (instead of the second argument)'
+      help_example 'fp question upload_limits "Is the 5 MB cap per file or per upload?"'
+      help_example 'fp questions upload_limits   # list open and resolved questions'
+
+      def run(args)
+        opts, positional = parse_flags(args, KNOWN_FLAGS)
+        require_project!(opts)
+
+        slug = positional.first
+        unless slug
+          output.error('Requirement slug is required. Usage: fp question <slug> "<question>" [--project <project>]')
+          exit 1
+        end
+
+        text = opts[:body] || positional[1..].join(' ')
+        if text.nil? || text.strip.empty?
+          output.error("A question needs its text. Usage: fp question #{slug} \"<question>\" " \
+                       "(run `fp questions #{slug}` to list them)")
+          exit 1
+        end
+
+        project_id = resolve_project_id(opts[:project])
+        requirement = find_requirement!(project_id, slug)
+
+        result = client.ask_requirement_question(requirement['id'], text.strip)
+        unless result[:ok]
+          output.error(result[:error], status: result[:status])
+          exit 1
+        end
+
+        output.success(result[:data]) do
+          open_count = result[:data]['open_count']
+          puts "Asked on '#{slug}': #{result[:data].dig('question', 'text')}"
+          puts
+          puts "#{open_count} open question#{'s' unless open_count == 1} on this requirement." if open_count
+          puts 'A human reviewing it in Quick Review sees the question and answers or dismisses it there.'
+          puts "Run `fp questions #{slug}` to read the answers."
+        end
+      end
+    end
+
+    # `fp questions <slug>` — the questions asked on a requirement, open and resolved.
+    class Questions < Base
+      include QuestionSupport
+
+      KNOWN_FLAGS = { project: :string, open: :boolean }.freeze
+
+      help_summary "List a requirement's questions for a human, with their answers"
+      help_usage 'fp questions <slug> [--open] [--project <slug>]'
+      help_flag :project, 'Project slug (falls back to .featureparity.yml)'
+      help_flag :open, 'Show only the questions still open'
+      help_example 'fp questions upload_limits'
+      help_example 'fp questions upload_limits --open --json'
+
+      def run(args)
+        opts, positional = parse_flags(args, KNOWN_FLAGS)
+        require_project!(opts)
+
+        slug = positional.first
+        unless slug
+          output.error('Requirement slug is required. Usage: fp questions <slug> [--project <project>]')
+          exit 1
+        end
+
+        project_id = resolve_project_id(opts[:project])
+        requirement = find_requirement!(project_id, slug)
+
+        result = client.list_requirement_questions(requirement['id'])
+        unless result[:ok]
+          output.error(result[:error], status: result[:status])
+          exit 1
+        end
+
+        questions = result[:data]['questions'] || []
+        questions = questions.select { |q| q['status'] == 'open' } if opts[:open]
+        payload = { 'requirement' => requirement, 'questions' => questions,
+                    'open_count' => result[:data]['open_count'] }
+
+        output.success(payload) do
+          puts "Questions on '#{slug}':"
+          if questions.empty?
+            puts opts[:open] ? '  (none open)' : '  (none asked)'
+          else
+            questions.each { |q| print_question(q) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fp/commands/report.rb

@@ -36,7 +36,7 @@ module Fp
       help_flag :ci_url, 'CI run URL'
       help_flag :title, 'Human-readable example/test name to display'
       help_flag :state, 'Evidence state: present or stub (default present)'
-      help_flag :junit, 'JUnit XML report to batch-upload evidence from'
+      help_flag :junit, 'JUnit XML report (or RSpec `--format json` output) to batch-upload evidence from'
       help_flag :base_dir, 'Where relative test-file paths in the report resolve from'
       help_flag :attach_media, 'With --junit: upload the videos/screenshots the run recorded ([[ATTACHMENT|…]]) as visual evidence'
       help_flag :stamp_all, 'With --junit: put --pr/--work-item on every receipt, not only on tests in files this branch changed'
@@ -124,6 +124,7 @@ module Fp
         params[:line] = opts[:line].to_i if opts[:line]
 
         # Try to report evidence
+        warn_if_not_production!
         result = client.report_evidence(params)
 
         # Evidence API may not exist yet (#1214)
@@ -208,7 +209,10 @@ module Fp
 
       # Batch-upload evidence from a JUnit XML report.
       def run_junit(opts)
-        require_project!(opts)
+        # The report's own checkout says which project it is: discovering from the
+        # report's directory (and up) means `fp report --junit hexhaven/junit.xml` run
+        # from a parent folder reads hexhaven's .featureparity.yml, not the parent's.
+        require_project!(opts, ProjectConfig.discover(File.dirname(File.expand_path(opts[:junit].to_s))))
         require_flag(opts, :repo)
         # --surface is the default for markers that don't pin their own surface
         # (e.g. `fp:slug@api,web`). It's only strictly required when at least one
@@ -244,6 +248,19 @@ module Fp
         result = Fp::JUnit.bind_markers(testcases, base_dir: base_dir,
                                         default_surface: opts[:surface],
                                         default_test_type: opts[:test_type])
+        # Run from outside the checkout, paths in the report resolve from nowhere. When
+        # nothing binds from here and no --base-dir was given, try the report's own
+        # directory before giving up.
+        report_dir = File.dirname(File.expand_path(opts[:junit].to_s))
+        if result[:bindings].empty? && opts[:base_dir].nil? && report_dir != File.expand_path(base_dir)
+          retried = Fp::JUnit.bind_markers(testcases, base_dir: report_dir,
+                                           default_surface: opts[:surface],
+                                           default_test_type: opts[:test_type])
+          if retried[:bindings].any?
+            base_dir = report_dir
+            result = retried
+          end
+        end
         bindings = result[:bindings]
         unmatched = result[:unmatched]
         missing_surface = result[:missing_surface] || []
@@ -272,6 +289,7 @@ module Fp
         # on requirements the branch never touched — "View Card" on an unrelated
         # requirement led to whichever card last ran the suite (Fizzy #1631). Only the
         # tests in files this branch changed get them; the rest keep their own.
+        warn_if_not_production!
         provenance = provenance_scope(opts, base_dir)
         reported = []
         skipped_unknown = []
@@ -488,7 +506,8 @@ module Fp
         stamping = !opts[:pr].nil? || !opts[:work_item].nil?
         data = {
           repo: opts[:repo],
-          reported: reported.map { |r| { slug: r[:binding].slug, surface: r[:binding].surface, test_type: r[:binding].test_type, file: r[:binding].file, line: r[:binding].line, state: r[:state], recorded: r[:recorded], stamped: (stamping ? r[:stamped] != false : nil), stored_status: r[:stored_status], superseded_by_ci: ci_superseded?(r[:state], r[:stored_status], r[:stored_source]) }.compact },
+          reported: reported.map { |r| { slug: r[:binding].slug, surface: r[:binding].surface, test_type: r[:binding].test_type, file: r[:binding].file, line: r[:binding].line, state: r[:state], recorded: r[:recorded], stamped: (stamping ? r[:stamped] != false : nil), stored_status: r[:stored_status], superseded_by_ci: ci_superseded?(r[:state], r[:stored_status], r[:stored_source]), tests: binding_tests(r[:binding]) }.compact },
+          tests_bound: reported.sum { |r| binding_tests(r[:binding]).length },
           provenance: (stamping ? { scoped: provenance.scoped?, mode: provenance.mode.to_s, stamped: reported.count { |r| r[:stamped] != false } } : nil),
           skipped_unknown: skipped_unknown.map { |b| { slug: b.slug, surface: b.surface, file: b.file } },
           unmatched_testcases: unmatched.map { |tc| { name: tc.name, file: tc.file } },
@@ -499,7 +518,8 @@ module Fp
         }
 
         output.success(data) do
-          puts "Reported evidence from #{opts[:junit]}:"
+          tests_bound = reported.sum { |r| binding_tests(r[:binding]).length }
+          puts "Reported evidence from #{opts[:junit]}: #{tests_bound} bound test(s) → #{reported.length} receipt(s)."
           reported.each do |r|
             superseded = ci_superseded?(r[:state], r[:stored_status], r[:stored_source])
             marker = if !r[:recorded]
@@ -514,6 +534,15 @@ module Fp
             shown_state = superseded ? "#{r[:stored_status]}, kept CI — #{r[:state]} ignored" : r[:state]
             stamp = stamping && provenance.scoped? && r[:stamped] ? '  ← this change' : ''
             puts "  #{marker} #{r[:binding].slug} → #{r[:binding].surface}#{type}  (#{shown_state})  #{loc}#{stamp}"
+            tests = binding_tests(r[:binding])
+            next if tests.length < 2
+
+            puts "      #{tests.length} tests share this receipt (one per requirement, surface, test type and file):"
+            tests.each do |t|
+              at = t[:line] ? ":#{t[:line]}" : ''
+              skipped = t[:state] == 'stub' ? '  (skipped)' : ''
+              puts "        - #{t[:name] || '(unnamed)'}  #{r[:binding].file}#{at}#{skipped}"
+            end
           end
 
           if stamping
@@ -573,6 +602,13 @@ module Fp
         end
       end
 
+      # The testcases that fed a receipt. Several tests proving one requirement is normal;
+      # the server keeps one row per cell, so the summary names each of them rather than
+      # leaving the reader to guess whether the rest were dropped.
+      def binding_tests(binding)
+        binding.tests || [{ name: binding.title, line: binding.line, state: binding.state }]
+      end
+
       # Warn when two or more distinct test files in this run resolved to the *same*
       # evidence cell. Server-side identity is (project, slug, surface, repo, test_type) —
       # the file isn't part of it — so those reports upsert the same row and the last one

data/lib/fp/commands/run.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+require 'fileutils'
+
+module Fp
+  module Commands
+    # fp run [report flags] -- <test command...>
+    #
+    # Runs the test suite with a report formatter fp adds itself, then reports it exactly
+    # as `fp report --junit <that report>` would (Fizzy #1786). Getting a report out of a
+    # suite usually meant adding a formatter gem or package to someone else's project
+    # (rspec_junit_formatter, say) before the first `fp report` could run; every runner
+    # here can write one with options it already has, so nothing in the project changes.
+    #
+    #   rspec       --format json --out <tmp> (+ --format progress when none was given)
+    #   pytest      -o junit_family=xunit1 --junitxml=<tmp>  (xunit1 carries file and line)
+    #   vitest      --reporter=default --reporter=junit --outputFile.junit=<tmp>
+    #   playwright  --reporter=list,junit with PLAYWRIGHT_JUNIT_OUTPUT_NAME=<tmp>
+    #
+    # It *is* `fp report`: same flags, same present/stub-only rule. The test command's
+    # own exit status wins, after reporting, so a red suite still fails the caller.
+    class Run < Base
+      help_summary 'Run a test command with a report formatter added, then fp report its results'
+      help_usage 'fp run [--surface X] [--repo org/repo] [fp report flags] -- <test command...>'
+      help_flag :surface, 'Surface the evidence is for (default for markers that pin none)'
+      help_flag :repo, 'Repository the tests live in (org/repo)', required: true
+      help_flag :project, 'Project slug (falls back to .featureparity.yml)'
+      help_flag :test_type, "Kind of test (project's vocabulary, e.g. unit, e2e)"
+      help_flag :work_item, 'Work-item URL that prompted the work'
+      help_flag :pr, 'Pull request URL'
+      help_flag :base_dir, 'Where relative test-file paths in the report resolve from'
+      help_example 'fp run --surface api --repo acme/shop -- bundle exec rspec'
+      help_example 'fp run --surface web --repo acme/shop -- npx vitest run'
+      help_example 'fp run --surface api --repo acme/shop -- pytest tests/'
+      help_example 'fp run --surface web --repo acme/shop -- npx playwright test'
+      help_note "Supports rspec, pytest, vitest and playwright. Any other runner: emit JUnit XML yourself\n" \
+                "and use `fp report --junit <file>`. Takes every `fp report` flag; agents still report\n" \
+                "present/stub only. Exits with the test command's status when it failed."
+
+      RUNNERS = %w[rspec pytest vitest playwright].freeze
+
+      def run(args)
+        split = args.index('--')
+        report_args = split ? args[0...split] : args
+        command = split ? args[(split + 1)..] : []
+
+        if command.empty?
+          output.error('Give the test command after `--`, e.g. fp run --surface api --repo org/repo -- bundle exec rspec')
+          exit 1
+        end
+        if report_args.include?('--junit') || report_args.any? { |a| a.start_with?('--junit=') }
+          output.error('fp run writes its own report; drop --junit (or use `fp report --junit <file>` instead).')
+          exit 1
+        end
+
+        runner = self.class.detect_runner(command)
+        unless runner
+          output.error("fp run doesn't recognise the runner in `#{command.join(' ')}`. " \
+                       "Supported: #{RUNNERS.join(', ')}. For anything else, make the runner write " \
+                       'JUnit XML and run `fp report --junit <file>`.')
+          exit 1
+        end
+
+        tmp = Dir.mktmpdir('fp-run-')
+        begin
+          report_path, argv, env = self.class.inject(runner, command, tmp)
+          warn "fp run: #{argv.join(' ')}"
+          test_status = execute(env, argv)
+          if test_status.nil?
+            output.error("Could not start `#{argv.first}`: is it installed and on PATH?")
+            exit 127
+          end
+
+          unless File.file?(report_path) && File.size(report_path).positive?
+            output.error("The test command exited #{test_status} without writing a report, so there is nothing to report.")
+            exit(test_status.zero? ? 1 : test_status)
+          end
+
+          report_status = report(report_args, report_path)
+          exit test_status unless test_status.zero?
+          exit report_status unless report_status.zero?
+        ensure
+          FileUtils.rm_rf(tmp)
+        end
+      end
+
+      # Which supported runner a command invokes, by its words: `bundle exec rspec`,
+      # `bin/rspec`, `python -m pytest`, `npx vitest run`, `pnpm exec playwright test`.
+      def self.detect_runner(command)
+        command.each_with_index do |word, i|
+          base = File.basename(word)
+          return 'rspec' if base == 'rspec'
+          return 'pytest' if %w[pytest py.test].include?(base)
+          return 'vitest' if base == 'vitest'
+          return 'playwright' if base == 'playwright' && command[i + 1] == 'test'
+        end
+        nil
+      end
+
+      # [report path, argv to run, extra env] for a runner.
+      def self.inject(runner, command, tmp)
+        case runner
+        when 'rspec'
+          path = File.join(tmp, 'rspec.json')
+          formatted = command.any? { |a| a.start_with?('--format') || a.start_with?('-f') }
+          extra = ['--format', 'json', '--out', path]
+          extra += ['--format', 'progress'] unless formatted
+          [path, command + extra, {}]
+        when 'pytest'
+          path = File.join(tmp, 'junit.xml')
+          [path, command + ['-o', 'junit_family=xunit1', "--junitxml=#{path}"], {}]
+        when 'vitest'
+          path = File.join(tmp, 'junit.xml')
+          [path, command + ['--reporter=default', '--reporter=junit', "--outputFile.junit=#{path}"], {}]
+        when 'playwright'
+          path = File.join(tmp, 'junit.xml')
+          [path, command + ['--reporter=list,junit'], { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME' => path }]
+        end
+      end
+
+      private
+
+      # The command's exit status, or nil when it couldn't be started.
+      def execute(env, argv)
+        ok = system(env, *argv)
+        return nil if ok.nil?
+
+        $?.exitstatus || 1
+      end
+
+      # `fp report --junit` with the caller's flags; its own exit status (0 on success).
+      def report(report_args, report_path)
+        # The report sits in a temp dir, so `fp report` can't find the project from it the
+        # way it does beside a checked-in report. The checkout we ran in says which it is.
+        unless report_args.include?('--project') || report_args.any? { |a| a.start_with?('--project=') }
+          project = ProjectConfig.discover(Dir.pwd).project
+          report_args += ['--project', project] if project
+        end
+        Report.new(client: client, output: output, config: config).run(report_args + ['--junit', report_path])
+        0
+      rescue SystemExit => e
+        e.status
+      end
+    end
+  end
+end

data/lib/fp/commands/show.rb

@@ -44,12 +44,17 @@ module Fp
         # Agents only ever speak slugs, so translate the parent's ID back into one.
         parent_slug = resolve_parent_slug(project_id, requirement['parent_id'])
 
+        # Open questions for a human (Fizzy #1786), read only when the row says it has some.
+        open_questions = fetch_open_questions(requirement)
+
         payload = result[:data].merge('evidence' => evidence, 'attachments' => attachments)
         payload['parent_slug'] = parent_slug if parent_slug
+        payload['open_questions'] = open_questions unless open_questions.empty?
 
         output.success(payload) do
           print_requirement(requirement, evidence: evidence, attachments: attachments,
                                           parent_slug: parent_slug, source_history: source_history)
+          print_open_questions(open_questions, requirement['slug'])
         end
       end
 
@@ -59,7 +64,34 @@ module Fp
         result = client.list_evidence(project_id, slug: slug)
         return [] unless result[:ok]
 
-        result[:data]['evidence'] || []
+        # The server names the receipts CI has since spoken for; flag them on the rows so
+        # both the listing and --json say so, rather than a stale `present` reading as a
+        # second verdict beside CI's (Fizzy #1762).
+        superseded = Array(result[:data]['superseded_ids'])
+        (result[:data]['evidence'] || []).map do |e|
+          superseded.include?(e['id']) ? e.merge('superseded' => true) : e
+        end
+      end
+
+      def fetch_open_questions(requirement)
+        return [] unless requirement['open_question_count'].to_f.positive?
+
+        result = client.list_requirement_questions(requirement['id'])
+        return [] unless result[:ok]
+
+        (result[:data]['questions'] || []).select { |q| q['status'] == 'open' }
+      end
+
+      def print_open_questions(questions, slug)
+        return if questions.empty?
+
+        puts
+        puts 'Open Questions (for a human):'
+        questions.each do |q|
+          puts "  - #{q['text']}"
+          puts "    asked by #{q['asked_by']} at #{q['asked_at']}"
+        end
+        puts "  (fp questions #{slug} lists answered ones too)"
       end
 
       def fetch_attachments(requirement_id)
@@ -140,6 +172,7 @@ module Fp
             line << src['url'] if src['url']
             puts "  - #{line.join(' ')}"
             puts "    #{src['context']}" if src['context']
+            Array(src['refs']).each { |ref| puts "    read from #{ref}" }
           end
         end
         puts
@@ -156,8 +189,13 @@ module Fp
             .group_by { |e| e['surface'] }
             .each do |surface, records|
               puts "  #{surface}:"
-              records.each do |e|
-                puts "    Status:  #{e['status']}"
+              # The receipts the matrix counts first; superseded ones after.
+              records.partition { |e| !e['superseded'] }.flatten(1).each do |e|
+                if e['superseded']
+                  puts "    Status:  superseded by CI (was #{e['status']}; CI's latest run is what counts)"
+                else
+                  puts "    Status:  #{e['status']}"
+                end
                 # A pull request's CI may have written over the merged verdict; show both
                 # when they differ so "failing" next to "PR: …" reads as the branch's
                 # result, not the default branch's.