featureparity 0.0.28 → 0.0.30

data/lib/fp/commands/question.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Fp
+  module Commands
+    # Shared by `fp question` and `fp questions`: find the requirement, print a question.
+    module QuestionSupport
+      private
+
+      def find_requirement!(project_id, slug)
+        result = client.find_requirement_by_slug(project_id, slug)
+        unless result[:ok]
+          output.error(result[:error], status: result[:status])
+          exit 1
+        end
+        result[:data]['requirement']
+      end
+
+      def print_question(question)
+        asker = question['asked_by'] || 'unknown'
+        kind = question['asked_by_kind'] == 'agent' ? 'agent' : 'human'
+        puts "  [#{question['status']}] #{question['text']}"
+        puts "    asked by #{asker} (#{kind}) at #{question['asked_at']}"
+        case question['status']
+        when 'answered'
+          puts "    answer: #{question['answer']} (#{question['resolved_by']})"
+        when 'dismissed'
+          puts "    dismissed by #{question['resolved_by']}"
+        end
+      end
+    end
+
+    # `fp question <slug> "<text>"` — attach an open question for a human to the draft it
+    # concerns (Fizzy #1786).
+    #
+    # An agent writing a batch of drafts usually has questions only a human can answer:
+    # a limit the source never set, a role it didn't name. Left in the chat, they're
+    # detached from the drafts. Asked here, the human reviewing the draft in Quick Review
+    # sees them beside it, and answers or dismisses each one there. Agents can ask but
+    # not answer; the web app is where a human resolves them.
+    class Question < Base
+      include QuestionSupport
+
+      KNOWN_FLAGS = { project: :string, body: :string }.freeze
+
+      help_summary 'Attach an open question for a human to a draft requirement'
+      help_usage 'fp question <slug> "<question>" [--project <slug>]'
+      help_flag :project, 'Project slug (falls back to .featureparity.yml)'
+      help_flag :body, 'The question text (instead of the second argument)'
+      help_example 'fp question upload_limits "Is the 5 MB cap per file or per upload?"'
+      help_example 'fp questions upload_limits   # list open and resolved questions'
+
+      def run(args)
+        opts, positional = parse_flags(args, KNOWN_FLAGS)
+        require_project!(opts)
+
+        slug = positional.first
+        unless slug
+          output.error('Requirement slug is required. Usage: fp question <slug> "<question>" [--project <project>]')
+          exit 1
+        end
+
+        text = opts[:body] || positional[1..].join(' ')
+        if text.nil? || text.strip.empty?
+          output.error("A question needs its text. Usage: fp question #{slug} \"<question>\" " \
+                       "(run `fp questions #{slug}` to list them)")
+          exit 1
+        end
+
+        project_id = resolve_project_id(opts[:project])
+        requirement = find_requirement!(project_id, slug)
+
+        result = client.ask_requirement_question(requirement['id'], text.strip)
+        unless result[:ok]
+          output.error(result[:error], status: result[:status])
+          exit 1
+        end
+
+        output.success(result[:data]) do
+          open_count = result[:data]['open_count']
+          puts "Asked on '#{slug}': #{result[:data].dig('question', 'text')}"
+          puts
+          puts "#{open_count} open question#{'s' unless open_count == 1} on this requirement." if open_count
+          puts 'A human reviewing it in Quick Review sees the question and answers or dismisses it there.'
+          puts "Run `fp questions #{slug}` to read the answers."
+        end
+      end
+    end
+
+    # `fp questions <slug>` — the questions asked on a requirement, open and resolved.
+    class Questions < Base
+      include QuestionSupport
+
+      KNOWN_FLAGS = { project: :string, open: :boolean }.freeze
+
+      help_summary "List a requirement's questions for a human, with their answers"
+      help_usage 'fp questions <slug> [--open] [--project <slug>]'
+      help_flag :project, 'Project slug (falls back to .featureparity.yml)'
+      help_flag :open, 'Show only the questions still open'
+      help_example 'fp questions upload_limits'
+      help_example 'fp questions upload_limits --open --json'
+
+      def run(args)
+        opts, positional = parse_flags(args, KNOWN_FLAGS)
+        require_project!(opts)
+
+        slug = positional.first
+        unless slug
+          output.error('Requirement slug is required. Usage: fp questions <slug> [--project <project>]')
+          exit 1
+        end
+
+        project_id = resolve_project_id(opts[:project])
+        requirement = find_requirement!(project_id, slug)
+
+        result = client.list_requirement_questions(requirement['id'])
+        unless result[:ok]
+          output.error(result[:error], status: result[:status])
+          exit 1
+        end
+
+        questions = result[:data]['questions'] || []
+        questions = questions.select { |q| q['status'] == 'open' } if opts[:open]
+        payload = { 'requirement' => requirement, 'questions' => questions,
+                    'open_count' => result[:data]['open_count'] }
+
+        output.success(payload) do
+          puts "Questions on '#{slug}':"
+          if questions.empty?
+            puts opts[:open] ? '  (none open)' : '  (none asked)'
+          else
+            questions.each { |q| print_question(q) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fp/commands/report.rb

@@ -36,7 +36,7 @@ module Fp
       help_flag :ci_url, 'CI run URL'
       help_flag :title, 'Human-readable example/test name to display'
       help_flag :state, 'Evidence state: present or stub (default present)'
-      help_flag :junit, 'JUnit XML report to batch-upload evidence from'
+      help_flag :junit, 'JUnit XML report (or RSpec `--format json` output) to batch-upload evidence from'
       help_flag :base_dir, 'Where relative test-file paths in the report resolve from'
       help_flag :attach_media, 'With --junit: upload the videos/screenshots the run recorded ([[ATTACHMENT|…]]) as visual evidence'
       help_flag :stamp_all, 'With --junit: put --pr/--work-item on every receipt, not only on tests in files this branch changed'
@@ -124,6 +124,7 @@ module Fp
         params[:line] = opts[:line].to_i if opts[:line]
 
         # Try to report evidence
+        warn_if_not_production!
         result = client.report_evidence(params)
 
         # Evidence API may not exist yet (#1214)
@@ -288,6 +289,7 @@ module Fp
         # on requirements the branch never touched — "View Card" on an unrelated
         # requirement led to whichever card last ran the suite (Fizzy #1631). Only the
         # tests in files this branch changed get them; the rest keep their own.
+        warn_if_not_production!
         provenance = provenance_scope(opts, base_dir)
         reported = []
         skipped_unknown = []
@@ -504,7 +506,8 @@ module Fp
         stamping = !opts[:pr].nil? || !opts[:work_item].nil?
         data = {
           repo: opts[:repo],
-          reported: reported.map { |r| { slug: r[:binding].slug, surface: r[:binding].surface, test_type: r[:binding].test_type, file: r[:binding].file, line: r[:binding].line, state: r[:state], recorded: r[:recorded], stamped: (stamping ? r[:stamped] != false : nil), stored_status: r[:stored_status], superseded_by_ci: ci_superseded?(r[:state], r[:stored_status], r[:stored_source]) }.compact },
+          reported: reported.map { |r| { slug: r[:binding].slug, surface: r[:binding].surface, test_type: r[:binding].test_type, file: r[:binding].file, line: r[:binding].line, state: r[:state], recorded: r[:recorded], stamped: (stamping ? r[:stamped] != false : nil), stored_status: r[:stored_status], superseded_by_ci: ci_superseded?(r[:state], r[:stored_status], r[:stored_source]), tests: binding_tests(r[:binding]) }.compact },
+          tests_bound: reported.sum { |r| binding_tests(r[:binding]).length },
           provenance: (stamping ? { scoped: provenance.scoped?, mode: provenance.mode.to_s, stamped: reported.count { |r| r[:stamped] != false } } : nil),
           skipped_unknown: skipped_unknown.map { |b| { slug: b.slug, surface: b.surface, file: b.file } },
           unmatched_testcases: unmatched.map { |tc| { name: tc.name, file: tc.file } },
@@ -515,7 +518,8 @@ module Fp
         }
 
         output.success(data) do
-          puts "Reported evidence from #{opts[:junit]}:"
+          tests_bound = reported.sum { |r| binding_tests(r[:binding]).length }
+          puts "Reported evidence from #{opts[:junit]}: #{tests_bound} bound test(s) → #{reported.length} receipt(s)."
           reported.each do |r|
             superseded = ci_superseded?(r[:state], r[:stored_status], r[:stored_source])
             marker = if !r[:recorded]
@@ -530,6 +534,15 @@ module Fp
             shown_state = superseded ? "#{r[:stored_status]}, kept CI — #{r[:state]} ignored" : r[:state]
             stamp = stamping && provenance.scoped? && r[:stamped] ? '  ← this change' : ''
             puts "  #{marker} #{r[:binding].slug} → #{r[:binding].surface}#{type}  (#{shown_state})  #{loc}#{stamp}"
+            tests = binding_tests(r[:binding])
+            next if tests.length < 2
+
+            puts "      #{tests.length} tests share this receipt (one per requirement, surface, test type and file):"
+            tests.each do |t|
+              at = t[:line] ? ":#{t[:line]}" : ''
+              skipped = t[:state] == 'stub' ? '  (skipped)' : ''
+              puts "        - #{t[:name] || '(unnamed)'}  #{r[:binding].file}#{at}#{skipped}"
+            end
           end
 
           if stamping
@@ -589,6 +602,13 @@ module Fp
         end
       end
 
+      # The testcases that fed a receipt. Several tests proving one requirement is normal;
+      # the server keeps one row per cell, so the summary names each of them rather than
+      # leaving the reader to guess whether the rest were dropped.
+      def binding_tests(binding)
+        binding.tests || [{ name: binding.title, line: binding.line, state: binding.state }]
+      end
+
       # Warn when two or more distinct test files in this run resolved to the *same*
       # evidence cell. Server-side identity is (project, slug, surface, repo, test_type) —
       # the file isn't part of it — so those reports upsert the same row and the last one

data/lib/fp/commands/run.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+require 'fileutils'
+
+module Fp
+  module Commands
+    # fp run [report flags] -- <test command...>
+    #
+    # Runs the test suite with a report formatter fp adds itself, then reports it exactly
+    # as `fp report --junit <that report>` would (Fizzy #1786). Getting a report out of a
+    # suite usually meant adding a formatter gem or package to someone else's project
+    # (rspec_junit_formatter, say) before the first `fp report` could run; every runner
+    # here can write one with options it already has, so nothing in the project changes.
+    #
+    #   rspec       --format json --out <tmp> (+ --format progress when none was given)
+    #   pytest      -o junit_family=xunit1 --junitxml=<tmp>  (xunit1 carries file and line)
+    #   vitest      --reporter=default --reporter=junit --outputFile.junit=<tmp>
+    #   playwright  --reporter=list,junit with PLAYWRIGHT_JUNIT_OUTPUT_NAME=<tmp>
+    #
+    # It *is* `fp report`: same flags, same present/stub-only rule. The test command's
+    # own exit status wins, after reporting, so a red suite still fails the caller.
+    class Run < Base
+      help_summary 'Run a test command with a report formatter added, then fp report its results'
+      help_usage 'fp run [--surface X] [--repo org/repo] [fp report flags] -- <test command...>'
+      help_flag :surface, 'Surface the evidence is for (default for markers that pin none)'
+      help_flag :repo, 'Repository the tests live in (org/repo)', required: true
+      help_flag :project, 'Project slug (falls back to .featureparity.yml)'
+      help_flag :test_type, "Kind of test (project's vocabulary, e.g. unit, e2e)"
+      help_flag :work_item, 'Work-item URL that prompted the work'
+      help_flag :pr, 'Pull request URL'
+      help_flag :base_dir, 'Where relative test-file paths in the report resolve from'
+      help_example 'fp run --surface api --repo acme/shop -- bundle exec rspec'
+      help_example 'fp run --surface web --repo acme/shop -- npx vitest run'
+      help_example 'fp run --surface api --repo acme/shop -- pytest tests/'
+      help_example 'fp run --surface web --repo acme/shop -- npx playwright test'
+      help_note "Supports rspec, pytest, vitest and playwright. Any other runner: emit JUnit XML yourself\n" \
+                "and use `fp report --junit <file>`. Takes every `fp report` flag; agents still report\n" \
+                "present/stub only. Exits with the test command's status when it failed."
+
+      RUNNERS = %w[rspec pytest vitest playwright].freeze
+
+      def run(args)
+        split = args.index('--')
+        report_args = split ? args[0...split] : args
+        command = split ? args[(split + 1)..] : []
+
+        if command.empty?
+          output.error('Give the test command after `--`, e.g. fp run --surface api --repo org/repo -- bundle exec rspec')
+          exit 1
+        end
+        if report_args.include?('--junit') || report_args.any? { |a| a.start_with?('--junit=') }
+          output.error('fp run writes its own report; drop --junit (or use `fp report --junit <file>` instead).')
+          exit 1
+        end
+
+        runner = self.class.detect_runner(command)
+        unless runner
+          output.error("fp run doesn't recognise the runner in `#{command.join(' ')}`. " \
+                       "Supported: #{RUNNERS.join(', ')}. For anything else, make the runner write " \
+                       'JUnit XML and run `fp report --junit <file>`.')
+          exit 1
+        end
+
+        tmp = Dir.mktmpdir('fp-run-')
+        begin
+          report_path, argv, env = self.class.inject(runner, command, tmp)
+          warn "fp run: #{argv.join(' ')}"
+          test_status = execute(env, argv)
+          if test_status.nil?
+            output.error("Could not start `#{argv.first}`: is it installed and on PATH?")
+            exit 127
+          end
+
+          unless File.file?(report_path) && File.size(report_path).positive?
+            output.error("The test command exited #{test_status} without writing a report, so there is nothing to report.")
+            exit(test_status.zero? ? 1 : test_status)
+          end
+
+          report_status = report(report_args, report_path)
+          exit test_status unless test_status.zero?
+          exit report_status unless report_status.zero?
+        ensure
+          FileUtils.rm_rf(tmp)
+        end
+      end
+
+      # Which supported runner a command invokes, by its words: `bundle exec rspec`,
+      # `bin/rspec`, `python -m pytest`, `npx vitest run`, `pnpm exec playwright test`.
+      def self.detect_runner(command)
+        command.each_with_index do |word, i|
+          base = File.basename(word)
+          return 'rspec' if base == 'rspec'
+          return 'pytest' if %w[pytest py.test].include?(base)
+          return 'vitest' if base == 'vitest'
+          return 'playwright' if base == 'playwright' && command[i + 1] == 'test'
+        end
+        nil
+      end
+
+      # [report path, argv to run, extra env] for a runner.
+      def self.inject(runner, command, tmp)
+        case runner
+        when 'rspec'
+          path = File.join(tmp, 'rspec.json')
+          formatted = command.any? { |a| a.start_with?('--format') || a.start_with?('-f') }
+          extra = ['--format', 'json', '--out', path]
+          extra += ['--format', 'progress'] unless formatted
+          [path, command + extra, {}]
+        when 'pytest'
+          path = File.join(tmp, 'junit.xml')
+          [path, command + ['-o', 'junit_family=xunit1', "--junitxml=#{path}"], {}]
+        when 'vitest'
+          path = File.join(tmp, 'junit.xml')
+          [path, command + ['--reporter=default', '--reporter=junit', "--outputFile.junit=#{path}"], {}]
+        when 'playwright'
+          path = File.join(tmp, 'junit.xml')
+          [path, command + ['--reporter=list,junit'], { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME' => path }]
+        end
+      end
+
+      private
+
+      # The command's exit status, or nil when it couldn't be started.
+      def execute(env, argv)
+        ok = system(env, *argv)
+        return nil if ok.nil?
+
+        $?.exitstatus || 1
+      end
+
+      # `fp report --junit` with the caller's flags; its own exit status (0 on success).
+      def report(report_args, report_path)
+        # The report sits in a temp dir, so `fp report` can't find the project from it the
+        # way it does beside a checked-in report. The checkout we ran in says which it is.
+        unless report_args.include?('--project') || report_args.any? { |a| a.start_with?('--project=') }
+          project = ProjectConfig.discover(Dir.pwd).project
+          report_args += ['--project', project] if project
+        end
+        Report.new(client: client, output: output, config: config).run(report_args + ['--junit', report_path])
+        0
+      rescue SystemExit => e
+        e.status
+      end
+    end
+  end
+end

data/lib/fp/commands/show.rb

@@ -44,12 +44,17 @@ module Fp
         # Agents only ever speak slugs, so translate the parent's ID back into one.
         parent_slug = resolve_parent_slug(project_id, requirement['parent_id'])
 
+        # Open questions for a human (Fizzy #1786), read only when the row says it has some.
+        open_questions = fetch_open_questions(requirement)
+
         payload = result[:data].merge('evidence' => evidence, 'attachments' => attachments)
         payload['parent_slug'] = parent_slug if parent_slug
+        payload['open_questions'] = open_questions unless open_questions.empty?
 
         output.success(payload) do
           print_requirement(requirement, evidence: evidence, attachments: attachments,
                                           parent_slug: parent_slug, source_history: source_history)
+          print_open_questions(open_questions, requirement['slug'])
         end
       end
 
@@ -68,6 +73,27 @@ module Fp
         end
       end
 
+      def fetch_open_questions(requirement)
+        return [] unless requirement['open_question_count'].to_f.positive?
+
+        result = client.list_requirement_questions(requirement['id'])
+        return [] unless result[:ok]
+
+        (result[:data]['questions'] || []).select { |q| q['status'] == 'open' }
+      end
+
+      def print_open_questions(questions, slug)
+        return if questions.empty?
+
+        puts
+        puts 'Open Questions (for a human):'
+        questions.each do |q|
+          puts "  - #{q['text']}"
+          puts "    asked by #{q['asked_by']} at #{q['asked_at']}"
+        end
+        puts "  (fp questions #{slug} lists answered ones too)"
+      end
+
       def fetch_attachments(requirement_id)
         return [] if requirement_id.to_s.empty?
 
@@ -146,6 +172,7 @@ module Fp
             line << src['url'] if src['url']
             puts "  - #{line.join(' ')}"
             puts "    #{src['context']}" if src['context']
+            Array(src['refs']).each { |ref| puts "    read from #{ref}" }
           end
         end
         puts

data/lib/fp/commands.rb

@@ -9,7 +9,9 @@ require_relative 'commands/list'
 require_relative 'commands/show'
 require_relative 'commands/propose'
 require_relative 'commands/propose_change'
+require_relative 'commands/mark'
 require_relative 'commands/comment'
+require_relative 'commands/question'
 require_relative 'commands/comments'
 require_relative 'commands/history'
 require_relative 'commands/analyze'
@@ -19,8 +21,11 @@ require_relative 'commands/ci_report'
 require_relative 'commands/changed_tests'
 require_relative 'commands/spot_check'
 require_relative 'commands/check'
+require_relative 'commands/coverage'
 require_relative 'commands/init'
 require_relative 'commands/tutorial'
+require_relative 'commands/doctor'
+require_relative 'commands/run'
 require_relative 'commands/matrix'
 require_relative 'commands/export'
 require_relative 'commands/profile'

data/lib/fp/junit.rb

@@ -3,6 +3,7 @@
 require 'rexml/document'
 require 'pathname'
 require_relative 'source_fingerprint'
+require_relative 'rspec_json'
 
 module Fp
   # Parses JUnit XML reports and binds testcases to fp:<slug> markers found in
@@ -83,7 +84,7 @@ module Fp
     # testcase's own line when the runner reported one, otherwise the line of
     # the fp:<slug> marker that annotates it (markers sit directly above the
     # test, so the marker line is a close stand-in when the runner omits lines).
-    Binding = Struct.new(:slug, :surface, :test_type, :file, :line, :title, :state, keyword_init: true)
+    Binding = Struct.new(:slug, :surface, :test_type, :file, :line, :title, :state, :tests, keyword_init: true)
 
     # A media file (video/screenshot) a run recorded for one bound testcase, ready for
     # `fp report --junit --attach-media` to upload as visual evidence. Unlike Binding,
@@ -120,17 +121,51 @@ module Fp
 
     module_function
 
-    # Parse a JUnit XML file into an array of TestCase structs.
+    # Parse a test report into an array of TestCase structs: JUnit XML, or RSpec's
+    # built-in `--format json` output (Fizzy #1786), told apart by content so every
+    # `--junit` flag takes either with no new option. RSpec's JSON needs no extra gem,
+    # where JUnit from RSpec needs `rspec_junit_formatter` in the project's Gemfile.
     # Raises ArgumentError if the file is missing or unparseable.
     def parse_file(path)
       raise ArgumentError, "JUnit file not found: #{path}" unless File.file?(path)
 
-      xml = File.read(path)
-      parse_string(xml)
+      content = File.read(path)
+      return RspecJson.parse(content, path) if RspecJson.json?(content)
+
+      parse_string(content)
     rescue REXML::ParseException => e
       raise ArgumentError, "Could not parse JUnit XML (#{path}): #{e.message}"
     end
 
+    # The report as JUnit XML, whatever format the file is in. The CLI resolves markers
+    # itself, but ci-report and check also send the raw report for servers old enough to
+    # re-resolve it, and those only read XML.
+    def read_xml(path)
+      content = File.read(path)
+      return content unless RspecJson.json?(content)
+
+      to_xml(RspecJson.parse(content, path))
+    end
+
+    # Render parsed testcases as a minimal JUnit document.
+    def to_xml(testcases)
+      doc = REXML::Document.new
+      doc << REXML::XMLDecl.new('1.0', 'UTF-8')
+      suite = doc.add_element('testsuites').add_element('testsuite', 'name' => 'fp',
+                                                                    'tests' => testcases.length.to_s)
+      testcases.each do |tc|
+        attrs = { 'name' => tc.name.to_s, 'classname' => tc.classname.to_s }
+        attrs['file'] = tc.file if tc.file
+        attrs['line'] = tc.line.to_s if tc.line
+        el = suite.add_element('testcase', attrs)
+        el.add_element('skipped') if tc.stub?
+        el.add_element('failure') if tc.failed?
+      end
+      out = +''
+      doc.write(out)
+      out
+    end
+
     def parse_string(xml)
       doc = REXML::Document.new(xml)
       cases = []
@@ -144,7 +179,7 @@ module Fp
           name: el.attributes['name'],
           classname: classname,
           file: resolve_file(el.attributes['file'], classname, suite_name),
-          line: (el.attributes['line'] && el.attributes['line'].to_i),
+          line: testcase_line(el, suite_name),
           status: testcase_status(el),
           failed: testcase_failed?(el),
           attachments: extract_attachments(el)
@@ -154,6 +189,16 @@ module Fp
       cases
     end
 
+    # pytest's `line` (written with junit_family=xunit1, the only family that carries a
+    # file) counts from 0, so a `def` directly under its marker would report the marker's
+    # own line and bind to the marker above that. Its suites are named "pytest".
+    def testcase_line(el, suite_name)
+      line = el.attributes['line']&.to_i
+      return line unless line && suite_name == 'pytest' && el.attributes['file'].to_s.end_with?('.py')
+
+      line + 1
+    end
+
     # Playwright's JUnit reporter records media (video, screenshots, trace) by writing
     # `[[ATTACHMENT|<path>]]` lines into the testcase's `<system-out>`. Pull those paths
     # out so `fp report --junit --attach-media` can upload them as visual evidence for the
@@ -600,12 +645,20 @@ module Fp
       # omitted line info.
       line = testcase.line || (marker && marker[:line])
 
+      # Every testcase that fed this receipt, so `fp report` can say "3 tests → one
+      # receipt" instead of printing one line and leaving the other two unaccounted for.
+      entry = { name: presence(testcase.name), line: line, state: state }
+
       existing = bindings[key]
       # Prefer a concrete 'present' state over 'stub' when a slug has both.
-      return if existing && !(existing.state == 'stub' && state == 'present')
+      if existing && !(existing.state == 'stub' && state == 'present')
+        existing.tests << entry
+        return
+      end
 
       bindings[key] = Binding.new(slug: slug, surface: surface, test_type: test_type, file: rel_file,
-                                  line: line, title: (testcase.name if titles_reliable), state: state)
+                                  line: line, title: (testcase.name if titles_reliable), state: state,
+                                  tests: (existing ? existing.tests : []) + [entry])
     end
 
     # Load and cache the fp:<slug> markers (with line numbers) for an already-resolved