featureparity 0.0.28 → 0.0.30

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dd6b0a540d21e123a2ceb43cb9f23005c218319866ff7e079b951d6288199bb1
-  data.tar.gz: 6e0d61e3a3d9e6f590c6e978b134736877b5abc8da2a9a80332ff6cd203dbf58
+  metadata.gz: 7b3bad4ccc28e0695bdb790ecc37d6836d5477a84de09442c5255df871d98444
+  data.tar.gz: 67fcd3d1ac1a1a7bd41197ae674942fea80d20bbea22d856075face50ad87007
 SHA512:
-  metadata.gz: fe1d8110555996bc72c885d502139e6f18dbef1d2ee8b8623eaf48c74dce6c40b33f46ffe4b42f09c9f92570380abab9b4dff77df3520d300ec6f43f35214890
-  data.tar.gz: 1f95d64e43e62a63411bd8cc665248777c3acdc58795e16552d4b5f671d3a1fb89a774bbf066e5f65f55bba7b716cc5f2fa78d51ea2b1a5afdc1fcd547acfa2d
+  metadata.gz: b276bc10363baf2fe27e843e0824ed205ed62f7836f4671cf53db15fc92ddc6a579a2fb1f5da34c4f34e672ffe2c14c418d10239e55066aa1a4b5806f078b930
+  data.tar.gz: 2769c008ecac2bdd2f02c13ef34cdc91619085709965138dc8750ab4127ebd22d119eae646a89202217db47235ddc4e75e1efae24528feeed1b1a7d53ec20f80

data/lib/fp/cli.rb

@@ -8,6 +8,8 @@ module Fp
     COMMANDS = {
       'init' => Commands::Init,
       'tutorial' => Commands::Tutorial,
+      'doctor' => Commands::Doctor,
+      'run' => Commands::Run,
       'projects' => Commands::Projects,
       'surfaces' => Commands::Surfaces,
       'showroom' => Commands::Showroom,
@@ -15,7 +17,10 @@ module Fp
       'show' => Commands::Show,
       'propose' => Commands::Propose,
       'propose-change' => Commands::ProposeChange,
+      'mark' => Commands::Mark,
       'comment' => Commands::Comment,
+      'question' => Commands::Question,
+      'questions' => Commands::Questions,
       'comments' => Commands::Comments,
       'history' => Commands::History,
       'analyze' => Commands::Analyze,
@@ -25,6 +30,7 @@ module Fp
       'changed-tests' => Commands::ChangedTests,
       'spot-check' => Commands::SpotCheck,
       'check' => Commands::Check,
+      'coverage' => Commands::Coverage,
       'matrix' => Commands::Matrix,
       'export' => Commands::Export,
       'profile' => Commands::Profile,
@@ -42,6 +48,9 @@ module Fp
     class << self
       def run(argv)
         args = argv.dup
+        # Everything after `--` belongs to another program (`fp run -- pytest -p no:cov`),
+        # so global flags and --help are only looked for before it.
+        passthrough = (split = args.index('--')) ? args.slice!(split..) : []
         global_opts = extract_global_options(args)
 
         command_name = args.shift || 'help'
@@ -71,7 +80,7 @@ module Fp
 
         # Profile/config commands don't need auth. `init` is here too: it only writes
         # files into the repo, and onboarding a repo must work before anyone has a key.
-        needs_auth = !%w[init profile config repos setup whoami version help].include?(command_name)
+        needs_auth = !%w[init doctor profile config repos setup whoami version help mark].include?(command_name)
 
         if needs_auth
           config = Config.new(profile: global_opts[:profile], api_url: global_opts[:api_url])
@@ -87,7 +96,7 @@ module Fp
 
         output = Output.new(json: global_opts[:json])
         command = command_class.new(client: client, output: output, config: config)
-        command.run(args)
+        command.run(args + passthrough)
       rescue Interrupt
         warn "\nAborted."
         exit 130

data/lib/fp/client.rb

@@ -247,6 +247,21 @@ module Fp
       post("/api/requirements/#{requirement_id}/changes/#{change_id}/comment", params)
     end
 
+    # GET /requirements/:id/questions
+    #
+    # The questions left for a human on a requirement (Fizzy #1786), open and resolved.
+    def list_requirement_questions(requirement_id)
+      get("/api/requirements/#{requirement_id}/questions")
+    end
+
+    # POST /requirements/:id/questions
+    #
+    # Attach an open question for a human to a draft (or active) requirement. There is
+    # no resolve method: only a human answers or dismisses, from the web app.
+    def ask_requirement_question(requirement_id, text)
+      post("/api/requirements/#{requirement_id}/questions", { text: text })
+    end
+
     # GET /requirements/:id/comments
     #
     # The team's comments on a requirement, oldest first, plus who can be tagged:
@@ -303,12 +318,16 @@ module Fp
     # only do that by fetching the test files from GitHub, which fails silently on a
     # private repo, so resolved results are always sent when we have them.
     def ingest_ci_results(project_id, ci_url:, surface:, junit_xml:, pr_url: nil, results: nil,
-                          test_revisions: nil, unmarked_failures: nil, orphan_markers: nil, sha: nil)
+                          test_revisions: nil, unmarked_failures: nil, orphan_markers: nil, sha: nil,
+                          local: false)
       body = {
         ci_url: ci_url,
         surface: surface,
         junit_xml: junit_xml
       }
+      # A run on someone's machine rather than in CI (`fp ci-report --local`): there may be
+      # no run URL to record, and the server only accepts that when told so.
+      body[:local] = true if local
       # Only send pr_url when set, so a merged-branch run stays a plain ingest.
       body[:pr_url] = pr_url if pr_url && !pr_url.empty?
       # The checkout commit, so the receipt (or a PR run's branch verdict) links the
@@ -367,6 +386,17 @@ module Fp
       get("/api/projects/#{project_id}/parity", params)
     end
 
+    # GET /projects/:project_id/parity?include_drafts=true — the same report plus a
+    # top-level `drafts` array: never-approved drafts with their cells resolved from
+    # evidence. Informational only; the server never gates on it (Fizzy #1786). Read by
+    # `fp coverage`, `fp list --status draft --gaps` and `fp check --include-drafts`.
+    def get_parity_with_drafts(project_id, level: nil, surface: nil)
+      params = { include_drafts: 'true' }
+      params[:level] = level if level
+      params[:surface] = surface if surface
+      get("/api/projects/#{project_id}/parity", params)
+    end
+
     # POST /projects/:project_id/parity/preview — the pull-request gate.
     #
     # Evaluates JUnit results as an overlay on the stored grid *without persisting them*,

data/lib/fp/commands/base.rb

@@ -210,6 +210,20 @@ module Fp
         project['id']
       end
 
+      # One line on stderr before a command writes to a project, when the API it's about
+      # to write to isn't production. FP_PROFILE / FP_API_URL left over in a shell put an
+      # agent's new project on a dev environment with nothing to say so (Fizzy #1786).
+      # Stderr keeps --json output parseable; it never stops the command.
+      def warn_if_not_production!
+        env = config&.environment
+        return if env.nil? || env == 'production'
+
+        profile = config.profile_name
+        via = profile ? " (profile '#{profile}')" : ''
+        $stderr.puts "⚠ Writing to the '#{env}' environment at #{config.api_url}#{via}, not production. " \
+                     'Run `fp whoami` to see where that comes from.'
+      end
+
       def require_flag(opts, flag, message = nil)
         unless opts[flag]
           message ||= "--#{flag.to_s.tr('_', '-')} is required"

data/lib/fp/commands/check.rb

@@ -51,15 +51,17 @@ module Fp
       help_flag :surface, 'Gate a single surface'
       help_flag :test_type, 'Gate a single test type'
       help_flag :level, 'Override the enforcement level for this run (off/regression/covered/strict)'
-      help_flag :junit, "Gate the branch on this run's results, overlaid on the grid (repeatable, one per surface)"
+      help_flag :junit, "Gate the branch on this run's results (JUnit XML or RSpec JSON), overlaid on the grid (repeatable, one per surface)"
       help_flag :base_dir, "Where the report's relative test-file paths resolve from"
       help_flag :pr, 'With --junit: this run is a pull request — fail only on parity it makes worse, ' \
                      'not on what the default branch already owes (optionally the PR URL)'
       help_flag :warn_only, 'Report violations but still exit 0'
+      help_flag :include_drafts, 'Also show draft coverage after the verdict (never changes the exit code)'
       help_example 'fp check', 'Gate on recorded state (project from .featureparity.yml)'
       help_example 'fp check --surface api --junit junit.xml', 'Gate a default-branch run on its own results'
       help_example 'fp check --surface api --junit junit.xml --pr', 'Gate a pull request on what it changes'
       help_example 'fp check --warn-only --json'
+      help_example 'fp check --include-drafts', 'Onboarding: which drafts have evidence, which have none'
       help_note "Exit 0 = passed, 1 = violations (or request failed). The level lives on the project\n" \
                 'server-side; --level only overrides this one run.'
 
@@ -73,7 +75,8 @@ module Fp
         # `--pr` alone is true; `--pr <url>` carries the URL. Not :boolean on purpose, so
         # the parser takes a following value when one is given.
         pr: :string,
-        warn_only: :boolean
+        warn_only: :boolean,
+        include_drafts: :boolean
       }.freeze
 
       # How many inherited violations to list before summarising the rest. They are
@@ -103,7 +106,9 @@ module Fp
         project_id = resolve_project_id(project_slug)
 
         result =
-          if junit_paths.empty?
+          if junit_paths.empty? && opts[:include_drafts]
+            client.get_parity_with_drafts(project_id, level: level, surface: surface)
+          elsif junit_paths.empty?
             client.get_parity(project_id, level: level, surface: surface)
           else
             preview(project_id, junit_paths, surface, level, opts)
@@ -125,7 +130,12 @@ module Fp
                'this verdict counts every violation, including those the default branch already has.'
         end
 
+        # Draft coverage rides along when asked for. Informational only: it is added after
+        # the verdict was decided and never feeds `passed` (Fizzy #1786).
+        report = report.merge('drafts' => draft_rows(project_id, level, surface, report)) if opts[:include_drafts]
+
         render(report, violations, passed, project_slug, opts)
+        print_drafts(report) if opts[:include_drafts] && !output.json?
 
         # A failing gate is the whole point of the command, so it must be the exit code
         # and not just something printed. --warn-only downgrades it to advisory.
@@ -134,6 +144,30 @@ module Fp
 
       private
 
+      # The standing read already carries `drafts` when asked for. The preview endpoint
+      # doesn't, so a --junit run reads them from the standing report instead; drafts are
+      # recorded state either way. A failed or too-old read is a warning, never a failed
+      # gate, because drafts can't change the verdict.
+      def draft_rows(project_id, level, surface, report)
+        return report['drafts'] if report.key?('drafts')
+
+        result = client.get_parity_with_drafts(project_id, level: level, surface: surface)
+        drafts = result[:ok] ? (result[:data]['parity'] || {})['drafts'] : nil
+        if drafts.nil?
+          warn 'Warning: draft coverage is unavailable from this API; the verdict is unaffected.'
+          return []
+        end
+        drafts
+      end
+
+      def print_drafts(report)
+        drafts = report['drafts'] || []
+        puts
+        puts "Draft coverage (#{drafts.size} draft(s), not gated):"
+        Coverage.print_drafts(output, drafts, report['surfaces'] || [])
+        Coverage.print_untested(Coverage.untested(drafts)) unless drafts.empty?
+      end
+
       # Preview mode needs to know which surface the results belong to, because the
       # server uses it to decide which cells the run had a chance to cover — and
       # therefore which vanished tests it may legitimately flag.
@@ -162,7 +196,7 @@ module Fp
         client.preview_parity(
           project_id,
           surface: surface,
-          junit_xmls: junit_paths.map { |p| File.read(p) },
+          junit_xmls: junit_paths.map { |p| Fp::JUnit.read_xml(p) },
           level: level,
           results: resolve_results(junit_paths, surface, opts),
           pull_request: !!opts[:pr],

data/lib/fp/commands/ci_report.rb

@@ -49,6 +49,12 @@ module Fp
     # of `passing`, and no merged evidence is demoted. Run it without --pr on pushes to
     # the default branch to record the real `passing`, superseding any pr_passing a prior
     # PR run left behind.
+    #
+    # --local is for when there is no CI to run the suite — a team out of CI minutes, or
+    # one without a pipeline yet (Fizzy #1837). A person runs the suite on their own
+    # machine and reports it the same way CI would; --ci-url becomes optional. The verdict
+    # is stamped on HEAD, so the working tree has to be clean: a result from edits nobody
+    # committed would be recorded against a commit that never ran them.
     class CiReport < Base
       help_summary 'CI only: ingest real pass/fail results (writes passing/failing evidence)'
       help_usage 'fp ci-report --junit <file>... --project <slug> --surface <surface> --ci-url URL [--pr URL]'
@@ -57,15 +63,18 @@ module Fp
       help_flag :test_type, 'Default test kind for markers that pin none'
       help_flag :ci_url, 'CI run URL to record on the evidence'
       help_flag :pr, "Mark this as a PR's CI: passing tests recorded as pr_passing, no merged evidence demoted"
-      help_flag :junit, 'JUnit XML report (repeatable to combine a surface split across shards)'
+      help_flag :junit, 'JUnit XML report, or RSpec `--format json` output (repeatable to combine a surface split across shards)'
       help_flag :base_dir, "Where the report's relative test-file paths resolve from (e.g. frontend)"
       help_flag :repo, 'Repository the tests live in (org/repo)'
       help_flag :sha, 'Checkout commit SHA'
       help_flag :allow_empty, 'Do not error when a report resolves no fp:<slug> markers'
+      help_flag :local, 'No CI available: report a suite a person ran locally at a clean, committed HEAD (--ci-url optional)'
       help_example 'fp ci-report --junit junit.xml --project stowzilla --surface api --ci-url "$CI_RUN_URL"'
       help_example 'fp ci-report --junit web.xml --surface web --base-dir frontend --pr "$PR_URL"'
+      help_example 'fp ci-report --local --junit junit.xml --surface api'
       help_note "This is CI's command, not an agent's: only it writes passing/failing. Run it on both\n" \
-                'green and red builds (if: always()) so failures show on the matrix.'
+                "green and red builds (if: always()) so failures show on the matrix.\n" \
+                'Out of CI minutes? A person can run the suite and report it with --local; agents report with fp report.'
 
       KNOWN_FLAGS = {
         project: :string,
@@ -77,7 +86,8 @@ module Fp
         base_dir: :string,
         repo: :string,
         sha: :string,
-        allow_empty: :boolean
+        allow_empty: :boolean,
+        local: :boolean
       }.freeze
 
       def run(args)
@@ -88,7 +98,7 @@ module Fp
 
         require_project!(opts)
         require_flag(opts, :surface)
-        require_flag(opts, :ci_url)
+        require_flag(opts, :ci_url) unless opts[:local]
 
         if junit_paths.empty?
           output.error('--junit <path> is required (may be given more than once).')
@@ -103,6 +113,7 @@ module Fp
 
         project_id = resolve_project_id(opts[:project])
         known = known_slugs(project_id)
+        require_clean_tree!(opts) if opts[:local]
         commit_sha = resolve_commit_sha(opts)
         repo = opts[:repo] || presence(ENV['GITHUB_REPOSITORY'])
 
@@ -127,7 +138,7 @@ module Fp
           totals[:unmarked_failures].concat(resolved[:unmarked_failures])
           totals[:test_revisions] += test_revisions.length
 
-          { path: path, xml: File.read(path), results: results, test_revisions: test_revisions }
+          { path: path, xml: Fp::JUnit.read_xml(path), results: results, test_revisions: test_revisions }
         end
 
         # One surface run may be split across several reports (unit + e2e, shards, etc.).
@@ -147,6 +158,7 @@ module Fp
           junit_xml: resolved_reports.first[:xml],
           pr_url: opts[:pr],
           sha: commit_sha,
+          local: opts[:local],
           results: combined_results,
           test_revisions: combined_revisions,
           unmarked_failures: totals[:unmarked_failures],
@@ -359,6 +371,23 @@ module Fp
         exit 1
       end
 
+      # A local run is recorded against HEAD, so HEAD has to be what ran. Untracked files
+      # are ignored: the report itself is usually one.
+      def require_clean_tree!(opts)
+        root = Fp::JUnit.repo_root_for(opts[:base_dir] || Dir.pwd)
+        stdout, _stderr, status = Open3.capture3('git', 'status', '--porcelain', '--untracked-files=no',
+                                                 chdir: root || Dir.pwd)
+        unless status.success?
+          output.error('--local needs a git checkout: the results are recorded against the commit that ran them.')
+          exit 1
+        end
+        return if stdout.strip.empty?
+
+        output.error('--local refuses a working tree with uncommitted changes: the results would be ' \
+                     'recorded against a commit that never ran them. Commit (and push) first, then re-run the suite.')
+        exit 1
+      end
+
       def resolve_commit_sha(opts)
         explicit = presence(opts[:sha])
         return explicit if explicit
@@ -403,6 +432,7 @@ module Fp
 
         output.success(data) do
           mode = opts[:pr] ? " (PR mode — passing tests recorded as pr_passing)" : ''
+          mode += ' (local run, no CI)' if opts[:local]
           puts "Ingested CI results for surface '#{opts[:surface]}'#{mode}:"
           totals[:reports].each do |report|
             report[:results].each do |r|

data/lib/fp/commands/comment.rb

@@ -17,7 +17,7 @@ module Fp
         source_type: :string, source_url: :string, source_context: :string
       }.freeze
 
-      VALID_SOURCE_TYPES = %w[fizzy discord github_issue jira notion manual ai_chat].freeze
+      VALID_SOURCE_TYPES = %w[fizzy discord github_issue jira notion manual ai_chat repo_scan].freeze
 
       help_summary "Comment on a requirement's open change proposal"
       help_usage 'fp comment <slug> --body "..." --project <slug>'

data/lib/fp/commands/coverage.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Fp
+  module Commands
+    # fp coverage [--project X] [--surface api] [--all] [--json]
+    #
+    # Draft coverage: which draft requirements have marked tests or evidence on which
+    # surfaces, and which have none.
+    #
+    # The gate measures active requirements only (plus ones an open proposal demoted), so
+    # on a freshly onboarded project — where every requirement is still a draft — `fp
+    # check` says "Nothing to enforce yet" and `fp list --gaps` says "No gaps found". The
+    # agent that just proposed the catalog and marked the tests had no way to confirm its
+    # work, and nothing to hand the human but "trust me" (Fizzy #1786). This reads the
+    # parity report's `drafts` section, where the server resolves each draft's cells from
+    # evidence exactly as it would once the draft is active.
+    #
+    # It ends with a plain list of drafts that have no evidence on a required surface —
+    # the "requirements with no test" section an onboarding agent hands the human.
+    #
+    # Informational only. Drafts are reported, never gated; this command always exits 0
+    # on a successful read.
+    class Coverage < Base
+      help_summary 'Show which draft requirements have evidence on which surfaces (never gated)'
+      help_usage 'fp coverage [--project <slug>] [--surface X] [--all]'
+      help_flag :project, 'Project slug (falls back to .featureparity.yml)'
+      help_flag :surface, 'Limit to one surface'
+      help_flag :all, 'Also list the measured requirements (active, or under review) the gate checks'
+      help_example 'fp coverage', 'Drafts × surfaces, then the drafts with no evidence'
+      help_example 'fp coverage --all --json'
+      help_note "Drafts are not enforced: a human activates a draft before fp check gates on it.\n" \
+                'A cell reads the same here as it will in the matrix once the draft is active.'
+
+      KNOWN_FLAGS = {
+        project: :string,
+        surface: :string,
+        all: :boolean
+      }.freeze
+
+      def run(args)
+        opts, = parse_flags(args, KNOWN_FLAGS)
+        project_config = ProjectConfig.discover
+        project_slug = require_project!(opts, project_config)
+        project_id = resolve_project_id(project_slug)
+        surface = opts[:surface] || project_config.surface
+
+        result = client.get_parity_with_drafts(project_id, surface: surface)
+        unless result[:ok]
+          output.error(result[:error], status: result[:status])
+          exit 1
+        end
+
+        report = result[:data]['parity'] || {}
+        unless report.key?('drafts')
+          output.error('This FeatureParity API does not report draft coverage yet (it predates ' \
+                       '`include_drafts`). Nothing was checked.')
+          exit 1
+        end
+
+        render(report, project_slug, surface, all: opts[:all])
+      end
+
+      # Drafts with no evidence on at least one required surface, as
+      # [{ 'slug', 'title', 'surfaces' => [...] }]. "No evidence" is the `missing` state:
+      # nothing — not even a stub or a red test — is filed for that surface.
+      def self.untested(drafts)
+        Array(drafts).filter_map do |row|
+          bare = Array(row['cells']).select { |c| c['state'] == 'missing' }.map { |c| c['surface'] }
+          next if bare.empty?
+
+          { 'slug' => row['slug'], 'title' => row['title'], 'surfaces' => bare }
+        end
+      end
+
+      # The drafts × surfaces table plus the "no evidence" list, for text output. Shared
+      # with `fp check --include-drafts`, so the two can't describe drafts differently.
+      def self.print_drafts(output, rows, surfaces, show_status: false)
+        if rows.empty?
+          puts 'No draft requirements.'
+          return
+        end
+
+        headers = %w[SLUG]
+        headers << 'STATUS' if show_status
+        headers += surfaces
+        table = rows.map do |row|
+          by_surface = Array(row['cells']).to_h { |c| [c['surface'], c['state']] }
+          cells = surfaces.map { |key| by_surface.key?(key) ? Parity.glyph(by_surface[key]) : Parity::NOT_REQUIRED }
+          line = [row['slug']]
+          line << row['status'] if show_status
+          line + cells
+        end
+        output.table(headers, table)
+        puts Parity::LEGEND
+      end
+
+      def self.print_untested(untested)
+        puts
+        if untested.empty?
+          puts 'Every draft has evidence on each of its required surfaces.'
+          return
+        end
+
+        puts "#{untested.size} draft(s) with no evidence on a required surface:"
+        untested.each { |u| puts "  #{u['slug']} — no evidence on: #{u['surfaces'].join(', ')}" }
+      end
+
+      private
+
+      def render(report, project_slug, surface, all:)
+        drafts = report['drafts'] || []
+        surfaces = report['surfaces'] || []
+        untested = self.class.untested(drafts)
+
+        payload = {
+          'project' => project_slug,
+          'surface' => surface,
+          'surfaces' => surfaces,
+          'drafts' => drafts,
+          'untested' => untested
+        }
+        payload['measured'] = report['requirements'] || [] if all
+
+        output.success(payload) do
+          scope = surface ? " (surface: #{surface})" : ''
+          puts "Draft coverage: #{project_slug}#{scope}"
+          puts
+
+          if all
+            measured = report['requirements'] || []
+            puts "Measured by the gate (#{measured.size}):"
+            if measured.empty?
+              puts 'None — no requirement is active yet.'
+            else
+              self.class.print_drafts(output, measured, surfaces, show_status: true)
+            end
+            puts
+            puts "Drafts (#{drafts.size}):"
+          end
+
+          self.class.print_drafts(output, drafts, surfaces)
+          self.class.print_untested(untested) unless drafts.empty?
+          puts
+          puts 'Drafts are reported, never gated. A human activates a draft before fp check enforces it.'
+        end
+      end
+    end
+  end
+end