metz-scan 0.5.3

data/lib/metz_scan/calibration/project_analyzer_evidence_runner/artifact_writer.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "time"
+
+module MetzScan
+  module Calibration
+    module ProjectAnalyzerEvidenceRunner
+      class ArtifactWriter
+        def initialize(summary:, output_dir:, run_id:)
+          @summary = summary
+          @output_dir = output_dir
+          @run_id = run_id || "project-analyzer-evidence-#{Time.now.utc.strftime('%Y%m%d-%H%M%S')}"
+        end
+
+        def call
+          FileUtils.mkdir_p(run_dir)
+          write_json
+          write_markdown
+          artifacts
+        end
+
+        private
+
+        attr_reader :summary, :output_dir, :run_id
+
+        def write_json
+          File.write(artifacts.fetch("json"), "#{JSON.pretty_generate(persisted_summary)}\n")
+        end
+
+        def write_markdown
+          File.write(artifacts.fetch("markdown"), MarkdownRenderer.new(persisted_summary).call)
+        end
+
+        def persisted_summary
+          summary.merge("artifacts" => artifacts)
+        end
+
+        def artifacts
+          { "directory" => run_dir, "json" => File.join(run_dir, "summary.json"),
+            "markdown" => File.join(run_dir, "summary.md") }
+        end
+
+        def run_dir
+          File.join(File.expand_path(output_dir), run_id)
+        end
+      end
+    end
+  end
+end

data/lib/metz_scan/calibration/project_analyzer_evidence_runner/baseline_delta.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+
+module MetzScan
+  module Calibration
+    module ProjectAnalyzerEvidenceRunner
+      class BaselineDelta
+        COUNT_KEYS = %w[finding_count offense_count].freeze
+        private_constant :COUNT_KEYS
+
+        def initialize(summary:, baseline_file:)
+          @summary = summary
+          @baseline = BaselineFile.load(baseline_file)
+        end
+
+        def to_h
+          validate_scope!
+          delta_payload
+        end
+
+        private
+
+        attr_reader :summary, :baseline
+
+        def delta_payload
+          { "baseline" => baseline.identity,
+            "finding_count" => metric_delta("finding_count", current_summary, baseline_summary),
+            "offense_count" => metric_delta("offense_count", current_summary, baseline_summary),
+            "rules" => rule_deltas,
+            "breakdowns" => breakdown_deltas }.compact
+        end
+
+        def validate_scope!
+          BaselineFile.scope_keys.each { |key| validate_scope_value!(key) if baseline_scope.key?(key) }
+        end
+
+        def validate_scope_value!(key)
+          return if normalized_scope_value(key, baseline_scope[key]) == normalized_scope_value(key, summary[key])
+
+          raise Error, "baseline scope mismatch for #{key}: expected #{baseline_scope[key].inspect}, " \
+                       "got #{summary[key].inspect}"
+        end
+
+        def normalized_scope_value(key, value)
+          case key
+          when "targets_file" then value && File.expand_path(value.to_s)
+          when "analyzer_filter" then Array(value).map(&:to_s).sort
+          else value
+          end
+        end
+
+        def rule_deltas
+          all_rule_names.map { |name| rule_delta_for(name) }
+        end
+
+        def rule_delta_for(name)
+          { "cop_name" => name,
+            "finding_count" => rule_metric_delta(name, "finding_count"),
+            "offense_count" => rule_metric_delta(name, "offense_count") }
+        end
+
+        def rule_metric_delta(name, key)
+          metric_delta(key, current_rules.fetch(name, {}), baseline_rules.fetch(name, {}))
+        end
+
+        def all_rule_names
+          (current_rules.keys | baseline_rules.keys).sort
+        end
+
+        def current_rules
+          @current_rules ||= rules_by_name(current_summary)
+        end
+
+        def baseline_rules
+          @baseline_rules ||= rules_by_name(baseline_summary)
+        end
+
+        def rules_by_name(source)
+          Array(rules_for(source)).to_h { |rule| [rule.fetch("cop_name"), rule] }
+        end
+
+        def rules_for(source)
+          BaselineFile.rules_for(source)
+        end
+
+        def breakdown_deltas
+          { "confidence" => breakdown_delta_for("confidence"),
+            "triage_severity" => breakdown_delta_for("triage_severity"),
+            "metadata" => metadata_breakdown_deltas }.compact
+        end
+
+        def metadata_breakdown_deltas
+          category_deltas = breakdown_delta_for("project_analyzer_category", metadata: true)
+          { "project_analyzer_category" => category_deltas } unless category_deltas.empty?
+        end
+
+        def breakdown_delta_for(key, metadata: false)
+          current = breakdown_counts(current_summary, key, metadata: metadata)
+          baseline = breakdown_counts(baseline_summary, key, metadata: metadata)
+
+          (current.keys | baseline.keys).sort.map do |value|
+            { "value" => value, "finding_count" => count_delta(current.fetch(value, 0), baseline.fetch(value, 0)) }
+          end
+        end
+
+        def breakdown_counts(source, key, metadata:)
+          values = metadata ? source.dig("breakdowns", "metadata", key) : source.dig("breakdowns", key)
+          Array(values).to_h { |entry| [entry.fetch("value"), entry.fetch("finding_count").to_i] }
+        end
+
+        def metric_delta(key, current, baseline)
+          count_delta(current.fetch(key, 0), baseline.fetch(key, 0))
+        end
+
+        def count_delta(current, baseline)
+          current = current.to_i
+          baseline = baseline.to_i
+          { "baseline" => baseline, "current" => current, "delta" => current - baseline }
+        end
+
+        def current_summary = summary
+
+        def baseline_summary
+          baseline.summary
+        end
+
+        def baseline_scope
+          baseline.scope
+        end
+      end
+
+      class BaselineFile
+        SCOPE_KEYS = %w[targets_file default_output analyzer_filter].freeze
+        private_constant :SCOPE_KEYS
+
+        def self.load(path)
+          new(path: path)
+        end
+
+        def self.from_summary(summary, label:, generated_from:, targets_file: nil)
+          document = document_for(summary, label: label, generated_from: generated_from, targets_file: targets_file)
+          new(document: document)
+        end
+
+        def self.scope_keys = SCOPE_KEYS
+
+        def self.rules_for(source)
+          source["rules"] || source.dig("project_analyzers", "rules") || []
+        end
+
+        def initialize(path: nil, document: nil)
+          @path = path && File.expand_path(path)
+          @document = document
+        end
+
+        attr_reader :path
+
+        def identity
+          { "file" => path, "label" => document["label"], "generated_from" => document["generated_from"] }.compact
+        end
+
+        def to_h
+          document
+        end
+
+        def to_yaml
+          YAML.dump(to_h)
+        end
+
+        def summary
+          document.fetch("summary", document)
+        end
+
+        def scope
+          document.fetch("scope", {})
+        end
+
+        class << self
+          private
+
+          def document_for(summary, label:, generated_from:, targets_file:)
+            { "label" => label, "generated_from" => generated_from,
+              "scope" => scope_for(summary, targets_file: targets_file),
+              "summary" => compact_summary_for(summary) }
+          end
+
+          def scope_for(summary, targets_file:)
+            { "targets_file" => targets_file || summary["targets_file"],
+              "default_output" => summary.fetch("default_output"),
+              "analyzer_filter" => summary.fetch("analyzer_filter", []) }
+          end
+
+          def compact_summary_for(summary)
+            { "finding_count" => summary.fetch("finding_count"),
+              "offense_count" => summary.fetch("offense_count"),
+              "rules" => compact_rules_for(summary),
+              "breakdowns" => summary.fetch("breakdowns", {}) }
+          end
+
+          def compact_rules_for(summary)
+            rules_for(summary).map { |rule| compact_rule_for(rule) }
+          end
+
+          def compact_rule_for(rule)
+            { "cop_name" => rule.fetch("cop_name"),
+              "finding_count" => rule.fetch("finding_count"),
+              "offense_count" => rule.fetch("offense_count") }
+          end
+        end
+
+        private
+
+        def document
+          @document ||= load_document
+        end
+
+        def load_document
+          JSON.parse(contents)
+        rescue JSON::ParserError
+          YAML.safe_load(contents, aliases: false)
+        end
+
+        def contents
+          @contents ||= File.read(path)
+        end
+      end
+    end
+  end
+end

data/lib/metz_scan/calibration/project_analyzer_evidence_runner/collaborators.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "../../project_index"
+require_relative "../../commands/scan/project_analyzer_runner"
+
+module MetzScan
+  module Calibration
+    module ProjectAnalyzerEvidenceRunner
+      class AnalyzerSelection
+        def initialize(rule_ids)
+          @rule_ids = Array(rule_ids).compact
+        end
+
+        def call
+          return nil if rule_ids.empty?
+
+          rule_ids.map { |rule_id| analyzer_for(rule_id) }
+        end
+
+        private
+
+        attr_reader :rule_ids
+
+        def analyzer_for(rule_id)
+          analyzer = analyzers_by_rule_id[rule_id]
+          raise Error, "unknown project analyzer: #{rule_id}" unless analyzer
+
+          analyzer
+        end
+
+        def analyzers_by_rule_id
+          Commands::Scan::ProjectAnalyzerRunner::ANALYZERS.to_h { |analyzer| [analyzer::RULE_ID, analyzer] }
+        end
+      end
+
+      class FindingRunner
+        def initialize(analyzers: nil)
+          @analyzers = analyzers
+        end
+
+        def call(paths, index:, default_output:)
+          return project_findings(paths, index: index, default_output: default_output) unless analyzers
+
+          selected = selected_analyzers(default_output)
+          filter_default_output(run_selected_analyzers(selected, paths, index), default_output)
+        end
+
+        private
+
+        attr_reader :analyzers
+
+        def project_findings(paths, index:, default_output:)
+          Commands::Scan::ProjectAnalyzerRunner.project_findings_for(
+            paths, index: index, default_output: default_output
+          )
+        end
+
+        def selected_analyzers(default_output)
+          return analyzers unless default_output
+
+          analyzers.select { |analyzer| Commands::Scan::ProjectAnalyzerRunner.default_output_analyzer?(analyzer) }
+        end
+
+        def run_selected_analyzers(selected, paths, index)
+          selected.flat_map { |analyzer| analyzer.new(paths: paths, index: index).call }
+        end
+
+        def filter_default_output(findings, default_output)
+          return findings unless default_output
+
+          findings.select { |finding| Commands::Scan::ProjectAnalyzerRunner.default_output_finding?(finding) }
+        end
+      end
+    end
+  end
+end

data/lib/metz_scan/calibration/project_analyzer_evidence_runner/markdown_renderer.rb

@@ -0,0 +1,353 @@
+# frozen_string_literal: true
+
+module MetzScan
+  module Calibration
+    module ProjectAnalyzerEvidenceRunner
+      class MarkdownRenderer
+        def initialize(summary)
+          @summary = summary
+        end
+
+        def call
+          "#{sections.join("\n")}\n"
+        end
+
+        private
+
+        attr_reader :summary
+
+        def sections
+          section_renderers.flat_map { |renderer| renderer.new(summary).call }
+        end
+
+        def section_renderers
+          [HeaderMarkdown, TargetsMarkdown, RulesMarkdown, ReadinessMarkdown,
+           NotableFindingsMarkdown, BaselineDeltaMarkdown, BreakdownsMarkdown]
+        end
+      end
+
+      module MarkdownTableCells
+        private
+
+        def cell(value)
+          value.to_s.gsub("|", "\\|")
+        end
+      end
+
+      class HeaderMarkdown
+        def initialize(summary)
+          @summary = summary
+        end
+
+        def call
+          ["# Project analyzer evidence", "", *metadata_lines, "", *count_lines, ""]
+        end
+
+        private
+
+        attr_reader :summary
+
+        def metadata_lines
+          ["Generated: #{summary.fetch('generated_at')}",
+           "Default output filter: #{summary.fetch('default_output')}",
+           "Analyzer filter: #{analyzer_filter_label}", *targets_file_lines,
+           "Fixture root: `#{summary.fetch('fixture_root')}`"]
+        end
+
+        def count_lines
+          ["Findings: #{summary.fetch('finding_count')}", "Offenses: #{summary.fetch('offense_count')}"]
+        end
+
+        def analyzer_filter_label
+          analyzer_names = summary.fetch("analyzer_filter", [])
+          analyzer_names.empty? ? "none" : analyzer_names.join(", ")
+        end
+
+        def targets_file_lines
+          targets_file = summary["targets_file"]
+          targets_file ? ["Targets file: `#{targets_file}`"] : []
+        end
+      end
+
+      class TargetsMarkdown
+        def initialize(summary)
+          @summary = summary
+        end
+
+        def call
+          ["## Targets", "", header, *rows, ""]
+        end
+
+        private
+
+        attr_reader :summary
+
+        def header
+          "| Target | Revision | Scan paths | Backend | Findings | Offenses |\n" \
+            "| --- | --- | --- | --- | ---: | ---: |"
+        end
+
+        def rows
+          summary.fetch("targets").map { |target| row(target) }
+        end
+
+        def row(target)
+          "| #{target.fetch('name')} | #{revision_label(target)} | #{scan_paths_label(target)} | " \
+            "#{target.fetch('index').fetch('backend')} | #{target.fetch('finding_count')} | " \
+            "#{target.fetch('offense_count')} |"
+        end
+
+        def revision_label(target)
+          target.fetch("git").fetch("revision", "unknown")[0, 12]
+        end
+
+        def scan_paths_label(target)
+          target.fetch("scan_paths").map { |path| "`#{path}`" }.join("<br>")
+        end
+      end
+
+      class RulesMarkdown
+        def initialize(summary)
+          @summary = summary
+        end
+
+        def call
+          return ["## Rules", "", "No project-analyzer findings."] if rules.empty?
+
+          ["## Rules", "", header, *rows]
+        end
+
+        private
+
+        attr_reader :summary
+
+        def rules
+          summary.fetch("project_analyzers").fetch("rules", [])
+        end
+
+        def header
+          "| Rule | Findings | Offenses | Status | Confidence | Severity |\n" \
+            "| --- | ---: | ---: | --- | --- | --- |"
+        end
+
+        def rows
+          rules.map { |rule| row(rule) }
+        end
+
+        def row(rule)
+          "| #{rule.fetch('cop_name')} | #{rule.fetch('finding_count')} | #{rule.fetch('offense_count')} | " \
+            "#{rule.fetch('status', '?')} | #{rule.fetch('confidence', '?')} | " \
+            "#{rule.fetch('triage_severity', '?')} |"
+        end
+      end
+
+      class ReadinessMarkdown
+        include MarkdownTableCells
+
+        def initialize(summary)
+          @summary = summary
+        end
+
+        def call
+          return [] if readiness.empty?
+
+          ["", "## Readiness", "", header, *rows, ""]
+        end
+
+        private
+
+        attr_reader :summary
+
+        def readiness
+          summary.fetch("project_analyzers").fetch("readiness", [])
+        end
+
+        def header
+          "| Rule | Disposition | Evidence | Next | Not next |\n" \
+            "| --- | --- | --- | --- | --- |"
+        end
+
+        def rows
+          readiness.map { |entry| row(entry) }
+        end
+
+        def row(entry)
+          "| #{cell(entry.fetch('rule_id'))} | #{cell(entry.fetch('disposition'))} | " \
+            "#{cell(entry.fetch('evidence'))} | #{cell(entry.fetch('next'))} | " \
+            "#{cell(entry.fetch('not_next'))} |"
+        end
+      end
+
+      class NotableFindingsMarkdown
+        include MarkdownTableCells
+
+        def initialize(summary)
+          @notable_findings = summary.fetch("notable_findings", [])
+        end
+
+        def call
+          return [] if notable_findings.empty?
+
+          ["", "## Notable Findings", "", header, *rows, ""]
+        end
+
+        private
+
+        attr_reader :notable_findings
+
+        def header
+          "| Target | Rule | Confidence | Severity | Category | Location | Message |\n" \
+            "| --- | --- | --- | --- | --- | --- | --- |"
+        end
+
+        def rows
+          notable_findings.map { |finding| row(finding) }
+        end
+
+        def row(finding)
+          "| #{cell(finding.fetch('target'))} | #{cell(finding.fetch('rule_id'))} | " \
+            "#{cell(finding.fetch('confidence', '?'))} | #{cell(finding.fetch('triage_severity', '?'))} | " \
+            "#{cell(finding.fetch('category', '?'))} | #{cell(location_label(finding))} | " \
+            "#{cell(finding.fetch('message'))} |"
+        end
+
+        def location_label(finding)
+          occurrence = finding.fetch("occurrence", {})
+          [occurrence["path"], occurrence["line"]].compact.join(":")
+        end
+      end
+
+      class BreakdownsMarkdown
+        def initialize(summary)
+          @breakdowns = summary.fetch("breakdowns", {})
+        end
+
+        def call
+          return [] if breakdowns.empty?
+
+          ["", "## Breakdowns", "", *subsections.flatten]
+        end
+
+        private
+
+        attr_reader :breakdowns
+
+        def subsections
+          [table("Confidence", breakdowns["confidence"]),
+           table("Severity", breakdowns["triage_severity"]),
+           *metadata_tables].compact
+        end
+
+        def metadata_tables
+          breakdowns.fetch("metadata", {}).map { |key, values| table("Metadata: `#{key}`", values) }
+        end
+
+        def table(title, values)
+          return if values.to_a.empty?
+
+          ["### #{title}", "", "| Value | Findings |", "| --- | ---: |",
+           *values.map { |value| row(value) }, ""]
+        end
+
+        def row(value)
+          "| #{value.fetch('value')} | #{value.fetch('finding_count')} |"
+        end
+      end
+
+      class BaselineDeltaMarkdown
+        include MarkdownTableCells
+
+        def initialize(summary)
+          @delta = summary["baseline_delta"]
+        end
+
+        def call
+          return [] unless delta
+
+          header_lines + totals_lines + detail_lines
+        end
+
+        private
+
+        attr_reader :delta
+
+        def baseline_label
+          baseline = delta.fetch("baseline")
+          label = baseline["label"] ? "#{baseline['label']} " : ""
+          "#{label}`#{baseline.fetch('file')}`"
+        end
+
+        def header_lines
+          ["", "## Baseline Deltas", "", "Baseline: #{baseline_label}", ""]
+        end
+
+        def totals_lines
+          [totals_header, total_row("Findings", delta.fetch("finding_count")),
+           total_row("Offenses", delta.fetch("offense_count")), ""]
+        end
+
+        def detail_lines
+          analyzer_table + confidence_table + severity_table + category_table
+        end
+
+        def analyzer_table
+          table("Analyzer", "Rule", analyzer_rows)
+        end
+
+        def confidence_table
+          table("Confidence", "Value", breakdown_rows(delta.dig("breakdowns", "confidence")))
+        end
+
+        def severity_table
+          table("Severity", "Value", breakdown_rows(delta.dig("breakdowns", "triage_severity")))
+        end
+
+        def category_table
+          table("Category", "Value", breakdown_rows(delta.dig("breakdowns", "metadata",
+                                                              "project_analyzer_category")))
+        end
+
+        def totals_header
+          "| Metric | Baseline | Current | Delta |\n| --- | ---: | ---: | ---: |"
+        end
+
+        def total_row(label, count)
+          "| #{label} | #{count.fetch('baseline')} | #{count.fetch('current')} | #{signed(count)} |"
+        end
+
+        def analyzer_rows
+          delta.fetch("rules", []).map do |rule|
+            ["#{cell(rule.fetch('cop_name'))} findings", rule.fetch("finding_count")]
+          end + delta.fetch("rules", []).map do |rule|
+            ["#{cell(rule.fetch('cop_name'))} offenses", rule.fetch("offense_count")]
+          end
+        end
+
+        def breakdown_rows(entries)
+          Array(entries).map { |entry| [cell(entry.fetch("value")), entry.fetch("finding_count")] }
+        end
+
+        def table(title, value_header, rows)
+          changed = rows.select { |_value, count| count.fetch("delta").nonzero? }
+          return ["### #{title}", "", "No #{title.downcase} count changes.", ""] if changed.empty?
+
+          ["### #{title}", "", "| #{value_header} | Baseline | Current | Delta |",
+           "| --- | ---: | ---: | ---: |",
+           *changed.map { |value, count| row(value, count) }, ""]
+        end
+
+        def row(value, count)
+          "| #{value} | #{count.fetch('baseline')} | #{count.fetch('current')} | #{signed(count)} |"
+        end
+
+        def signed(count)
+          delta_value = count.fetch("delta")
+          delta_value.positive? ? "+#{delta_value}" : delta_value.to_s
+        end
+      end
+
+      private_constant :MarkdownTableCells, :HeaderMarkdown, :TargetsMarkdown, :RulesMarkdown
+      private_constant :ReadinessMarkdown, :NotableFindingsMarkdown, :BaselineDeltaMarkdown, :BreakdownsMarkdown
+    end
+  end
+end