necropsy 0.2.0 → 0.3.0

data/lib/necropsy/bench/precision_gate.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Bench
+    class PrecisionGate
+      MAX_FEATURES = 100
+      IMPROVEMENT_DIRECTIONS = {
+        'candidate_precision' => :increase,
+        'known_positive_recall' => :increase,
+        'candidate_count' => :increase,
+        'candidate_loc' => :increase,
+        'blocked_count' => :decrease,
+        'blocked_rate' => :decrease,
+        'unknown_finding_count' => :decrease,
+        'unknown_finding_rate' => :decrease,
+        'unknown_resolution_rate' => :decrease
+      }.freeze
+
+      def initialize(policy:, candidate_union_summary:, feature_ablation:)
+        @policy = policy
+        @candidate_union_summary = candidate_union_summary
+        @feature_ablation = feature_ablation
+      end
+
+      def call
+        validate!
+        quality = quality_status
+        features = feature_status
+        checks = {
+          'precision' => quality.fetch('precision_passed'),
+          'candidate_yield' => quality.fetch('yield_passed'),
+          'default_features_evaluated' => features.values.all? { |feature| feature['evaluated'] },
+          'default_features_improve' => features.values.all? { |feature| feature['passed'] }
+        }
+        {
+          'schema_version' => 1,
+          'enforced' => true,
+          'policy' => normalized_policy,
+          'quality' => quality,
+          'features' => features,
+          'checks' => checks,
+          'passed' => checks.values.all?
+        }
+      end
+
+      private
+
+      attr_reader :policy, :candidate_union_summary, :feature_ablation
+
+      def validate!
+        raise Error, 'Precision gate policy must be a mapping' unless policy.is_a?(Hash)
+        raise Error, 'Candidate-union summary must be a mapping' unless candidate_union_summary.is_a?(Hash)
+        raise Error, 'Feature ablation results must be a mapping' unless feature_ablation.is_a?(Hash)
+        raise Error, "Feature ablation exceeds #{MAX_FEATURES} entries" if feature_ablation.length > MAX_FEATURES
+
+        threshold = minimum_precision
+        raise Error, 'Precision gate minimum_precision must be between 0.0 and 1.0' unless threshold.between?(0.0, 1.0)
+
+        features = default_features
+        raise Error, 'Precision gate default_features must not be empty' if features.empty?
+        raise Error, "Precision gate default_features exceed #{MAX_FEATURES} entries" if features.length > MAX_FEATURES
+        raise Error, 'Precision gate default_features must be unique strings' unless
+          features.all? { |name| name.is_a?(String) && !name.empty? } && features.uniq == features
+      end
+
+      def normalized_policy
+        {
+          'minimum_precision' => minimum_precision,
+          'default_features' => default_features.sort
+        }
+      end
+
+      def minimum_precision
+        Float(policy.fetch('minimum_precision', 0.85))
+      rescue ArgumentError, TypeError
+        Float::NAN
+      end
+
+      def default_features
+        value = policy['default_features']
+        raise Error, 'Precision gate default_features must be an array' unless value.is_a?(Array)
+
+        value
+      end
+
+      def quality_status
+        metrics = candidate_union_summary.dig('tool_metrics', 'necropsy')
+        metrics = {} unless metrics.is_a?(Hash)
+        precision = metrics['candidate_precision']
+        candidate_count = nonnegative_integer(metrics['candidate_count'])
+        candidate_loc = nonnegative_integer(metrics['candidate_loc'])
+        measured = metrics['precision_status'] == 'measured' && precision.is_a?(Numeric) && precision.finite?
+        {
+          'candidate_precision' => precision,
+          'minimum_precision' => minimum_precision,
+          'precision_status' => metrics['precision_status'] || 'missing',
+          'candidate_count' => candidate_count,
+          'candidate_loc' => candidate_loc,
+          'precision_passed' => measured && precision >= minimum_precision,
+          'yield_passed' => candidate_count&.positive? && candidate_loc&.positive?
+        }
+      end
+
+      def feature_status
+        default_features.sort.to_h do |name|
+          result = feature_ablation[name]
+          difference = result['difference'] if result.is_a?(Hash)
+          evaluated = result.is_a?(Hash) && result['on'].is_a?(Hash) && result['off'].is_a?(Hash) &&
+                      difference.is_a?(Hash)
+          precision_delta = numeric_delta(difference, 'candidate_precision')
+          improved_metrics = evaluated ? improved_metrics(difference) : []
+          precision_preserved = evaluated && !precision_delta.nil? && precision_delta >= 0
+          [name, {
+            'evaluated' => evaluated,
+            'precision_delta' => precision_delta,
+            'precision_preserved' => precision_preserved,
+            'improved_metrics' => improved_metrics,
+            'passed' => precision_preserved && !improved_metrics.empty?
+          }]
+        end
+      end
+
+      def improved_metrics(difference)
+        IMPROVEMENT_DIRECTIONS.filter_map do |metric, direction|
+          delta = numeric_delta(difference, metric)
+          next if delta.nil? || delta.zero?
+
+          metric if (direction == :increase && delta.positive?) || (direction == :decrease && delta.negative?)
+        end
+      end
+
+      def numeric_delta(difference, key)
+        return nil unless difference.is_a?(Hash)
+
+        value = difference[key]
+        value if value.is_a?(Numeric) && value.finite?
+      end
+
+      def nonnegative_integer(value)
+        value if value.is_a?(Integer) && value >= 0
+      end
+    end
+  end
+end

data/lib/necropsy/bench/release_audit/adversarial_runner.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Necropsy
+  module Bench
+    class ReleaseAudit
+      class AdversarialRunner
+        def initialize(root:, suites:, clock: Process.method(:clock_gettime))
+          @root = root
+          @suites = suites
+          @clock = clock
+        end
+
+        def call
+          suites.sort.map do |name, definition|
+            command = definition.fetch('command').map(&:to_s)
+            started_at = monotonic_time
+            stdout, stderr, status = Open3.capture3(*command, chdir: root)
+            {
+              'name' => name,
+              'command' => command,
+              'passed' => status.success?,
+              'exit_status' => status.exitstatus,
+              'duration_seconds' => (monotonic_time - started_at).round(6),
+              'summary' => result_summary(stdout, stderr)
+            }
+          rescue SystemCallError => e
+            {
+              'name' => name,
+              'command' => command,
+              'passed' => false,
+              'exit_status' => nil,
+              'duration_seconds' => (monotonic_time - started_at).round(6),
+              'summary' => "#{e.class}: #{e.message}"
+            }
+          end
+        end
+
+        private
+
+        attr_reader :root, :suites, :clock
+
+        def monotonic_time
+          clock.call(Process::CLOCK_MONOTONIC)
+        end
+
+        def result_summary(stdout, stderr)
+          combined = [stdout, stderr].join("\n")
+          combined.lines.map(&:strip).reverse.find { |line| line.match?(/\d+ examples?, \d+ failures?/) } ||
+            combined.lines.map(&:strip).reject(&:empty?).last.to_s
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/bench/release_audit/artifact_writer.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module Necropsy
+  module Bench
+    class ReleaseAudit
+      class ArtifactWriter
+        def initialize(audit:, output_dir:)
+          @audit = audit
+          @output_dir = output_dir
+        end
+
+        def call
+          FileUtils.mkdir_p(output_dir)
+          File.write(json_path, "#{JSON.pretty_generate(audit)}\n")
+          File.write(markdown_path, markdown)
+          { json: json_path, markdown: markdown_path }
+        end
+
+        private
+
+        attr_reader :audit, :output_dir
+
+        def json_path = File.join(output_dir, 'audit.json')
+        def markdown_path = File.join(output_dir, 'audit.md')
+
+        def markdown
+          lines = [
+            "# #{audit.fetch('release')} safety release audit",
+            '',
+            "Status: **#{audit.fetch('status').upcase}**",
+            '',
+            "Baseline: `#{audit.dig('baseline', 'git_ref')}` — #{audit.dig('baseline', 'reason')}",
+            '',
+            '## Candidate changes',
+            '',
+            '| corpus | baseline | current | added | removed | state changed | newly high |',
+            '|---|---:|---:|---:|---:|---:|---:|'
+          ]
+          append_candidate_rows(lines)
+          append_review(lines)
+          append_performance(lines)
+          append_adversarial(lines)
+          append_claim_gate(lines)
+          append_gates(lines)
+          "#{lines.join("\n")}\n"
+        end
+
+        def append_review(lines)
+          lines.push('', '## Difference review', '',
+                     '| corpus | strategy | changes | required | completed | zero difference |',
+                     '|---|---|---:|---:|---:|:---:|')
+          audit.dig('review', 'coverage').sort.each do |corpus, coverage|
+            lines << "| #{corpus} | #{coverage['strategy']} | #{coverage['changes']} | #{coverage['required']} | " \
+                     "#{coverage['completed']} | #{coverage['zero_difference'] ? 'yes' : 'no'} |"
+          end
+        end
+
+        def append_candidate_rows(lines)
+          new_high = audit.fetch('new_high_candidates').group_by { |candidate| candidate.fetch('corpus') }
+          audit.fetch('corpora').sort.each do |corpus, comparison|
+            lines << "| #{corpus} | #{comparison.dig('baseline_metrics', 'findings')} | " \
+                     "#{comparison.dig('current_metrics', 'findings')} | #{comparison['added'].length} | " \
+                     "#{comparison['removed'].length} | #{comparison['state_changed'].length} | " \
+                     "#{new_high.fetch(corpus, []).length} |"
+          end
+        end
+
+        def append_performance(lines)
+          lines.push('', '## Performance', '',
+                     '| corpus | wall baseline/current/limit (s) | RSS baseline/current/limit (KiB) | pass |',
+                     '|---|---:|---:|:---:|')
+          audit.fetch('performance').sort.each do |corpus, result|
+            lines << "| #{corpus} | #{result['baseline_wall_time_seconds']} / " \
+                     "#{result['current_wall_time_seconds']} / #{result['wall_time_limit_seconds']} | " \
+                     "#{result['baseline_rss_kb']} / #{result['current_rss_kb']} / #{result['rss_limit_kb']} | " \
+                     "#{result['passed'] ? 'yes' : 'no'} |"
+          end
+        end
+
+        def append_adversarial(lines)
+          lines.push('', '## Adversarial suites', '', '| suite | result | summary |', '|---|:---:|---|')
+          audit.fetch('adversarial_suites').each do |suite|
+            lines << "| #{suite['name']} | #{suite['passed'] ? 'pass' : 'fail'} | #{suite['summary']} |"
+          end
+        end
+
+        def append_claim_gate(lines)
+          claim = audit.fetch('claim_gate')
+          return unless claim['enforced']
+
+          lines.push('', '## Public claim gate', '',
+                     "Result: **#{claim['passed'] ? 'PASS' : 'FAIL'}**",
+                     "Unexplained high candidates: #{claim.fetch('unexplained_high_candidates').length}",
+                     "Reviewed high candidates: #{claim.fetch('reviewed_high_candidates')}")
+        end
+
+        def append_gates(lines)
+          lines.push('', '## Release gates', '', '| gate | result | failures |', '|---|:---:|---:|')
+          audit.fetch('gates').sort.each do |name, gate|
+            lines << "| #{name} | #{gate['passed'] ? 'pass' : 'fail'} | #{gate['failures']} |"
+          end
+          lines << ''
+          lines << "Reviewed RuboCop/Rails changes: #{audit.dig('review', 'completed').length}; " \
+                   "missing: #{audit.dig('review', 'missing').length}."
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/bench/release_audit/config_validator.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'rubygems'
+
+module Necropsy
+  module Bench
+    class ReleaseAudit
+      class ConfigValidator
+        REQUIRED_CORPORA = %w[dynamic_evidence plain_ruby rails rubocop_1_75_0 self].freeze
+        REQUIRED_ADVERSARIAL_SUITES = %w[ambiguity dynamic parse remote_input].freeze
+        REQUIRED_REVIEW_POLICIES = %w[rails rubocop_1_75_0].freeze
+
+        def initialize(config, strict_release: true)
+          @config = config
+          @strict_release = strict_release
+        end
+
+        def validate!
+          raise Error, 'Release audit config must be a mapping' unless config.is_a?(Hash)
+          raise Error, 'Release audit schema_version must be 1' unless config['schema_version'] == 1
+
+          validate_nonempty_unique('corpora', config['corpora'])
+          validate_suites
+          validate_review_policies
+          validate_precision_gate
+          validate_strict_release if strict_release
+          config
+        end
+
+        private
+
+        attr_reader :config, :strict_release
+
+        def validate_nonempty_unique(label, values)
+          items = Array(values)
+          raise Error, "Release audit #{label} must not be empty" if items.empty?
+          return if items.uniq.length == items.length
+
+          raise Error, "Release audit #{label} must not contain duplicates"
+        end
+
+        def validate_suites
+          suites = config['adversarial_suites']
+          raise Error, 'Release audit adversarial_suites must not be empty' unless suites.is_a?(Hash) && !suites.empty?
+
+          suites.each do |name, definition|
+            command = definition.is_a?(Hash) ? definition['command'] : nil
+            next unless Array(command).empty?
+
+            raise Error, "Adversarial suite #{name} must define a non-empty command"
+          end
+        end
+
+        def validate_review_policies
+          policies = config.dig('review', 'corpora')
+          raise Error, 'Release audit review policies must not be empty' unless policies.is_a?(Hash) && !policies.empty?
+
+          policies.each do |corpus, policy|
+            strategy = policy['strategy']
+            raise Error, "Invalid review strategy for #{corpus}" unless %w[all stratified].include?(strategy)
+            next unless strategy == 'stratified'
+
+            minimum = Integer(policy['minimum_per_stratum'], exception: false)
+            raise Error, "Review sample size for #{corpus} must be positive" unless minimum&.positive?
+          end
+        end
+
+        def validate_precision_gate
+          required = Gem::Version.new(config.fetch('release', '0')) >= Gem::Version.new('0.4.0')
+          policy = config['precision_gate']
+          raise Error, 'Release audit 0.4+ requires precision_gate policy' if required && !policy.is_a?(Hash)
+          return unless policy
+          raise Error, 'Release audit precision_gate must be a mapping' unless policy.is_a?(Hash)
+
+          threshold = Float(policy.fetch('minimum_precision', 0.85), exception: false)
+          raise Error, 'Release audit minimum_precision must be between 0.0 and 1.0' unless
+            threshold&.finite? && threshold.between?(0.0, 1.0)
+
+          features = policy['default_features']
+          raise Error, 'Release audit default_features must be a non-empty array' unless
+            features.is_a?(Array) && !features.empty?
+          return if features.all? { |feature| feature.is_a?(String) && !feature.empty? } && features.uniq == features
+
+          raise Error, 'Release audit default_features must contain unique strings'
+        rescue ArgumentError
+          raise Error, "Invalid release version #{config['release'].inspect}"
+        end
+
+        def validate_strict_release
+          baseline_ref = config.dig('baseline', 'git_ref').to_s
+          raise Error, 'Release audit baseline must use a full Git SHA' unless baseline_ref.match?(/\A[0-9a-f]{40}\z/)
+
+          validate_exact_set('corpora', config.fetch('corpora'), REQUIRED_CORPORA)
+          validate_exact_set('adversarial suites', config.fetch('adversarial_suites').keys,
+                             REQUIRED_ADVERSARIAL_SUITES)
+          policies = config.dig('review', 'corpora')
+          validate_exact_set('review policies', policies.keys, REQUIRED_REVIEW_POLICIES)
+          raise Error, 'Rails release review must inspect all changes' unless policies.dig('rails', 'strategy') == 'all'
+          return if policies.dig('rubocop_1_75_0', 'strategy') == 'stratified'
+
+          raise Error, 'RuboCop release review must use stratified sampling'
+        end
+
+        def validate_exact_set(label, actual, expected)
+          return if Array(actual).sort == expected.sort
+
+          raise Error, "Release audit #{label} must be exactly #{expected.sort.inspect}"
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/bench/release_audit/git_snapshot.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'open3'
+
+module Necropsy
+  module Bench
+    class ReleaseAudit
+      class GitSnapshot
+        def initialize(root:, git_ref:, reports_path:)
+          @root = root
+          @git_ref = git_ref
+          @reports_path = reports_path
+        end
+
+        def reports(corpora)
+          corpora.to_h { |corpus| [corpus, report(corpus)] }
+        end
+
+        private
+
+        attr_reader :root, :git_ref, :reports_path
+
+        def report(corpus)
+          object = "#{git_ref}:#{File.join(reports_path, "#{corpus}.json")}"
+          output, status = Open3.capture2e('git', 'show', object, chdir: root)
+          raise Error, "Could not read audit baseline #{object}: #{output.strip}" unless status.success?
+
+          JSON.parse(output)
+        rescue JSON::ParserError => e
+          raise Error, "Could not parse audit baseline #{object}: #{e.message}"
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/bench/release_audit/performance_gate.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Bench
+    class ReleaseAudit
+      class PerformanceGate
+        PROVENANCE_FIELDS = %w[ruby os command rss_kind rss_scope].freeze
+
+        def initialize(config:, baseline:, current_summary:, current_provenance:)
+          @config = config
+          @baseline_document = baseline
+          @baseline = baseline.fetch('corpora')
+          @current = current_summary.fetch('corpora').to_h { |corpus| [corpus.fetch('id'), corpus] }
+          @current_provenance = current_provenance
+        end
+
+        def call
+          config.fetch('corpora').sort.to_h do |corpus|
+            [corpus, compare_or_unavailable(corpus)]
+          end
+        end
+
+        private
+
+        attr_reader :config, :baseline_document, :baseline, :current, :current_provenance
+
+        def compare_or_unavailable(corpus)
+          return unavailable(provenance_error) if provenance_error
+
+          baseline_result = baseline[corpus]
+          current_run = current[corpus]
+          return unavailable('baseline measurement unavailable') unless baseline_result
+          return unavailable('current corpus was not generated') unless current_run&.fetch('status', 'generated') == 'generated'
+
+          current_result = current_run['performance']
+          return unavailable('current performance measurement unavailable') unless current_result
+          return unavailable('current RSS measurement unavailable') unless rss_value(current_result)
+          return unavailable('current RSS kind differs from the baseline') unless compatible_rss?(current_result)
+
+          compare(corpus, baseline_result, current_result)
+        rescue KeyError, ArgumentError, TypeError => e
+          unavailable("invalid performance measurement: #{e.message}")
+        end
+
+        def compare(corpus, baseline_result, current_result)
+          budget = config.fetch('performance')
+          baseline_wall = Float(baseline_result.fetch('wall_time_seconds'))
+          current_wall = Float(current_result.fetch('wall_time_seconds'))
+          baseline_p95 = Float(baseline_result.fetch('wall_time_p95_seconds', baseline_wall))
+          current_p95 = Float(current_result.fetch('wall_time_p95_seconds', current_wall))
+          baseline_max = Float(baseline_result.fetch('wall_time_max_seconds', baseline_wall))
+          current_max = Float(current_result.fetch('wall_time_max_seconds', current_wall))
+          baseline_rss = Integer(baseline_result.fetch('rss_kb'))
+          current_rss = rss_value(current_result)
+          wall_limit = relative_limit(
+            baseline_wall,
+            ratio: budget.fetch('wall_time_ratio'),
+            allowance: budget.fetch('wall_time_allowance_seconds'),
+            absolute: budget.fetch('max_wall_time_seconds').fetch(corpus)
+          )
+          p95_limit = relative_limit(
+            baseline_p95,
+            ratio: budget.fetch('wall_time_ratio'),
+            allowance: budget.fetch('wall_time_allowance_seconds'),
+            absolute: budget.fetch('max_wall_time_seconds').fetch(corpus)
+          )
+          max_limit = relative_limit(
+            baseline_max,
+            ratio: budget.fetch('wall_time_ratio'),
+            allowance: budget.fetch('wall_time_allowance_seconds'),
+            absolute: budget.fetch('max_wall_time_seconds').fetch(corpus)
+          )
+          rss_limit = relative_limit(
+            baseline_rss,
+            ratio: budget.fetch('rss_ratio'),
+            allowance: budget.fetch('rss_allowance_kb'),
+            absolute: budget.fetch('max_rss_kb')
+          )
+          allocations = bounded_metric(
+            current_result, 'allocated_objects_max', budget['max_allocated_objects'], corpus
+          )
+          artifact = bounded_metric(
+            current_result, 'artifact_size_max_bytes', budget['max_artifact_size_bytes'], corpus
+          )
+          passed = current_wall <= wall_limit && current_p95 <= p95_limit && current_max <= max_limit &&
+                   current_rss <= rss_limit && allocations.fetch('passed') && artifact.fetch('passed')
+          {
+            'available' => true,
+            'baseline_wall_time_seconds' => baseline_wall,
+            'current_wall_time_seconds' => current_wall,
+            'wall_time_limit_seconds' => wall_limit.round(6),
+            'current_wall_time_p95_seconds' => current_p95,
+            'wall_time_p95_limit_seconds' => p95_limit.round(6),
+            'current_wall_time_max_seconds' => current_max,
+            'wall_time_max_limit_seconds' => max_limit.round(6),
+            'baseline_rss_kb' => baseline_rss,
+            'current_rss_kb' => current_rss,
+            'rss_limit_kb' => rss_limit.round,
+            'rss_kind' => current_result['rss_kind'],
+            'rss_scope' => current_result['rss_scope'],
+            'allocation_gate' => allocations,
+            'artifact_size_gate' => artifact,
+            'passed' => passed
+          }
+        end
+
+        def bounded_metric(result, key, configured_limit, corpus)
+          return { 'configured' => false, 'passed' => true } if configured_limit.nil?
+
+          limit = configured_limit.is_a?(Hash) ? configured_limit.fetch(corpus) : configured_limit
+          value = Integer(result.fetch(key))
+          limit = Integer(limit)
+          raise ArgumentError, "#{key} limit must be positive" unless limit.positive?
+
+          { 'configured' => true, 'value' => value, 'limit' => limit, 'passed' => value <= limit }
+        rescue KeyError
+          { 'configured' => true, 'passed' => false, 'diagnostic' => "current #{key} measurement unavailable" }
+        end
+
+        def provenance_error
+          return @provenance_error if defined?(@provenance_error)
+
+          @provenance_error = validate_provenance
+        end
+
+        def validate_provenance
+          return 'baseline performance schema_version must be 1' unless baseline_document['schema_version'] == 1
+          return 'current performance provenance schema_version must be 1' unless current_provenance['schema_version'] == 1
+          unless baseline_document['git_ref'] == config.dig('baseline', 'git_ref')
+            return 'baseline performance git_ref does not match the audit baseline'
+          end
+
+          baseline_environment = baseline_document['environment']
+          current_environment = current_provenance['environment']
+          return 'baseline performance environment is missing' unless baseline_environment.is_a?(Hash)
+          return 'current performance environment is missing' unless current_environment.is_a?(Hash)
+
+          mismatch = PROVENANCE_FIELDS.find do |field|
+            baseline_environment[field].to_s != current_environment[field].to_s
+          end
+          "performance environment mismatch for #{mismatch}" if mismatch
+        end
+
+        def compatible_rss?(current_result)
+          environment = baseline_document.fetch('environment')
+          current_result['rss_kind'] == environment['rss_kind'] &&
+            current_result['rss_scope'] == environment['rss_scope']
+        end
+
+        def relative_limit(baseline_value, ratio:, allowance:, absolute:)
+          [baseline_value * ratio, baseline_value + allowance].max.clamp(0, absolute)
+        end
+
+        def rss_value(performance)
+          value = performance['process_hwm_kb'] || performance['process_rss_kb']
+          Integer(value) if value
+        end
+
+        def unavailable(diagnostic)
+          { 'passed' => false, 'available' => false, 'diagnostic' => diagnostic }
+        end
+      end
+    end
+  end
+end