necropsy 0.2.0 → 0.3.0

data/lib/necropsy/bench/release_audit/run_provenance.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+require 'open3'
+require 'rbconfig'
+
+module Necropsy
+  module Bench
+    class ReleaseAudit
+      class RunProvenance
+        SCHEMA_VERSION = 1
+
+        def initialize(root:, manifest_path:, config_path:, output_dir:, command:)
+          @root = File.expand_path(root)
+          @manifest_path = File.expand_path(manifest_path)
+          @config_path = File.expand_path(config_path)
+          @output_dir = File.expand_path(output_dir)
+          @command = command
+        end
+
+        def capture_source!
+          ensure_clean!
+          {
+            'schema_version' => SCHEMA_VERSION,
+            'git_ref' => git_head,
+            'clean' => true,
+            'manifest_sha256' => digest(manifest_path),
+            'audit_config_sha256' => digest(config_path)
+          }
+        end
+
+        def complete(source, summary)
+          source.merge(
+            'environment' => environment(summary),
+            'artifacts' => artifact_digests
+          )
+        end
+
+        def write(path, payload)
+          File.write(path, "#{JSON.pretty_generate(payload)}\n")
+        end
+
+        def load_and_validate!(path)
+          payload = JSON.parse(File.read(path))
+          expected = capture_source!
+          expected.each do |key, value|
+            raise Error, "Benchmark provenance mismatch for #{key}" unless payload[key] == value
+          end
+          validate_environment!(payload.fetch('environment'))
+          raise Error, 'Benchmark provenance mismatch for artifacts' unless payload['artifacts'] == artifact_digests
+
+          payload
+        rescue Errno::ENOENT, JSON::ParserError => e
+          raise Error, "Could not load benchmark provenance: #{e.message}"
+        end
+
+        private
+
+        attr_reader :root, :manifest_path, :config_path, :output_dir, :command
+
+        def ensure_clean!
+          output, status = Open3.capture2e('git', 'status', '--porcelain', chdir: root)
+          raise Error, "Could not inspect audit worktree: #{output.strip}" unless status.success?
+          raise Error, 'Release audit requires a clean tracked worktree' unless output.strip.empty?
+        end
+
+        def git_head
+          output, status = Open3.capture2e('git', 'rev-parse', 'HEAD', chdir: root)
+          raise Error, "Could not resolve audit HEAD: #{output.strip}" unless status.success?
+
+          output.strip
+        end
+
+        def digest(path)
+          Digest::SHA256.file(path).hexdigest
+        end
+
+        def environment(summary)
+          performances = summary.fetch('corpora').filter_map { |corpus| corpus['performance'] }
+          rss_kinds = performances.filter_map { |item| item['rss_kind'] }.uniq
+          rss_scopes = performances.filter_map { |item| item['rss_scope'] }.uniq
+          {
+            'ruby' => RUBY_DESCRIPTION,
+            'os' => operating_system,
+            'command' => command,
+            'rss_kind' => rss_kinds.one? ? rss_kinds.first : 'unavailable_or_mixed',
+            'rss_scope' => rss_scopes.one? ? rss_scopes.first : 'unavailable_or_mixed'
+          }
+        end
+
+        def validate_environment!(recorded)
+          actual = {
+            'ruby' => RUBY_DESCRIPTION,
+            'os' => operating_system,
+            'command' => command
+          }
+          mismatch = actual.find { |key, value| recorded[key] != value }
+          raise Error, "Benchmark provenance mismatch for environment #{mismatch.first}" if mismatch
+        end
+
+        def artifact_digests
+          paths = [File.join(output_dir, 'summary.json'), *Dir.glob(File.join(output_dir, 'reports', '*.json'))]
+          raise Error, 'Benchmark provenance requires a summary and normalized reports' unless paths.length > 1
+
+          paths.sort.to_h do |path|
+            relative = path.delete_prefix("#{output_dir}/")
+            [relative, digest(path)]
+          end
+        rescue Errno::ENOENT => e
+          raise Error, "Could not hash benchmark artifacts: #{e.message}"
+        end
+
+        def operating_system
+          version = capture('sw_vers', '-productVersion')
+          build = capture('sw_vers', '-buildVersion')
+          kernel = capture('uname', '-r')
+          machine = capture('uname', '-m')
+          return "macOS #{version} (#{build}), Darwin #{kernel} #{machine}" if version && build && kernel && machine
+
+          [RbConfig::CONFIG['host_os'], RbConfig::CONFIG['host_cpu']].compact.join(' ')
+        end
+
+        def capture(*command_parts)
+          output, status = Open3.capture2e(*command_parts)
+          status.success? ? output.strip : nil
+        rescue SystemCallError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/bench/release_audit.rb

@@ -0,0 +1,360 @@
+# frozen_string_literal: true
+
+require_relative 'release_audit/config_validator'
+require_relative 'release_audit/performance_gate'
+require_relative 'precision_gate'
+require_relative 'claim_gate'
+
+module Necropsy
+  module Bench
+    class ReleaseAudit
+      HIGH_CONFIDENCES = %w[high certain].freeze
+      FALSE_POSITIVE_LABELS = %w[alive external].freeze
+      REVIEW_FIELDS = %w[outcome rationale reviewer].freeze
+      REVIEW_OUTCOMES = %w[expected_safety_change false_positive true_positive].freeze
+      MAX_ENTRIES = 100_000
+      MAX_STRING_BYTES = 4_096
+
+      def initialize(inputs)
+        @inputs = inputs
+      end
+
+      def call
+        validate_inputs!
+        comparisons = compare_reports
+        new_high = newly_high_candidates(comparisons)
+        review = review_status(comparisons)
+        performance = performance_status
+        precision = precision_gate_status
+        claim = claim_gate_status
+        gates = build_gates(new_high, review, performance, precision, claim)
+        {
+          'schema_version' => 1,
+          'release' => config.fetch('release'),
+          'baseline' => config.fetch('baseline'),
+          'corpora' => comparisons,
+          'new_high_candidates' => new_high,
+          'review' => review,
+          'performance' => performance,
+          'precision_gate' => precision,
+          'claim_gate' => claim,
+          'performance_provenance' => performance_provenance,
+          'adversarial_suites' => adversarial_results,
+          'gates' => gates,
+          'status' => gates.values.all? { |gate| gate['passed'] } ? 'pass' : 'fail'
+        }
+      end
+
+      private
+
+      attr_reader :inputs
+
+      def config = inputs.fetch(:config)
+      def baseline_reports = inputs.fetch(:baseline_reports)
+      def current_reports = inputs.fetch(:current_reports)
+      def current_summary = inputs.fetch(:current_summary)
+      def labels = inputs.fetch(:labels)
+      def reviews = inputs.fetch(:reviews)
+      def baseline_performance = inputs.fetch(:baseline_performance)
+      def adversarial_results = inputs.fetch(:adversarial_results)
+      def current_provenance = inputs.fetch(:current_provenance)
+
+      def validate_inputs!
+        ConfigValidator.new(config, strict_release: false).validate!
+        validate_mapping!(baseline_reports, 'baseline reports')
+        validate_mapping!(current_reports, 'current reports')
+        config.fetch('corpora').each do |corpus|
+          validate_report!(baseline_reports.fetch(corpus), "baseline report #{corpus}")
+          validate_report!(current_reports.fetch(corpus), "current report #{corpus}")
+        end
+        validate_mapping!(current_summary, 'current summary')
+        validate_mapping!(labels, 'labels')
+        validate_array!(reviews, 'reviews')
+        reviews.each { |review| validate_mapping!(review, 'review') }
+        validate_array!(adversarial_results, 'adversarial results')
+        adversarial_results.each { |result| validate_mapping!(result, 'adversarial result') }
+        expected = config.fetch('adversarial_suites').keys.sort
+        actual = adversarial_results.map { |result| result.fetch('name') }.sort
+        return if actual == expected
+
+        raise Error, "Adversarial results do not match configured suites: #{actual.inspect}"
+      end
+
+      def compare_reports
+        config.fetch('corpora').sort.to_h do |corpus|
+          baseline = index_findings(baseline_reports.fetch(corpus))
+          current = index_findings(current_reports.fetch(corpus))
+          added_ids = current.keys - baseline.keys
+          removed_ids = baseline.keys - current.keys
+          common_ids = baseline.keys & current.keys
+          state_changed_ids = common_ids.reject { |id| baseline[id]['state'] == current[id]['state'] }
+          confidence_changed_ids = common_ids.reject do |id|
+            baseline[id]['confidence'] == current[id]['confidence']
+          end
+          [corpus, {
+            'baseline_metrics' => baseline_reports.fetch(corpus).fetch('metrics'),
+            'current_metrics' => current_reports.fetch(corpus).fetch('metrics'),
+            'added' => change_list(added_ids, nil, current),
+            'removed' => change_list(removed_ids, baseline, nil),
+            'state_changed' => change_list(state_changed_ids, baseline, current),
+            'confidence_changed' => change_list(confidence_changed_ids, baseline, current)
+          }]
+        end
+      end
+
+      def index_findings(report)
+        report.fetch('findings').each_with_object({}) do |finding, index|
+          identity = finding_identity(finding)
+          raise Error, "Duplicate release-audit finding identity #{identity}" if index.key?(identity)
+
+          index[identity] = finding
+        end
+      end
+
+      def change_list(ids, baseline, current)
+        ids.sort.map do |identity|
+          finding = current&.fetch(identity) || baseline&.fetch(identity)
+          {
+            'id' => finding.fetch('id'),
+            'definition_id' => finding['definition_id'],
+            'identity' => identity,
+            'before' => baseline&.fetch(identity),
+            'after' => current&.fetch(identity)
+          }.compact
+        end
+      end
+
+      def newly_high_candidates(comparisons)
+        comparisons.flat_map do |corpus, _comparison|
+          baseline = index_findings(baseline_reports.fetch(corpus))
+          current = index_findings(current_reports.fetch(corpus))
+          current.filter_map do |identity, finding|
+            next unless high_actionable?(finding) && !high_actionable?(baseline[identity])
+
+            label, identity_match = label_for(corpus, finding)
+            finding.slice('id', 'definition_id', 'path', 'line', 'state', 'confidence').merge(
+              'corpus' => corpus,
+              'identity' => identity,
+              'label' => label,
+              'label_identity_match' => identity_match
+            ).compact
+          end
+        end.sort_by { |candidate| [candidate['corpus'], candidate['id'], candidate['definition_id'].to_s] }
+      end
+
+      def label_for(corpus, finding)
+        definition_id = finding['definition_id']
+        physical = labels[[corpus, definition_id]] if definition_id
+        return [physical, 'physical'] if physical
+
+        legacy = labels[[corpus, finding.fetch('id')]]
+        [legacy, legacy ? 'legacy_logical_fallback' : nil]
+      end
+
+      def high?(finding)
+        finding && HIGH_CONFIDENCES.include?(finding['confidence'])
+      end
+
+      def high_actionable?(finding)
+        high?(finding) && finding.fetch('candidate') { %w[unreachable unused candidate].include?(finding['state']) }
+      end
+
+      def review_status(comparisons)
+        required = required_reviews(comparisons)
+        supplied = review_indexes
+        missing = required.reject { |item| valid_review?(review_for(item, supplied)) }
+        reviewed = required.filter_map do |item|
+          supplied_review = review_for(item, supplied)
+          item.merge(supplied_review) if valid_review?(supplied_review)
+        end
+        invalid = required.filter_map do |item|
+          supplied_review = review_for(item, supplied)
+          item.merge(supplied_review) if supplied_review && !valid_review?(supplied_review)
+        end
+        {
+          'required' => required,
+          'completed' => reviewed.sort_by { |item| review_key(item) },
+          'missing' => missing,
+          'invalid' => invalid.sort_by { |item| review_key(item) },
+          'coverage' => review_coverage(comparisons, required, reviewed),
+          'confirmed_false_positives' => reviewed.select { |item| item['outcome'] == 'false_positive' }
+        }
+      end
+
+      def review_indexes
+        reviews.each_with_object({ physical: {}, legacy: {} }) do |review, indexes|
+          base = [review['corpus'], review['change_type']]
+          identity = review['definition_id']
+          target = identity ? indexes[:physical] : indexes[:legacy]
+          key = base + [identity || review['id']]
+          raise Error, "Duplicate release-audit review identity #{key.join(':')}" if target.key?(key)
+
+          target[key] = review
+        end
+      end
+
+      def review_for(item, indexes)
+        base = [item['corpus'], item['change_type']]
+        definition_id = review_definition_id(item)
+        physical = indexes[:physical][base + [definition_id]] if definition_id
+        physical || indexes[:legacy][base + [item['id']]]
+      end
+
+      def valid_review?(review)
+        review && REVIEW_FIELDS.all? { |field| !review[field].to_s.strip.empty? } &&
+          REVIEW_OUTCOMES.include?(review['outcome'])
+      end
+
+      def review_coverage(comparisons, required, reviewed)
+        config.fetch('review').fetch('corpora').sort.to_h do |corpus, policy|
+          changes = review_records(corpus, comparisons.fetch(corpus))
+          [corpus, {
+            'strategy' => policy.fetch('strategy'),
+            'changes' => changes.length,
+            'required' => required.count { |item| item['corpus'] == corpus },
+            'completed' => reviewed.count { |item| item['corpus'] == corpus },
+            'zero_difference' => changes.empty?
+          }]
+        end
+      end
+
+      def required_reviews(comparisons)
+        config.fetch('review').fetch('corpora').sort.flat_map do |corpus, policy|
+          records = review_records(corpus, comparisons.fetch(corpus))
+          next records if policy.fetch('strategy') == 'all'
+
+          records.group_by { |record| review_stratum(record) }.sort.flat_map do |_stratum, items|
+            items.sort_by { |item| [item['id'], review_definition_id(item).to_s] }
+                 .first(Integer(policy.fetch('minimum_per_stratum')))
+          end
+        end
+      end
+
+      def review_records(corpus, comparison)
+        %w[added removed state_changed].flat_map do |type|
+          comparison.fetch(type).map do |change|
+            change.merge('corpus' => corpus, 'change_type' => type)
+          end
+        end
+      end
+
+      def review_stratum(record)
+        [record['change_type'], record.dig('before', 'state'), record.dig('after', 'state')].join(':')
+      end
+
+      def review_key(record)
+        [record['corpus'], record['change_type'], review_definition_id(record) || record['id']]
+      end
+
+      def review_definition_id(record)
+        record['definition_id'] || record.dig('after', 'definition_id') || record.dig('before', 'definition_id')
+      end
+
+      def performance_status
+        PerformanceGate.new(
+          config: config,
+          baseline: baseline_performance,
+          current_summary: current_summary,
+          current_provenance: current_provenance
+        ).call
+      end
+
+      def performance_provenance
+        {
+          'baseline' => baseline_performance.except('corpora'),
+          'current' => current_provenance
+        }
+      end
+
+      def precision_gate_status
+        policy = config['precision_gate']
+        unless policy
+          return {
+            'schema_version' => 1,
+            'enforced' => false,
+            'compatibility' => 'releases before 0.4 retain the safety-only release policy',
+            'passed' => true
+          }
+        end
+
+        PrecisionGate.new(
+          policy: policy,
+          candidate_union_summary: current_summary['candidate_union'],
+          feature_ablation: current_summary['feature_ablation']
+        ).call
+      end
+
+      def claim_gate_status
+        ClaimGate.new(
+          config: config,
+          reports: current_reports,
+          summary: current_summary,
+          adversarial_results: adversarial_results
+        ).call
+      end
+
+      def build_gates(new_high, review, performance, precision, claim)
+        invalid_labels = new_high.reject { |candidate| valid_label?(candidate['label']) }
+        false_positive_labels = new_high.select do |candidate|
+          FALSE_POSITIVE_LABELS.include?(candidate.dig('label', 'value'))
+        end
+        unresolved_labels = new_high.select { |candidate| candidate.dig('label', 'value') == 'unknown' }
+        gates = {
+          'new_high_reviewed' => gate(invalid_labels.empty?, invalid_labels.length),
+          'new_high_false_positives' => gate(false_positive_labels.empty?, false_positive_labels.length),
+          'new_high_unresolved' => gate(unresolved_labels.empty?, unresolved_labels.length),
+          'difference_review' => gate(review['missing'].empty?, review['missing'].length),
+          'review_false_positives' => gate(review['confirmed_false_positives'].empty?,
+                                           review['confirmed_false_positives'].length),
+          'performance' => gate(performance.values.all? { |result| result['passed'] },
+                                performance.count { |_corpus, result| !result['passed'] }),
+          'adversarial' => gate(adversarial_results.all? { |result| result['passed'] },
+                                adversarial_results.count { |result| !result['passed'] })
+        }
+        gates['precision_quality'] = gate(precision['passed'], precision['passed'] ? 0 : 1) if precision['enforced']
+        gates['public_claim'] = gate(claim['passed'], claim['passed'] ? 0 : 1) if claim['enforced']
+        gates
+      end
+
+      def valid_label?(label)
+        label && %w[dead alive external unknown].include?(label['value']) &&
+          !label['rationale'].to_s.strip.empty? && !label['reviewer'].to_s.strip.empty?
+      end
+
+      def finding_identity(finding)
+        finding['definition_id'] || finding.fetch('id')
+      end
+
+      def validate_report!(report, label)
+        validate_mapping!(report, label)
+        validate_mapping!(report['metrics'], "#{label} metrics")
+        validate_array!(report['findings'], "#{label} findings")
+        report['findings'].each do |finding|
+          validate_mapping!(finding, "#{label} finding")
+          validate_identifier!(finding['id'], "#{label} finding id")
+          validate_identifier!(finding['definition_id'], "#{label} finding definition_id") if
+            finding.key?('definition_id')
+        end
+      end
+
+      def validate_mapping!(value, label)
+        raise Error, "#{label} must be a mapping" unless value.is_a?(Hash)
+        raise Error, "#{label} exceeds #{MAX_ENTRIES} entries" if value.length > MAX_ENTRIES
+      end
+
+      def validate_array!(value, label)
+        raise Error, "#{label} must be an array" unless value.is_a?(Array)
+        raise Error, "#{label} exceeds #{MAX_ENTRIES} entries" if value.length > MAX_ENTRIES
+      end
+
+      def validate_identifier!(value, label)
+        raise Error, "#{label} must be a non-empty string" unless value.is_a?(String) && !value.empty?
+        raise Error, "#{label} exceeds #{MAX_STRING_BYTES} bytes" if value.bytesize > MAX_STRING_BYTES
+      end
+
+      def gate(passed, failures)
+        { 'passed' => passed, 'failures' => failures }
+      end
+    end
+  end
+end

data/lib/necropsy/bench/report_normalizer.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative 'finding_facts'
+
+module Necropsy
+  module Bench
+    class ReportNormalizer
+      VERSION = 1
+
+      def initialize(report:, corpus:)
+        @report = report
+        @corpus = corpus
+      end
+
+      def call
+        {
+          'schema_version' => VERSION,
+          'corpus' => corpus,
+          'metrics' => metrics,
+          'states' => state_counts,
+          'quality' => quality,
+          'findings' => normalized_findings
+        }
+      end
+
+      def dump
+        "#{JSON.pretty_generate(call)}\n"
+      end
+
+      private
+
+      attr_reader :report, :corpus
+
+      def metrics
+        {
+          'analysis_graph' => {
+            'scope' => 'analysis',
+            'nodes' => report.graph.nodes.length,
+            'definitions' => report.graph.method_nodes.length,
+            'call_sites' => report.graph.call_sites.length,
+            'edges' => report.graph.edges.length
+          },
+          'findings' => reportable_findings.length,
+          'actionable_candidates' => actionable_findings.length,
+          'candidate_loc' => actionable_findings.sum { |finding| FindingFacts.loc(finding) },
+          'diagnostic_findings' => diagnostic_findings.length
+        }
+      end
+
+      def state_counts
+        reportable_findings.group_by(&:classification).transform_values(&:length).transform_keys(&:to_s).sort.to_h
+      end
+
+      def normalized_findings
+        reportable_findings.sort_by do |finding|
+          [finding.node.id, finding.node.file, finding.node.line, finding.node.definition_id]
+        end.map do |finding|
+          {
+            'id' => finding.node.id,
+            'definition_id' => finding.node.definition_id,
+            'path' => finding.node.file,
+            'line' => finding.node.line,
+            'end_line' => finding.node.end_line,
+            'loc' => FindingFacts.loc(finding),
+            'state' => finding.classification.to_s,
+            'confidence' => finding.confidence.to_s,
+            'candidate' => FindingFacts.actionable?(finding),
+            'diagnostic' => !FindingFacts.actionable?(finding),
+            'category' => FindingFacts.category(finding),
+            'unknown' => FindingFacts.unknown?(finding),
+            'rule_hits' => FindingFacts.rule_hits(finding),
+            'risk_flags' => FindingFacts.risk_flags(finding),
+            'blocker_kinds' => finding.blockers.map { |blocker| blocker.kind.to_s }.uniq.sort,
+            'reasons' => finding.reasons.map(&:to_s).sort
+          }
+        end
+      end
+
+      def quality
+        resolutions = FindingFacts.resolution_counts(report)
+        grouped = reportable_findings.group_by { |finding| FindingFacts.category(finding) }
+        {
+          'scope' => 'report',
+          'candidate_count' => actionable_findings.length,
+          'candidate_loc' => actionable_findings.sum { |finding| FindingFacts.loc(finding) },
+          'diagnostic_count' => diagnostic_findings.length,
+          'blocked_count' => blocked_findings.length,
+          'blocked_rate' => FindingFacts.ratio(blocked_findings.length, reportable_findings.length),
+          'unknown_finding_count' => unknown_findings.length,
+          'unknown_finding_rate' => FindingFacts.ratio(unknown_findings.length, reportable_findings.length),
+          'resolution_counts' => resolutions,
+          'unknown_resolution_rate' => FindingFacts.ratio(resolutions.fetch('unknown'), resolutions.fetch('total')),
+          'rule_counts' => FindingFacts.report_rule_counts(report),
+          'risk_counts' => FindingFacts.report_risk_counts(report),
+          'by_category' => grouped.sort.to_h do |category, findings|
+            [category, category_quality(findings)]
+          end
+        }
+      end
+
+      def category_quality(findings)
+        candidates = findings.select { |finding| FindingFacts.actionable?(finding) }
+        blocked = findings.select { |finding| finding.classification == :blocked }
+        unknown = findings.select { |finding| FindingFacts.unknown?(finding) }
+        {
+          'findings' => findings.length,
+          'candidate_count' => candidates.length,
+          'candidate_loc' => candidates.sum { |finding| FindingFacts.loc(finding) },
+          'blocked_count' => blocked.length,
+          'blocked_rate' => FindingFacts.ratio(blocked.length, findings.length),
+          'unknown_count' => unknown.length,
+          'unknown_rate' => FindingFacts.ratio(unknown.length, findings.length),
+          'rule_counts' => FindingFacts.tally(findings.flat_map { |finding| FindingFacts.rule_hits(finding) }),
+          'risk_counts' => FindingFacts.tally(findings.flat_map { |finding| FindingFacts.risk_flags(finding) })
+        }
+      end
+
+      def actionable_findings
+        @actionable_findings ||= reportable_findings.select { |finding| FindingFacts.actionable?(finding) }
+      end
+
+      def diagnostic_findings
+        @diagnostic_findings ||= reportable_findings.reject { |finding| FindingFacts.actionable?(finding) }
+      end
+
+      def blocked_findings
+        @blocked_findings ||= reportable_findings.select { |finding| finding.classification == :blocked }
+      end
+
+      def unknown_findings
+        @unknown_findings ||= reportable_findings.select { |finding| FindingFacts.unknown?(finding) }
+      end
+
+      def reportable_findings
+        @reportable_findings ||= report.reportable_findings
+      end
+    end
+  end
+end