necropsy 0.2.0 → 0.3.0

data/lib/necropsy/bench/review_queue.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+
+module Necropsy
+  module Bench
+    # Builds a deterministic, explicitly-unreviewed queue from normalized reports.
+    # The queue is evidence collection tooling; it never supplies labels or passes
+    # the public claim gate by itself.
+    class ReviewQueue
+      ACTIONABLE_STATES = %w[unreachable unused candidate].freeze
+      HIGH_CONFIDENCES = %w[high certain].freeze
+      CONFIDENCE_ORDER = { 'certain' => 0, 'high' => 1, 'medium' => 2, 'low' => 3 }.freeze
+      VERSION = 1
+
+      def initialize(reports:, target_reviewed_high: 300, limit: nil)
+        @reports = reports
+        @target_reviewed_high = Integer(target_reviewed_high)
+        @limit = limit.nil? ? @target_reviewed_high : Integer(limit)
+        validate_options!
+      rescue ArgumentError, TypeError
+        raise Error, 'Review queue target and limit must be integers'
+      end
+
+      def call
+        validate_reports!
+        available = actionable_entries
+        entries = available.sort_by { |entry| sort_key(entry) }.first(limit)
+        high_entries = entries.select { |entry| HIGH_CONFIDENCES.include?(entry.fetch('confidence')) }
+        {
+          'schema_version' => VERSION,
+          'status' => 'pending',
+          'selection' => 'actionable findings, confidence ascending, then corpus and identity',
+          'target_reviewed_high_candidates' => target_reviewed_high,
+          'available_actionable_candidates' => available.length,
+          'queued_candidates' => entries.length,
+          'reviewed_candidates' => 0,
+          'reviewed_high_candidates' => 0,
+          'pending_candidates' => entries.length,
+          'pending_high_candidates' => high_entries.length,
+          'target_shortfall' => [target_reviewed_high - high_entries.length, 0].max,
+          'claim_gate_passed' => false,
+          'corpora' => corpus_summary(available, entries),
+          'provenance' => provenance,
+          'entries' => entries
+        }
+      end
+
+      private
+
+      attr_reader :reports, :target_reviewed_high, :limit
+
+      def validate_options!
+        raise Error, 'Review queue target must be positive' unless target_reviewed_high.positive?
+        raise Error, 'Review queue limit must be positive' unless limit.positive?
+      end
+
+      def validate_reports!
+        raise Error, 'Review queue reports must be a mapping' unless reports.is_a?(Hash)
+
+        reports.each do |corpus, report|
+          raise Error, 'Review queue corpus names must be non-empty strings' if corpus.to_s.empty?
+          raise Error, "Review queue report #{corpus} must contain findings" unless
+            report.is_a?(Hash) && report['findings'].is_a?(Array)
+        end
+      end
+
+      def actionable_entries
+        report_pairs.flat_map do |corpus, report|
+          report.fetch('findings').filter_map do |finding|
+            next unless actionable?(finding)
+
+            {
+              'corpus' => corpus.to_s,
+              'id' => finding.fetch('id').to_s,
+              'definition_id' => finding['definition_id'],
+              'path' => finding['path'],
+              'line' => finding['line'],
+              'end_line' => finding['end_line'],
+              'loc' => finding['loc'],
+              'state' => finding.fetch('state').to_s,
+              'confidence' => finding.fetch('confidence').to_s,
+              'category' => finding['category'].to_s,
+              'status' => 'pending',
+              'review_class' => review_class(finding)
+            }.compact
+          end
+        end
+      end
+
+      def actionable?(finding)
+        finding['candidate'] == true || ACTIONABLE_STATES.include?(finding['state'].to_s)
+      end
+
+      def review_class(finding)
+        return 'high_candidate' if HIGH_CONFIDENCES.include?(finding['confidence'].to_s)
+
+        'candidate'
+      end
+
+      def sort_key(entry)
+        [
+          CONFIDENCE_ORDER.fetch(entry.fetch('confidence'), CONFIDENCE_ORDER.length),
+          entry.fetch('corpus'),
+          entry.fetch('category'),
+          entry.fetch('id'),
+          entry['definition_id'].to_s
+        ]
+      end
+
+      def corpus_summary(available, entries)
+        available_by_corpus = available.group_by { |entry| entry.fetch('corpus') }
+        reports.keys.map(&:to_s).sort.to_h do |corpus|
+          available_entries = available_by_corpus.fetch(corpus, [])
+          queued = entries.select { |entry| entry.fetch('corpus') == corpus }
+          [corpus, {
+            'available_candidates' => available_entries.length,
+            'queued_candidates' => queued.length,
+            'queued_high_candidates' => queued.count { |entry| HIGH_CONFIDENCES.include?(entry.fetch('confidence')) }
+          }]
+        end
+      end
+
+      def provenance
+        report_pairs.to_h do |corpus, report|
+          canonical = canonicalize(report)
+          [corpus.to_s, {
+            'sha256' => Digest::SHA256.hexdigest(JSON.generate(canonical)),
+            'finding_count' => report.fetch('findings').length
+          }]
+        end
+      end
+
+      def report_pairs
+        reports.to_a.sort_by { |corpus, _report| corpus.to_s }
+      end
+
+      def canonicalize(value)
+        case value
+        when Hash
+          value.keys.map(&:to_s).sort.to_h do |key|
+            original = value.key?(key) ? key : value.keys.find { |candidate| candidate.to_s == key }
+            [key, canonicalize(value.fetch(original))]
+          end
+        when Array
+          value.map { |item| canonicalize(item) }
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/bench/safety_mutation_harness.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Bench
+    class SafetyMutationHarness
+      Result = Data.define(:name, :detected, :violations)
+      HEALTH_RANK = { invalid: 0, degraded: 1, complete: 2 }.freeze
+
+      def initialize(baseline:)
+        @baseline = baseline
+      end
+
+      def evaluate(mutants)
+        Hash(mutants).sort.to_h do |name, report|
+          violations = violations_for(report)
+          [name.to_s, Result.new(name: name.to_s, detected: violations.any?, violations: violations.freeze)]
+        end
+      end
+
+      def assert_all_detected!(mutants)
+        results = evaluate(mutants)
+        survivors = results.values.reject(&:detected).map(&:name)
+        raise Error, "Safety mutations survived: #{survivors.join(', ')}" if survivors.any?
+
+        results
+      end
+
+      private
+
+      attr_reader :baseline
+
+      def violations_for(mutant)
+        violations = []
+        added = actionable_ids(mutant) - actionable_ids(baseline)
+        violations << "actionable candidates grew: #{added.sort.join(', ')}" if added.any?
+        bypassed = blocked_ids(baseline) & actionable_ids(mutant)
+        violations << "blocked definitions became actionable: #{bypassed.sort.join(', ')}" if bypassed.any?
+        if health_rank(mutant) > health_rank(baseline) && !baseline.analysis_health.complete?
+          violations << 'incomplete analysis was promoted to healthier status'
+        end
+        violations
+      end
+
+      def actionable_ids(report)
+        report.actionable_candidates(min_confidence: :low).to_set { |finding| finding.node.graph_id }
+      end
+
+      def blocked_ids(report)
+        report.findings.select { |finding| finding.classification == :blocked }.to_set do |finding|
+          finding.node.graph_id
+        end
+      end
+
+      def health_rank(report)
+        HEALTH_RANK.fetch(report.analysis_health.status)
+      end
+    end
+  end
+end

data/lib/necropsy/bench/seed_runner.rb

@@ -0,0 +1,408 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'digest'
+require 'json'
+require 'open3'
+require 'yaml'
+
+require_relative 'candidate_union'
+require_relative 'precision_gate'
+require_relative 'report_normalizer'
+
+module Necropsy
+  module Bench
+    class SeedRunner
+      def initialize(
+        manifest_path:,
+        output_dir:,
+        io: $stdout,
+        clock: Process.method(:clock_gettime),
+        analyzer: nil,
+        feature_ablation: {},
+        rss_reader: nil,
+        revision_reader: nil,
+        dirty_reader: nil
+      )
+        @manifest_path = File.expand_path(manifest_path)
+        @output_dir = File.expand_path(output_dir)
+        @io = io
+        @clock = clock
+        @analyzer = analyzer || lambda { |root, config_path|
+          Necropsy.analyze(root: root, config_path: config_path, profile: true)
+        }
+        @feature_ablation = feature_ablation
+        @rss_reader = rss_reader || method(:read_process_rss)
+        @revision_reader = revision_reader || method(:read_git_revision)
+        @dirty_reader = dirty_reader || method(:tracked_git_dirty?)
+      end
+
+      def call(update_golden_reason: nil)
+        prepare_output
+        reports = {}
+        corpus_runs = manifest.fetch('corpora').sort.map do |id, definition|
+          run_corpus(id, definition, reports)
+        end
+        union = CandidateUnion.new(
+          manifest: manifest,
+          repository_root: repository_root,
+          reports: reports,
+          diagnostics: diagnostics
+        ).call
+        write_json(File.join(output_dir, 'candidate_union.json'), union)
+        update_golden(update_golden_reason, corpus_runs) if update_golden_reason
+        golden = golden_status
+        report_golden_status(golden)
+        summary = build_summary(corpus_runs, union, golden)
+        write_json(File.join(output_dir, 'summary.json'), summary)
+        summary
+      end
+
+      private
+
+      attr_reader :manifest_path, :output_dir, :io, :clock, :analyzer, :feature_ablation, :rss_reader,
+                  :revision_reader, :dirty_reader
+
+      def manifest
+        @manifest ||= YAML.safe_load_file(manifest_path, aliases: false).tap do |payload|
+          raise Error, 'Benchmark manifest schema_version must be 1' unless payload&.fetch('schema_version', nil) == 1
+        end
+      rescue Psych::Exception => e
+        raise Error, "Could not parse benchmark manifest: #{e.message}"
+      end
+
+      def repository_root
+        @repository_root ||= File.expand_path(manifest.fetch('repository_root'), File.dirname(manifest_path))
+      end
+
+      def reports_dir
+        File.join(output_dir, 'reports')
+      end
+
+      def diagnostics
+        @diagnostics ||= []
+      end
+
+      def prepare_output
+        FileUtils.mkdir_p(reports_dir)
+        Dir.glob(File.join(reports_dir, '*.json')).each { |path| FileUtils.rm_f(path) }
+      end
+
+      def run_corpus(id, definition, reports)
+        path = corpus_path(definition)
+        return skipped_corpus(id, definition, path) unless path && File.directory?(path)
+
+        revision_error = pinned_corpus_error(definition, path)
+        return failed_revision(id, definition, revision_error) if revision_error
+
+        samples = performance_sample_count
+        measurements = Array.new(samples) do
+          started_at = monotonic_time
+          report = analyzer.call(path, config_path(path, definition))
+          wall_time = monotonic_time - started_at
+          normalized = ReportNormalizer.new(report: report, corpus: id).call
+          {
+            report: report,
+            normalized: normalized,
+            wall_time: wall_time,
+            rss: rss_measurement(id),
+            allocated_objects: report.performance_profile&.dig('totals', 'allocated_objects'),
+            artifact_size_bytes: JSON.generate(normalized).bytesize
+          }
+        end
+        report = measurements.last.fetch(:report)
+        normalized = measurements.last.fetch(:normalized)
+        reports[id] = normalized
+        write_json(File.join(reports_dir, "#{id}.json"), normalized)
+        io.puts "#{id}: generated"
+        performance = performance_distribution(measurements)
+        analysis_profile = report.performance_profile if report.respond_to?(:performance_profile)
+        performance['analysis_profile'] = analysis_profile if analysis_profile
+        {
+          'id' => id,
+          'status' => 'generated',
+          'revision' => definition['revision'],
+          'metrics' => normalized.fetch('metrics'),
+          'performance' => performance
+        }.compact
+      rescue Error, SystemCallError => e
+        message = "#{id} failed: #{e.message}"
+        diagnostics << message
+        io.puts message
+        { 'id' => id, 'status' => 'failed', 'diagnostic' => message }
+      end
+
+      def performance_sample_count
+        value = Integer(manifest.fetch('performance_samples', 1))
+        raise Error, 'performance_samples must be between 1 and 20' unless value.between?(1, 20)
+
+        value
+      rescue ArgumentError, TypeError
+        raise Error, 'performance_samples must be between 1 and 20'
+      end
+
+      def performance_distribution(measurements)
+        wall_times = measurements.map { |measurement| measurement.fetch(:wall_time) }
+        allocations = measurements.filter_map { |measurement| measurement[:allocated_objects] }
+        artifact_sizes = measurements.map { |measurement| measurement.fetch(:artifact_size_bytes) }
+        rss = measurements.map { |measurement| measurement.fetch(:rss) }.max_by { |value| rss_value(value) || -1 }
+        {
+          'sample_count' => measurements.length,
+          'wall_time_seconds' => mean(wall_times).round(6),
+          'wall_time_p95_seconds' => percentile(wall_times, 0.95).round(6),
+          'wall_time_max_seconds' => wall_times.max.round(6),
+          'allocated_objects_p95' => percentile(allocations, 0.95),
+          'allocated_objects_max' => allocations.max,
+          'artifact_size_p95_bytes' => percentile(artifact_sizes, 0.95),
+          'artifact_size_max_bytes' => artifact_sizes.max
+        }.compact.merge(rss)
+      end
+
+      def percentile(values, quantile)
+        return if values.empty?
+
+        sorted = values.sort
+        sorted.fetch([(sorted.length * quantile).ceil - 1, 0].max)
+      end
+
+      def mean(values)
+        values.sum.to_f / values.length
+      end
+
+      def rss_value(measurement)
+        measurement['process_hwm_kb'] || measurement['process_rss_kb']
+      end
+
+      def corpus_path(definition)
+        env_path = ENV[definition['path_env'].to_s] if definition['path_env']
+        relative = env_path || definition['path']
+        File.expand_path(relative, repository_root) if relative && !relative.empty?
+      end
+
+      def config_path(path, definition)
+        config = definition['config']
+        return unless config
+
+        repository_config = File.expand_path(config, repository_root)
+        return repository_config if File.file?(repository_config)
+
+        File.expand_path(config, path)
+      end
+
+      def skipped_corpus(id, definition, path)
+        source = definition['path_env'] ? "set #{definition['path_env']}" : path.inspect
+        message = "#{id} skipped: corpus unavailable (#{source})"
+        diagnostics << message
+        io.puts message
+        { 'id' => id, 'status' => 'skipped', 'revision' => definition['revision'], 'diagnostic' => message }.compact
+      end
+
+      def pinned_corpus_error(definition, path)
+        expected = definition['git_commit']
+        return unless expected
+
+        actual = revision_reader.call(path, 'HEAD')
+        return "expected Git HEAD #{expected}, got #{actual || 'not a Git checkout'} at #{path}" unless actual == expected
+
+        tag = definition['git_tag']
+        tag_revision = revision_reader.call(path, "refs/tags/#{tag}^{commit}") if tag
+        return "expected Git tag #{tag} at #{expected}, got #{tag_revision || 'missing'}" if tag && tag_revision != expected
+
+        "tracked Git changes present at #{path}" if dirty_reader.call(path)
+      end
+
+      def failed_revision(id, definition, reason)
+        message = "#{id} failed: #{reason}"
+        diagnostics << message
+        io.puts message
+        { 'id' => id, 'status' => 'failed', 'revision' => definition['git_commit'], 'diagnostic' => message }
+      end
+
+      def rss_measurement(id)
+        measurement = rss_reader.call
+        return measurement if measurement
+
+        message = "#{id}: RSS unavailable on this platform"
+        diagnostics << message unless diagnostics.include?(message)
+        { 'rss_status' => 'unavailable', 'rss_diagnostic' => message }
+      end
+
+      def read_process_rss
+        status_path = '/proc/self/status'
+        if File.file?(status_path)
+          match = File.read(status_path).match(/^VmHWM:\s+(\d+)\s+kB$/)
+          if match
+            return {
+              'process_hwm_kb' => match[1].to_i,
+              'rss_kind' => 'cumulative_process_high_water_mark',
+              'rss_scope' => 'benchmark_runner_process'
+            }
+          end
+        end
+
+        output = IO.popen(['ps', '-o', 'rss=', '-p', Process.pid.to_s], &:read)
+        rss = Integer(output.strip, exception: false)
+        return unless rss
+
+        {
+          'process_rss_kb' => rss,
+          'rss_kind' => 'current_process_rss_after_corpus',
+          'rss_scope' => 'benchmark_runner_process'
+        }
+      rescue ArgumentError, SystemCallError
+        nil
+      end
+
+      def read_git_revision(path, reference)
+        output, status = Open3.capture2e('git', '-C', path, 'rev-parse', reference)
+        status.success? ? output.strip : nil
+      rescue SystemCallError
+        nil
+      end
+
+      def tracked_git_dirty?(path)
+        output, status = Open3.capture2e('git', '-C', path, 'status', '--porcelain', '--untracked-files=no')
+        !status.success? || !output.strip.empty?
+      rescue SystemCallError
+        true
+      end
+
+      def monotonic_time
+        clock.call(Process::CLOCK_MONOTONIC)
+      end
+
+      def build_summary(corpus_runs, union, golden)
+        {
+          'schema_version' => 1,
+          'manifest' => File.basename(manifest_path),
+          'corpora' => corpus_runs,
+          'candidate_union' => union.fetch('summary'),
+          'feature_ablation' => deterministic_payload(feature_ablation),
+          'precision_gate' => precision_gate(union),
+          'golden' => golden,
+          'diagnostics' => diagnostics.sort
+        }
+      end
+
+      def precision_gate(union)
+        policy = manifest['precision_gate']
+        unless policy
+          return {
+            'schema_version' => 1,
+            'enforced' => false,
+            'compatibility' => 'manifest without precision_gate retains the benchmark-v1 policy',
+            'passed' => true
+          }
+        end
+
+        PrecisionGate.new(
+          policy: policy,
+          candidate_union_summary: union.fetch('summary'),
+          feature_ablation: feature_ablation
+        ).call
+      end
+
+      def deterministic_payload(value)
+        case value
+        when Hash
+          value.sort.to_h { |key, nested| [key.to_s, deterministic_payload(nested)] }
+        when Array
+          value.map { |nested| deterministic_payload(nested) }
+        else
+          value
+        end
+      end
+
+      def golden_status
+        golden_dir = File.expand_path(manifest.fetch('golden_dir'), repository_root)
+        return { 'status' => 'missing', 'differences' => deterministic_artifacts } unless File.directory?(golden_dir)
+
+        integrity = golden_integrity(golden_dir)
+        return integrity unless integrity['status'] == 'valid'
+
+        expected = artifact_contents(golden_dir)
+        actual = artifact_contents(output_dir)
+        differences = (expected.keys | actual.keys).reject { |path| expected[path] == actual[path] }.sort
+        { 'status' => differences.empty? ? 'match' : 'drift', 'differences' => differences }
+      end
+
+      def golden_integrity(golden_dir)
+        metadata_path = File.join(golden_dir, 'metadata.json')
+        return { 'status' => 'invalid', 'differences' => ['metadata.json missing'] } unless File.file?(metadata_path)
+
+        metadata = JSON.parse(File.read(metadata_path))
+        reason = metadata['update_reason'].to_s.strip
+        return { 'status' => 'invalid', 'differences' => ['update reason missing'] } if reason.empty?
+
+        actual = artifact_digests(golden_dir)
+        expected = metadata.fetch('artifacts', {})
+        differences = (actual.keys | expected.keys).reject { |path| actual[path] == expected[path] }.sort
+        return { 'status' => 'invalid', 'differences' => differences } unless differences.empty?
+
+        { 'status' => 'valid', 'differences' => [] }
+      rescue JSON::ParserError
+        { 'status' => 'invalid', 'differences' => ['metadata.json malformed'] }
+      end
+
+      def deterministic_artifacts
+        ['candidate_union.json'] + Dir.glob(File.join(output_dir, 'reports', '*.json')).map do |path|
+          File.join('reports', File.basename(path))
+        end.sort
+      end
+
+      def artifact_contents(root)
+        paths = ['candidate_union.json'] + Dir.glob(File.join(root, 'reports', '*.json')).map do |path|
+          File.join('reports', File.basename(path))
+        end.sort
+        paths.to_h do |relative|
+          path = File.join(root, relative)
+          [relative, File.file?(path) ? File.binread(path) : nil]
+        end
+      end
+
+      def artifact_digests(root)
+        artifact_contents(root).transform_values { |contents| Digest::SHA256.hexdigest(contents) }
+      end
+
+      def report_golden_status(golden)
+        differences = golden.fetch('differences')
+        suffix = differences.empty? ? '' : " (#{differences.join(', ')})"
+        io.puts "golden: #{golden.fetch('status')}#{suffix}"
+      end
+
+      def write_json(path, payload)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, "#{JSON.pretty_generate(payload)}\n")
+      end
+
+      def update_golden(reason, corpus_runs)
+        raise Error, 'Updating benchmark golden files requires a non-empty reason' if reason.to_s.strip.empty?
+
+        ensure_required_corpora_generated!(corpus_runs)
+
+        golden_dir = File.expand_path(manifest.fetch('golden_dir'), repository_root)
+        FileUtils.mkdir_p(File.join(golden_dir, 'reports'))
+        FileUtils.cp(File.join(output_dir, 'candidate_union.json'), File.join(golden_dir, 'candidate_union.json'))
+        sources = Dir.glob(File.join(output_dir, 'reports', '*.json'))
+        sources.each do |source|
+          FileUtils.cp(source, File.join(golden_dir, 'reports', File.basename(source)))
+        end
+        write_json(File.join(golden_dir, 'metadata.json'), {
+                     'schema_version' => 1,
+                     'update_reason' => reason.strip,
+                     'artifacts' => artifact_digests(golden_dir).sort.to_h
+                   })
+      end
+
+      def ensure_required_corpora_generated!(corpus_runs)
+        statuses = corpus_runs.to_h { |corpus| [corpus.fetch('id'), corpus.fetch('status')] }
+        missing = manifest.fetch('corpora').filter_map do |id, definition|
+          id if definition['required'] == true && statuses[id] != 'generated'
+        end.sort
+        return if missing.empty?
+
+        raise Error, "Cannot update golden files; required corpora were not generated: #{missing.join(', ')}"
+      end
+    end
+  end
+end