necropsy 0.2.0 → 0.3.0

data/lib/necropsy/analyzers/dynamic/coverband_importer.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'openssl'
-require 'socket'
-require 'uri'
 require 'yaml'
 
+require_relative 'redis_payload_loader'
+require_relative 'runtime_reference'
+
 module Necropsy
   module Analyzers
     module Dynamic
@@ -19,11 +19,23 @@ module Necropsy
           return AnalyzerResult.empty unless source
 
           payload = load_payload(source, project.root)
-          alive = alive_from_payload(graph, payload).map do |node|
+          observation = ObservationPolicy.metadata(payload, expected_revision: config['expected_source_revision'])
+          node_ids, line_diagnostics = alive_ids_from_payload(graph, payload)
+          observation = observation.merge(line_diagnostics) unless line_diagnostics.empty?
+          alive = node_ids.map do |node_id|
+            node = graph.nodes.lookup(node_id).node
+            details = node ? "Coverband executed #{node.file}:#{node.line}" : "Coverband marked #{node_id} as executed"
             AliveEvidence.new(
-              node_id: node.id,
-              evidence: evidence(kind: :alive, details: "Coverband executed #{node.file}:#{node.line}",
-                                 metadata: payload['observation'] || {})
+              node_id: node_id,
+              evidence: evidence(
+                kind: :alive,
+                details: details,
+                metadata: observation.merge('node_reference' => node_id),
+                grade: :observed,
+                relation: :execution,
+                source: { 'type' => 'coverband', 'node_reference' => node_id },
+                scope: ObservationPolicy.evidence_scope(observation).merge('node_reference' => node_id)
+              )
             )
           end
 
@@ -31,7 +43,9 @@ module Necropsy
             edge_evidences: [],
             alive_evidences: alive,
             uncertainties: {},
-            observation: { 'coverband' => payload['observation'] || {} }
+            observation: { 'coverband' => observation },
+            resolutions: [],
+            evidences: result_evidences(alive)
           )
         end
 
@@ -40,7 +54,9 @@ module Necropsy
             name: :coverband,
             kind: :dynamic,
             soundness: :observational,
-            description: 'Imports file and line execution data compatible with coverband-style exports.'
+            description: 'Imports file and line execution data compatible with coverband-style exports.',
+            version: Necropsy::VERSION,
+            assumptions: %w[line_execution_mapping positive_observations_only]
           )
         end
 
@@ -68,17 +84,25 @@ module Necropsy
           RedisPayloadLoader.new(source: source, config: config).load
         end
 
-        def alive_from_payload(graph, payload)
-          executed_nodes = Array(payload['executed'] || payload['nodes'])
-          by_id = executed_nodes.filter_map { |id| graph.nodes[id] }
+        def alive_ids_from_payload(graph, payload)
+          structured = payload['node_references'] if payload.key?('node_references')
+          malformed = []
+          executed_nodes = RuntimeReference.preferred(
+            structured: structured,
+            legacy: payload['executed'] || payload['nodes']
+          ).map do |reference|
+            normalized = RuntimeReference.normalize(reference)
+            next normalized if normalized
 
-          files = coverage_files(payload)
-          by_line = graph.method_nodes.select do |node|
-            executed_lines = lines_for(files, node.file)
-            executed_lines.any? { |line| line.between?(node.line, node.end_line) }
+            malformed << { 'kind' => 'node', 'reference' => reference }
+            reference
           end
 
-          (by_id + by_line).uniq(&:id)
+          files = coverage_files(payload)
+          by_line, diagnostics = line_references(graph, files)
+          diagnostics['malformed_references'] = malformed unless malformed.empty?
+
+          [(executed_nodes + by_line).uniq, diagnostics]
         end
 
         def coverage_files(payload)
@@ -135,294 +159,40 @@ module Necropsy
           left.merge(right) { |_file, left_lines, right_lines| (left_lines + right_lines).uniq }
         end
 
-        def lines_for(files, relative_file)
-          exact = files[relative_file] || files[File.join('.', relative_file)]
-          return exact if exact
-
-          files.each do |file, lines|
-            return lines if file.end_with?("/#{relative_file}")
-          end
-
-          []
-        end
-      end
-
-      class RedisPayloadLoader
-        DEFAULT_PATTERN = 'coverband*'
-        DEFAULT_CONNECT_TIMEOUT = 5.0
-        DEFAULT_READ_TIMEOUT = 5.0
-
-        def initialize(source:, config:)
-          @uri = URI(source)
-          @config = config
-          @socket = nil
-        end
-
-        def load
-          connect
-          authenticate
-          select_database
-          merge_payloads(keys.filter_map { |key| payload_for_key(key) })
-        ensure
-          socket&.close
-        end
-
-        private
-
-        attr_reader :uri, :config, :socket
-
-        def connect
-          tcp_socket = Socket.tcp(uri.host, uri.port || 6379, connect_timeout: connect_timeout)
-          apply_read_timeout(tcp_socket)
-          @socket = uri.scheme == 'rediss' ? tls_socket(tcp_socket) : tcp_socket
-        rescue SystemCallError, SocketError, IO::TimeoutError => e
-          tcp_socket&.close
-          raise Error, "Could not connect to Redis at #{uri.host}:#{uri.port || 6379}: #{e.message}"
-        end
-
-        def tls_socket(tcp_socket)
-          context = OpenSSL::SSL::SSLContext.new
-          socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, context)
-          socket.hostname = uri.host if socket.respond_to?(:hostname=)
-          socket.connect
-          socket
-        end
-
-        def authenticate
-          return unless uri.password
-
-          password = URI.decode_www_form_component(uri.password)
-          return command('AUTH', password) unless uri.user
-
-          command('AUTH', URI.decode_www_form_component(uri.user), password)
-        end
-
-        def select_database
-          db = uri.path.to_s.delete_prefix('/')
-          command('SELECT', db) unless db.empty?
-        end
-
-        def keys
-          configured = Array(config['keys'] || config['key'] || query['key']).compact
-          return configured unless configured.empty?
-
-          scan(pattern: config['pattern'] || query['pattern'] || DEFAULT_PATTERN)
-        end
-
-        def query
-          @query ||= URI.decode_www_form(uri.query.to_s).to_h
-        end
-
-        def scan(pattern:)
-          cursor = '0'
-          found = []
-          loop do
-            cursor, keys = command('SCAN', cursor, 'MATCH', pattern)
-            found.concat(keys)
-            break if cursor == '0'
-          end
-          found
-        end
-
-        def command(*parts)
-          socket.write(redis_command(parts))
-          read_response
-        rescue IOError, SystemCallError, OpenSSL::SSL::SSLError => e
-          raise Error, "Redis connection failed: #{e.message}"
-        end
-
-        def payload_for_key(key)
-          parse_payload(command('GET', key))
-        rescue Error => e
-          raise unless e.message.include?('WRONGTYPE')
-
-          parse_hash_payload(command('HGETALL', key))
-        end
-
-        def redis_command(parts)
-          "#{["*#{parts.length}", *parts.flat_map { |part| ["$#{part.to_s.bytesize}", part.to_s] }].join("\r\n")}\r\n"
-        end
-
-        def read_response
-          prefix = read_exact(1)
-          case prefix
-          when '+'
-            read_line
-          when '-'
-            raise Error, read_line
-          when ':'
-            read_line.to_i
-          when '$'
-            read_bulk_string
-          when '*'
-            length = read_line.to_i
-            return nil if length.negative?
-
-            Array.new(length) { read_response }
-          else
-            raise Error, "Unsupported Redis response prefix #{prefix.inspect}"
-          end
-        end
-
-        def read_bulk_string
-          length = read_line.to_i
-          return nil if length.negative?
-
-          value = read_exact(length)
-          read_exact(2)
-          value
-        end
-
-        def read_line
-          line = socket.gets("\r\n")
-          raise Error, 'Redis closed the connection while reading a response' unless line
-
-          line.delete_suffix("\r\n")
-        end
-
-        def read_exact(length)
-          value = socket.read(length)
-          raise Error, 'Redis closed the connection while reading a response' unless value&.bytesize == length
-
-          value
-        end
-
-        def connect_timeout
-          Float(config.fetch('connect_timeout', DEFAULT_CONNECT_TIMEOUT))
-        rescue ArgumentError, TypeError
-          DEFAULT_CONNECT_TIMEOUT
-        end
-
-        def read_timeout
-          Float(config.fetch('read_timeout', DEFAULT_READ_TIMEOUT))
-        rescue ArgumentError, TypeError
-          DEFAULT_READ_TIMEOUT
-        end
-
-        def apply_read_timeout(tcp_socket)
-          seconds = read_timeout
-          timeout = [seconds.to_i, ((seconds % 1) * 1_000_000).to_i].pack('l_2')
-          tcp_socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, timeout)
-        end
-
-        def parse_payload(value)
-          return nil if value.nil? || value.empty?
-
-          JSON.parse(value)
-        rescue JSON::ParserError
-          parse_yaml_payload(value)
-        end
-
-        def parse_yaml_payload(value)
-          YAML.safe_load(value, aliases: false)
-        rescue Psych::Exception, ArgumentError, EncodingError => e
-          raise Error, "Invalid Redis coverage payload: #{e.message}"
-        end
-
-        def parse_hash_payload(entries)
-          hash = Array(entries).each_slice(2).to_h
-          return coverband_hash_payload(hash) if coverband_hash_payload?(hash)
-
-          payloads = []
-          files = {}
-          hash.each do |field, value|
-            parsed = parse_payload(value)
-            if payload_shape?(parsed)
-              payloads << parsed
-            else
-              files[field.to_s] = parsed
+        def line_references(graph, files)
+          definition_ids = []
+          line_ambiguities = []
+          path_ambiguities = []
+          graph.method_nodes.group_by(&:file).sort.each do |relative_file, definitions|
+            paths = matching_coverage_paths(files, relative_file)
+            next if paths.empty?
+
+            path_ambiguities << { 'file' => relative_file, 'coverage_paths' => paths } if paths.length > 1
+            paths.flat_map { |path| files.fetch(path) }.uniq.sort.each do |line|
+              matches = definitions.select { |definition| line.between?(definition.line, definition.end_line) }
+              definition_ids.concat(matches.map(&:graph_id))
+              next unless matches.length > 1
+
+              line_ambiguities << {
+                'file' => relative_file,
+                'line' => line,
+                'definition_ids' => matches.map(&:graph_id).sort
+              }
             end
           end
 
-          payloads << { 'files' => files } unless files.empty?
-          merge_payloads(payloads)
-        end
-
-        def coverband_hash_payload?(hash)
-          hash.key?('file') && hash.key?('file_length')
-        end
-
-        def coverband_hash_payload(hash)
-          {
-            'files' => { hash.fetch('file') => coverband_line_counts(hash) },
-            'observation' => coverband_observation(hash)
-          }
-        end
-
-        def coverband_line_counts(hash)
-          length = hash.fetch('file_length').to_i
-          Array.new(length) do |index|
-            count = hash[index.to_s]
-            count&.to_i
-          end
+          diagnostics = {}
+          diagnostics['path_ambiguities'] = path_ambiguities unless path_ambiguities.empty?
+          diagnostics['line_ambiguities'] = line_ambiguities unless line_ambiguities.empty?
+          [definition_ids.uniq.sort, diagnostics]
         end
 
-        def coverband_observation(hash)
-          %w[first_updated_at last_updated_at file_hash].each_with_object({}) do |key, observation|
-            observation[key] = hash[key] if hash[key]
-          end
-        end
-
-        def payload_shape?(value)
-          value.is_a?(Hash) && value.keys.any? do |key|
-            %w[files coverage runtime production nodes executed observation].include?(key.to_s)
-          end
-        end
-
-        def merge_payloads(payloads)
-          payloads.compact.reduce({}) { |merged, payload| merge_payload(merged, payload) }
-        end
-
-        def merge_payload(merged, payload)
-          merged.merge(payload) do |key, left, right|
-            case key
-            when 'files', 'coverage', 'runtime', 'production'
-              merge_files(left, right)
-            when 'nodes', 'executed'
-              (Array(left) + Array(right)).uniq
-            when 'edges'
-              Array(left) + Array(right)
-            when 'observation'
-              left.merge(right)
-            else
-              right
-            end
-          end
-        end
-
-        def merge_files(left, right)
-          return right unless left.is_a?(Hash)
-          return left unless right.is_a?(Hash)
-
-          left.merge(right) do |_file, left_lines, right_lines|
-            merge_file_coverage(left_lines, right_lines)
-          end
-        end
-
-        def merge_file_coverage(left, right)
-          return merge_count_hashes(left, right) if left.is_a?(Hash) && right.is_a?(Hash)
-          return merge_count_arrays(left, right) if count_array?(left) || count_array?(right)
-
-          (Array(left) + Array(right)).uniq
-        end
-
-        def merge_count_hashes(left, right)
-          left.merge(right) { |_line, left_count, right_count| left_count.to_i + right_count.to_i }
-        end
-
-        def merge_count_arrays(left, right)
-          length = [Array(left).length, Array(right).length].max
-          Array.new(length) do |index|
-            left_value = Array(left)[index]
-            right_value = Array(right)[index]
-            next if left_value.nil? && right_value.nil?
-
-            left_value.to_i + right_value.to_i
-          end
-        end
+        def matching_coverage_paths(files, relative_file)
+          exact = [relative_file, File.join('.', relative_file)].select { |path| files.key?(path) }
+          return exact.sort unless exact.empty?
 
-        def count_array?(value)
-          value.is_a?(Array) && value.any? { |item| item.nil? || item.to_i.zero? }
+          suffix = "/#{relative_file}"
+          files.keys.select { |path| path.end_with?(suffix) }.sort
         end
       end
     end

data/lib/necropsy/analyzers/dynamic/coverband_payload_set.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      class CoverbandPayloadSet
+        PAYLOAD_KEYS = %w[files coverage runtime production nodes executed observation].freeze
+
+        def initialize(limits:, deadline:)
+          @limits = limits
+          @deadline = deadline
+        end
+
+        def merge(payloads)
+          payloads.compact.reduce({}) do |merged, payload|
+            validate_payload_hash!(payload)
+            deadline.check!
+            limits.validate_payload!(merge_payload(merged, payload))
+          end
+        end
+
+        def from_hash_entries(entries, &)
+          domain_error('Malformed Redis hash response') unless entries.is_a?(Array) && entries.length.even?
+
+          hash = entries.each_slice(2).to_h
+          return coverband_hash_payload(hash) if coverband_hash_payload?(hash)
+
+          generic_hash_payload(hash, &)
+        end
+
+        def payload_shape?(value)
+          value.is_a?(Hash) && value.keys.any? { |key| PAYLOAD_KEYS.include?(key.to_s) }
+        end
+
+        private
+
+        attr_reader :limits, :deadline
+
+        def generic_hash_payload(hash)
+          payloads = []
+          files = {}
+          hash.each do |field, value|
+            deadline.check!
+            parsed = yield(value)
+            if payload_shape?(parsed)
+              payloads << parsed
+            elsif parsed.is_a?(Hash) || parsed.is_a?(Array)
+              files[field.to_s] = parsed
+            else
+              domain_error('Invalid Redis coverage payload value')
+            end
+          end
+
+          payloads << { 'files' => files } unless files.empty?
+          merge(payloads)
+        end
+
+        def coverband_hash_payload?(hash)
+          hash.key?('file') && hash.key?('file_length')
+        end
+
+        def coverband_hash_payload(hash)
+          payload = {
+            'files' => { hash.fetch('file') => coverband_line_counts(hash) },
+            'observation' => coverband_observation(hash)
+          }
+          limits.validate_payload!(payload)
+        end
+
+        def coverband_line_counts(hash)
+          length = Integer(hash.fetch('file_length'), exception: false)
+          domain_error('Invalid Coverband file length') unless length && length >= 0
+          domain_error('Coverband file length exceeds configured limit') if length > limits.max_array_elements
+
+          Array.new(length) do |index|
+            deadline.check!
+            count = hash[index.to_s]
+            count&.to_i
+          end
+        end
+
+        def coverband_observation(hash)
+          %w[first_updated_at last_updated_at file_hash].each_with_object({}) do |key, observation|
+            observation[key] = hash[key] if hash[key]
+          end
+        end
+
+        def validate_payload_hash!(payload)
+          domain_error('Redis coverage payload must be a mapping') unless payload.is_a?(Hash)
+
+          limits.validate_payload!(payload)
+        end
+
+        def merge_payload(merged, payload)
+          merged.merge(payload) do |key, left, right|
+            case key.to_s
+            when 'files', 'coverage', 'runtime', 'production' then merge_files(left, right)
+            when 'nodes', 'executed' then (Array(left) + Array(right)).uniq
+            when 'edges' then Array(left) + Array(right)
+            when 'observation' then ObservationPolicy.compatible_merge(left, right)
+            else right
+            end
+          end
+        end
+
+        def merge_files(left, right)
+          return right unless left.is_a?(Hash)
+          return left unless right.is_a?(Hash)
+
+          left.merge(right) do |_file, left_lines, right_lines|
+            merge_file_coverage(left_lines, right_lines)
+          end
+        end
+
+        def merge_file_coverage(left, right)
+          return merge_count_hashes(left, right) if left.is_a?(Hash) && right.is_a?(Hash)
+          return merge_count_arrays(left, right) if count_array?(left) || count_array?(right)
+
+          (Array(left) + Array(right)).uniq
+        end
+
+        def merge_count_hashes(left, right)
+          left.merge(right) { |_line, left_count, right_count| left_count.to_i + right_count.to_i }
+        end
+
+        def merge_count_arrays(left, right)
+          length = [Array(left).length, Array(right).length].max
+          domain_error('Merged Redis coverage exceeds configured collection size') if length > limits.max_array_elements
+
+          Array.new(length) do |index|
+            left_value = Array(left)[index]
+            right_value = Array(right)[index]
+            next if left_value.nil? && right_value.nil?
+
+            left_value.to_i + right_value.to_i
+          end
+        end
+
+        def count_array?(value)
+          value.is_a?(Array) && value.any? { |item| item.nil? || item.to_i.zero? }
+        end
+
+        def domain_error(message)
+          raise Error, message
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/analyzers/dynamic/observation_policy.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      module ObservationPolicy
+        module_function
+
+        def metadata(payload = nil, expected_revision: nil, **keyword_payload)
+          payload ||= keyword_payload
+          observation = payload['observation']
+          observation = {} unless observation.is_a?(Hash)
+          schema_version = Integer(payload.fetch('schema_version', 1), exception: false)
+          raise Error, 'Dynamic observation schema version must be 1 or 2' unless [1, 2].include?(schema_version)
+
+          revision = source_revision(payload, observation)
+
+          status = source_revision_status(payload, observation, revision, expected_revision)
+          normalized = observation.merge(
+            'schema_version' => schema_version,
+            'positive_evidence_policy' => 'alive_only',
+            'source_revision_status' => status,
+            'source_revision_policy' => 'accepted_for_liveness_only'
+          )
+          normalized = normalized.merge('source_revision' => revision) if revision
+          normalized = normalized.merge('expected_source_revision' => expected_revision) if expected_revision
+          return normalized unless schema_version == 2
+
+          collector = payload['collector']
+          collector = {} unless collector.is_a?(Hash)
+          source = payload['source']
+          source = {} unless source.is_a?(Hash)
+          scope = payload['scope']
+          scope = {} unless scope.is_a?(Hash)
+          quality = payload['quality']
+          quality = {} unless quality.is_a?(Hash)
+          normalized.merge(
+            'collector_name' => collector['name'] || payload['collector_name'],
+            'collector_version' => collector['version'] || payload['collector_version'],
+            'source_metadata' => source,
+            'environment' => scope['environment'] || observation['environment'],
+            'sample_unit' => scope['sample_unit'] || observation['sample_unit'],
+            'sample_rate' => scope['sample_rate'] || observation['sample_rate'],
+            'quality' => {
+              'dropped_events' => quality.fetch('dropped_events', 0),
+              'overflowed' => quality.fetch('overflowed', false)
+            }
+          ).compact
+        end
+
+        def compatible_merge(left, right)
+          return left.merge(right) unless left.is_a?(Hash) && right.is_a?(Hash)
+
+          %w[source_revision source_revision_status environment collector_name collector_version].each do |key|
+            next if left[key].nil? || right[key].nil? || left[key] == right[key]
+
+            raise Error, "Dynamic observation #{key} is incompatible"
+          end
+          left.merge(right)
+        end
+
+        def evidence_scope(observation)
+          declared_scope = observation['scope']
+          declared_scope = {} unless declared_scope.is_a?(Hash)
+          declared_scope = declared_scope.reject do |key, _value|
+            %w[revision source_revision].include?(key.to_s)
+          end
+          selectors = {
+            'revision' => observation['source_revision'],
+            'source_revision_status' => observation['source_revision_status'],
+            'environment' => observation['environment'],
+            'collector' => observation['collector']
+          }.compact
+          declared_scope.merge(selectors)
+        end
+
+        def source_revision(payload, observation)
+          source = payload['source']
+          source_revision = source['git_sha'] if source.is_a?(Hash)
+          source_revision || payload['source_revision'] || observation['source_revision']
+        end
+        private_class_method :source_revision
+
+        def source_revision_status(payload, observation, revision, expected_revision)
+          stale = payload['stale'] || observation['stale'] || payload['source_revision_status'] == 'stale'
+          return 'stale' if stale
+          return 'unknown' unless revision
+          return 'provided_unverified' unless expected_revision
+
+          revision.to_s == expected_revision.to_s ? 'match' : 'mismatch'
+        end
+        private_class_method :source_revision_status
+      end
+    end
+  end
+end