necropsy 0.1.0 → 0.3.0

data/lib/necropsy/analyzers/dynamic/coverage_importer.rb

@@ -2,6 +2,7 @@
 
 require 'json'
 require 'yaml'
+require_relative 'runtime_reference'
 
 module Necropsy
   module Analyzers
@@ -16,29 +17,64 @@ module Necropsy
           return AnalyzerResult.empty unless source
 
           payload = load_payload(File.expand_path(source, project.root))
-          alive = Array(payload['executed'] || payload['nodes']).map do |node_id|
+          observation = ObservationPolicy.metadata(payload, expected_revision: config['expected_source_revision'])
+          malformed = []
+          alive = node_references(payload).map do |raw_reference|
+            node_id = normalized_or_raw(raw_reference, malformed, :node)
             AliveEvidence.new(
               node_id: node_id,
-              evidence: evidence(kind: :alive, details: "Coverage marked #{node_id} as executed",
-                                 metadata: payload['observation'] || {})
+              evidence: evidence(
+                kind: :alive,
+                details: "Coverage marked #{reference_label(node_id)} as executed",
+                metadata: observation.merge('node_reference' => node_id),
+                grade: :observed,
+                relation: :execution,
+                source: { 'type' => profile.name.to_s, 'node_reference' => node_id },
+                scope: ObservationPolicy.evidence_scope(observation).merge('node_reference' => node_id)
+              )
             )
           end
 
-          edge_evidences = Array(payload['edges']).map do |edge|
-            caller_id = edge['caller_id'] || edge[:caller_id]
-            callee_id = edge['callee_id'] || edge[:callee_id]
+          edge_evidences = edge_references(payload).filter_map do |edge|
+            unless edge.is_a?(Hash)
+              malformed << malformed_reference(:edge, edge)
+              next
+            end
+
+            caller = edge['caller_id'] || edge[:caller_id]
+            callee = edge['callee_id'] || edge[:callee_id]
+            caller_id = normalized_or_raw(caller, malformed, :edge_caller)
+            callee_id = normalized_or_raw(callee, malformed, :edge_callee)
             EdgeEvidence.new(
               caller_id: caller_id,
               callee_id: callee_id,
-              evidence: evidence(kind: :call_edge, details: "Coverage observed #{caller_id} -> #{callee_id}")
+              evidence: evidence(
+                kind: :call_edge,
+                details: "Coverage observed #{reference_label(caller_id)} -> #{reference_label(callee_id)}",
+                metadata: { 'caller_reference' => caller_id, 'callee_reference' => callee_id },
+                grade: :observed,
+                relation: :observed_call,
+                source: {
+                  'type' => profile.name.to_s,
+                  'caller_reference' => caller_id,
+                  'callee_reference' => callee_id
+                },
+                scope: ObservationPolicy.evidence_scope(observation).merge(
+                  'caller_reference' => caller_id,
+                  'callee_reference' => callee_id
+                )
+              )
             )
           end
+          observation = observation.merge('malformed_references' => malformed) unless malformed.empty?
 
           AnalyzerResult.new(
             edge_evidences: edge_evidences,
             alive_evidences: alive,
             uncertainties: {},
-            observation: { 'coverage' => payload['observation'] || {} }
+            observation: { 'coverage' => observation },
+            resolutions: [],
+            evidences: result_evidences(edge_evidences, alive)
           )
         end
 
@@ -47,7 +83,9 @@ module Necropsy
             name: :coverage,
             kind: :dynamic,
             soundness: :observational,
-            description: 'Imports method execution and observed edges from Ruby Coverage output.'
+            description: 'Imports method execution and observed edges from Ruby Coverage output.',
+            version: Necropsy::VERSION,
+            assumptions: ['positive_observations_only']
           )
         end
 
@@ -55,13 +93,69 @@ module Necropsy
 
         attr_reader :config
 
+        def node_references(payload)
+          structured = payload['node_references'] if payload.key?('node_references')
+          RuntimeReference.preferred(structured: structured, legacy: payload['executed'] || payload['nodes'])
+        end
+
+        def edge_references(payload)
+          structured = payload['edge_references'] if payload.key?('edge_references')
+          return Array(payload['edges']) if structured.nil?
+
+          structured_edges = Array(structured)
+          structured_pairs = structured_edges.filter_map { |edge| edge_symbol_pair(edge) }.to_set
+          structured_edges + Array(payload['edges']).reject do |edge|
+            structured_pairs.include?(edge_symbol_pair(edge))
+          end
+        end
+
+        def edge_symbol_pair(edge)
+          return unless edge.is_a?(Hash)
+
+          caller = RuntimeReference.normalize(edge['caller_id'] || edge[:caller_id])
+          callee = RuntimeReference.normalize(edge['callee_id'] || edge[:callee_id])
+          return unless caller && callee
+
+          [reference_symbol(caller), reference_symbol(callee)]
+        end
+
+        def reference_symbol(reference)
+          reference.is_a?(Hash) ? reference['symbol_id'] : reference
+        end
+
+        def normalized_or_raw(reference, malformed, kind)
+          normalized = RuntimeReference.normalize(reference)
+          return normalized if normalized
+
+          malformed << malformed_reference(kind, reference)
+          reference
+        end
+
+        def malformed_reference(kind, reference)
+          { 'kind' => kind.to_s, 'reference' => reference }
+        end
+
+        def reference_label(reference)
+          return reference.to_s unless reference.is_a?(Hash)
+
+          symbol = reference['symbol_id'] || reference[:symbol_id] || '<missing symbol>'
+          file = reference['file'] || reference[:file]
+          line = reference['line'] || reference[:line]
+          location = [file, line].compact.join(':')
+          location.empty? ? symbol.to_s : "#{symbol} at #{location}"
+        end
+
         def load_payload(path)
+          raise Error, "Coverage source does not exist: #{path}" unless File.file?(path)
+
           case File.extname(path)
           when '.json'
             JSON.parse(File.read(path))
           else
-            YAML.load_file(path) || {}
+            YAML.safe_load_file(path, aliases: false) || {}
           end
+        rescue JSON::ParserError, Psych::Exception => e
+          raise Error, "Could not parse coverage source #{path}: #{e.message}"
         end
       end
     end

data/lib/necropsy/analyzers/dynamic/coverband_importer.rb

@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 require 'json'
-require 'openssl'
-require 'socket'
-require 'uri'
 require 'yaml'
 
+require_relative 'redis_payload_loader'
+require_relative 'runtime_reference'
+
 module Necropsy
   module Analyzers
     module Dynamic
@@ -19,11 +19,23 @@ module Necropsy
           return AnalyzerResult.empty unless source
 
           payload = load_payload(source, project.root)
-          alive = alive_from_payload(graph, payload).map do |node|
+          observation = ObservationPolicy.metadata(payload, expected_revision: config['expected_source_revision'])
+          node_ids, line_diagnostics = alive_ids_from_payload(graph, payload)
+          observation = observation.merge(line_diagnostics) unless line_diagnostics.empty?
+          alive = node_ids.map do |node_id|
+            node = graph.nodes.lookup(node_id).node
+            details = node ? "Coverband executed #{node.file}:#{node.line}" : "Coverband marked #{node_id} as executed"
             AliveEvidence.new(
-              node_id: node.id,
-              evidence: evidence(kind: :alive, details: "Coverband executed #{node.file}:#{node.line}",
-                                 metadata: payload['observation'] || {})
+              node_id: node_id,
+              evidence: evidence(
+                kind: :alive,
+                details: details,
+                metadata: observation.merge('node_reference' => node_id),
+                grade: :observed,
+                relation: :execution,
+                source: { 'type' => 'coverband', 'node_reference' => node_id },
+                scope: ObservationPolicy.evidence_scope(observation).merge('node_reference' => node_id)
+              )
             )
           end
 
@@ -31,7 +43,9 @@ module Necropsy
             edge_evidences: [],
             alive_evidences: alive,
             uncertainties: {},
-            observation: { 'coverband' => payload['observation'] || {} }
+            observation: { 'coverband' => observation },
+            resolutions: [],
+            evidences: result_evidences(alive)
           )
         end
 
@@ -40,7 +54,9 @@ module Necropsy
             name: :coverband,
             kind: :dynamic,
             soundness: :observational,
-            description: 'Imports file and line execution data compatible with coverband-style exports.'
+            description: 'Imports file and line execution data compatible with coverband-style exports.',
+            version: Necropsy::VERSION,
+            assumptions: %w[line_execution_mapping positive_observations_only]
           )
         end
 
@@ -52,29 +68,41 @@ module Necropsy
           return load_redis_payload(source) if source.start_with?('redis://', 'rediss://')
 
           path = File.expand_path(source, root)
+          raise Error, "Coverband source does not exist: #{path}" unless File.file?(path)
+
           case File.extname(path)
           when '.json'
             JSON.parse(File.read(path))
           else
-            YAML.load_file(path) || {}
+            YAML.safe_load_file(path, aliases: true) || {}
           end
+        rescue JSON::ParserError, Psych::Exception => e
+          raise Error, "Could not parse Coverband source #{path}: #{e.message}"
         end
 
         def load_redis_payload(source)
           RedisPayloadLoader.new(source: source, config: config).load
         end
 
-        def alive_from_payload(graph, payload)
-          executed_nodes = Array(payload['executed'] || payload['nodes'])
-          by_id = executed_nodes.filter_map { |id| graph.nodes[id] }
+        def alive_ids_from_payload(graph, payload)
+          structured = payload['node_references'] if payload.key?('node_references')
+          malformed = []
+          executed_nodes = RuntimeReference.preferred(
+            structured: structured,
+            legacy: payload['executed'] || payload['nodes']
+          ).map do |reference|
+            normalized = RuntimeReference.normalize(reference)
+            next normalized if normalized
 
-          files = coverage_files(payload)
-          by_line = graph.method_nodes.select do |node|
-            executed_lines = lines_for(files, node.file)
-            executed_lines.any? { |line| line.between?(node.line, node.end_line) }
+            malformed << { 'kind' => 'node', 'reference' => reference }
+            reference
           end
 
-          (by_id + by_line).uniq(&:id)
+          files = coverage_files(payload)
+          by_line, diagnostics = line_references(graph, files)
+          diagnostics['malformed_references'] = malformed unless malformed.empty?
+
+          [(executed_nodes + by_line).uniq, diagnostics]
         end
 
         def coverage_files(payload)
@@ -131,244 +159,40 @@ module Necropsy
           left.merge(right) { |_file, left_lines, right_lines| (left_lines + right_lines).uniq }
         end
 
-        def lines_for(files, relative_file)
-          exact = files[relative_file] || files[File.join('.', relative_file)]
-          return exact if exact
-
-          files.each do |file, lines|
-            return lines if file.end_with?("/#{relative_file}")
-          end
-
-          []
-        end
-      end
-
-      class RedisPayloadLoader
-        DEFAULT_PATTERN = 'coverband*'
-
-        def initialize(source:, config:)
-          @uri = URI(source)
-          @config = config
-          @socket = nil
-        end
-
-        def load
-          connect
-          authenticate
-          select_database
-          merge_payloads(keys.filter_map { |key| payload_for_key(key) })
-        ensure
-          socket&.close
-        end
-
-        private
-
-        attr_reader :uri, :config, :socket
-
-        def connect
-          tcp_socket = TCPSocket.new(uri.host, uri.port || 6379)
-          @socket = uri.scheme == 'rediss' ? tls_socket(tcp_socket) : tcp_socket
-        end
-
-        def tls_socket(tcp_socket)
-          context = OpenSSL::SSL::SSLContext.new
-          socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, context)
-          socket.hostname = uri.host if socket.respond_to?(:hostname=)
-          socket.connect
-          socket
-        end
-
-        def authenticate
-          return unless uri.password
-
-          command('AUTH', uri.password)
-        end
-
-        def select_database
-          db = uri.path.to_s.delete_prefix('/')
-          command('SELECT', db) unless db.empty?
-        end
-
-        def keys
-          configured = Array(config['keys'] || config['key'] || query['key']).compact
-          return configured unless configured.empty?
-
-          scan(pattern: config['pattern'] || query['pattern'] || DEFAULT_PATTERN)
-        end
-
-        def query
-          @query ||= URI.decode_www_form(uri.query.to_s).to_h
-        end
-
-        def scan(pattern:)
-          cursor = '0'
-          found = []
-          loop do
-            cursor, keys = command('SCAN', cursor, 'MATCH', pattern)
-            found.concat(keys)
-            break if cursor == '0'
-          end
-          found
-        end
-
-        def command(*parts)
-          socket.write(redis_command(parts))
-          read_response
-        end
-
-        def payload_for_key(key)
-          parse_payload(command('GET', key))
-        rescue Error => e
-          raise unless e.message.include?('WRONGTYPE')
-
-          parse_hash_payload(command('HGETALL', key))
-        end
-
-        def redis_command(parts)
-          ["*#{parts.length}", *parts.flat_map { |part| ["$#{part.to_s.bytesize}", part.to_s] }].join("\r\n") + "\r\n"
-        end
-
-        def read_response
-          prefix = socket.read(1)
-          case prefix
-          when '+'
-            socket.gets("\r\n").delete_suffix("\r\n")
-          when '-'
-            raise Error, socket.gets("\r\n").delete_suffix("\r\n")
-          when ':'
-            socket.gets("\r\n").to_i
-          when '$'
-            read_bulk_string
-          when '*'
-            Array.new(socket.gets("\r\n").to_i) { read_response }
-          else
-            raise Error, "Unsupported Redis response prefix #{prefix.inspect}"
-          end
-        end
-
-        def read_bulk_string
-          length = socket.gets("\r\n").to_i
-          return nil if length.negative?
-
-          value = socket.read(length)
-          socket.read(2)
-          value
-        end
-
-        def parse_payload(value)
-          return nil if value.nil? || value.empty?
-
-          JSON.parse(value)
-        rescue JSON::ParserError
-          YAML.load(value)
-        rescue Psych::SyntaxError
-          Marshal.load(value)
-        end
-
-        def parse_hash_payload(entries)
-          hash = Array(entries).each_slice(2).to_h
-          return coverband_hash_payload(hash) if coverband_hash_payload?(hash)
-
-          payloads = []
-          files = {}
-          hash.each do |field, value|
-            parsed = parse_payload(value)
-            if payload_shape?(parsed)
-              payloads << parsed
-            else
-              files[field.to_s] = parsed
-            end
-          end
-
-          payloads << { 'files' => files } unless files.empty?
-          merge_payloads(payloads)
-        end
-
-        def coverband_hash_payload?(hash)
-          hash.key?('file') && hash.key?('file_length')
-        end
-
-        def coverband_hash_payload(hash)
-          {
-            'files' => { hash.fetch('file') => coverband_line_counts(hash) },
-            'observation' => coverband_observation(hash)
-          }
-        end
-
-        def coverband_line_counts(hash)
-          length = hash.fetch('file_length').to_i
-          Array.new(length) do |index|
-            count = hash[index.to_s]
-            count&.to_i
-          end
-        end
-
-        def coverband_observation(hash)
-          %w[first_updated_at last_updated_at file_hash].each_with_object({}) do |key, observation|
-            observation[key] = hash[key] if hash[key]
-          end
-        end
-
-        def payload_shape?(value)
-          value.is_a?(Hash) && value.keys.any? do |key|
-            %w[files coverage runtime production nodes executed observation].include?(key.to_s)
-          end
-        end
-
-        def merge_payloads(payloads)
-          payloads.compact.reduce({}) { |merged, payload| merge_payload(merged, payload) }
-        end
-
-        def merge_payload(merged, payload)
-          merged.merge(payload) do |key, left, right|
-            case key
-            when 'files', 'coverage', 'runtime', 'production'
-              merge_files(left, right)
-            when 'nodes', 'executed'
-              (Array(left) + Array(right)).uniq
-            when 'edges'
-              Array(left) + Array(right)
-            when 'observation'
-              left.merge(right)
-            else
-              right
+        def line_references(graph, files)
+          definition_ids = []
+          line_ambiguities = []
+          path_ambiguities = []
+          graph.method_nodes.group_by(&:file).sort.each do |relative_file, definitions|
+            paths = matching_coverage_paths(files, relative_file)
+            next if paths.empty?
+
+            path_ambiguities << { 'file' => relative_file, 'coverage_paths' => paths } if paths.length > 1
+            paths.flat_map { |path| files.fetch(path) }.uniq.sort.each do |line|
+              matches = definitions.select { |definition| line.between?(definition.line, definition.end_line) }
+              definition_ids.concat(matches.map(&:graph_id))
+              next unless matches.length > 1
+
+              line_ambiguities << {
+                'file' => relative_file,
+                'line' => line,
+                'definition_ids' => matches.map(&:graph_id).sort
+              }
             end
           end
-        end
 
-        def merge_files(left, right)
-          return right unless left.is_a?(Hash)
-          return left unless right.is_a?(Hash)
-
-          left.merge(right) do |_file, left_lines, right_lines|
-            merge_file_coverage(left_lines, right_lines)
-          end
-        end
-
-        def merge_file_coverage(left, right)
-          return merge_count_hashes(left, right) if left.is_a?(Hash) && right.is_a?(Hash)
-          return merge_count_arrays(left, right) if count_array?(left) || count_array?(right)
-
-          (Array(left) + Array(right)).uniq
+          diagnostics = {}
+          diagnostics['path_ambiguities'] = path_ambiguities unless path_ambiguities.empty?
+          diagnostics['line_ambiguities'] = line_ambiguities unless line_ambiguities.empty?
+          [definition_ids.uniq.sort, diagnostics]
         end
 
-        def merge_count_hashes(left, right)
-          left.merge(right) { |_line, left_count, right_count| left_count.to_i + right_count.to_i }
-        end
-
-        def merge_count_arrays(left, right)
-          length = [Array(left).length, Array(right).length].max
-          Array.new(length) do |index|
-            left_value = Array(left)[index]
-            right_value = Array(right)[index]
-            next if left_value.nil? && right_value.nil?
-
-            left_value.to_i + right_value.to_i
-          end
-        end
+        def matching_coverage_paths(files, relative_file)
+          exact = [relative_file, File.join('.', relative_file)].select { |path| files.key?(path) }
+          return exact.sort unless exact.empty?
 
-        def count_array?(value)
-          value.is_a?(Array) && value.any? { |item| item.nil? || item.to_i.zero? }
+          suffix = "/#{relative_file}"
+          files.keys.select { |path| path.end_with?(suffix) }.sort
         end
       end
     end