necropsy 0.1.0 → 0.3.0

data/lib/necropsy/definition_identity/canonical_digest.rb

@@ -0,0 +1,278 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'json'
+
+module Necropsy
+  module DefinitionIdentity
+    class CanonicalizationError < StandardError; end
+    class LimitExceeded < CanonicalizationError; end
+    class CycleError < CanonicalizationError; end
+    class UnsupportedTypeError < CanonicalizationError; end
+
+    class CanonicalDigest
+      MAX_DEPTH = 256
+      MAX_ITEMS = 1_000_000
+      MAX_SCALAR_BYTES = 64 * 1024 * 1024
+      MAX_TOTAL_BYTES = 256 * 1024 * 1024
+
+      def initialize(max_depth: MAX_DEPTH, max_items: MAX_ITEMS, max_scalar_bytes: MAX_SCALAR_BYTES,
+                     max_total_bytes: MAX_TOTAL_BYTES)
+        @max_depth = positive_limit(max_depth, :max_depth)
+        @max_items = positive_limit(max_items, :max_items)
+        @max_scalar_bytes = positive_limit(max_scalar_bytes, :max_scalar_bytes)
+        @max_total_bytes = positive_limit(max_total_bytes, :max_total_bytes)
+      end
+
+      def hexdigest(value)
+        reset_state
+        output = DigestOutput.new(byte_budget)
+        process([[:value, value, 0, output]])
+        output.hexdigest
+      end
+
+      def hexdigest_payload(values)
+        reset_state
+        output = DigestOutput.new(byte_budget)
+        output.write('[')
+        values.each_with_index do |value, index|
+          count_item!
+          output.write(',') if index.positive?
+          output.write(payload_scalar(value))
+        end
+        output.write(']')
+        output.hexdigest
+      end
+
+      private
+
+      attr_reader :max_depth, :max_items, :max_scalar_bytes, :max_total_bytes, :active_containers, :byte_budget
+
+      def reset_state
+        @active_containers = {}
+        @items = 0
+        @byte_budget = ByteBudget.new(max_total_bytes)
+      end
+
+      def process(stack)
+        until stack.empty?
+          action, *arguments = stack.pop
+          send("process_#{action}", stack, *arguments)
+        end
+      rescue CanonicalizationError
+        raise
+      rescue SystemStackError => e
+        raise CanonicalizationError, "Could not canonicalize definition: #{e.message}"
+      rescue StandardError => e
+        raise CanonicalizationError, "Could not canonicalize definition: #{e.class}: #{e.message}"
+      end
+
+      def process_value(stack, value, depth, output)
+        count_item!
+        check_depth!(depth)
+        case value
+        when Prism::Node then start_node(stack, value, depth, output)
+        when Prism::Location then output.write('null')
+        when Array then start_array(stack, value, depth, output)
+        when Hash then start_hash(stack, value, depth, output)
+        when Symbol then tagged_scalar(output, 'symbol', value.to_s)
+        when String then tagged_scalar(output, 'string', value)
+        when Integer then tagged_scalar(output, 'integer', value.to_s)
+        when Float then write_float(output, value)
+        when true, false then output.write(%(["boolean",#{value}]))
+        when nil then output.write('["nil"]')
+        else raise UnsupportedTypeError, "Unsupported definition payload type: #{value.class}"
+        end
+      end
+
+      def start_node(stack, node, depth, output)
+        enter_container!(node)
+        fields = node.deconstruct_keys(nil).filter_map do |key, value|
+          [key.to_s, value] unless DefinitionIdentity.send(:excluded_key?, key)
+        end.sort_by(&:first)
+        type = checked_string(node.type.to_s)
+        flags = node.send(:flags) & ~Prism::NodeFlags::NEWLINE
+        output.write("[\"node\",#{json_string(type)},#{flags},[")
+        stack << [:node_field, node, fields, 0, depth, output]
+      end
+
+      def process_node_field(stack, node, fields, index, depth, output)
+        if index >= fields.length
+          output.write(']]')
+          leave_container(node)
+          return
+        end
+
+        key, value = fields.fetch(index)
+        output.write(',') if index.positive?
+        output.write("[#{json_string(checked_string(key))},")
+        stack << [:node_field_complete, node, fields, index, depth, output]
+        stack << [:value, value, depth + 1, output]
+      end
+
+      def process_node_field_complete(stack, node, fields, index, depth, output)
+        output.write(']')
+        stack << [:node_field, node, fields, index + 1, depth, output]
+      end
+
+      def start_array(stack, value, depth, output)
+        enter_container!(value)
+        output.write('["array",[')
+        stack << [:array_next, value, 0, depth, output]
+      end
+
+      def process_array_next(stack, value, index, depth, output)
+        if index >= value.length
+          output.write(']]')
+          leave_container(value)
+          return
+        end
+
+        output.write(',') if index.positive?
+        stack << [:array_next, value, index + 1, depth, output]
+        stack << [:value, value.fetch(index), depth + 1, output]
+      end
+
+      def start_hash(stack, value, depth, output)
+        enter_container!(value)
+        stack << [:hash_next, value, value.each_pair, [], depth, output]
+      end
+
+      def process_hash_next(stack, value, iterator, pairs, depth, output)
+        key, item = iterator.next
+        pair = StringOutput.new(byte_budget)
+        pair.write('[')
+        stack << [:hash_pair_complete, value, iterator, pairs, depth, output, pair]
+        stack << [:raw, pair, ']']
+        stack << [:value, item, depth + 1, pair]
+        stack << [:raw, pair, ',']
+        stack << [:value, key, depth + 1, pair]
+      rescue StopIteration
+        output.write('["hash",[')
+        pairs.sort.each_with_index do |pair, index|
+          output.write(',') if index.positive?
+          output.write(pair)
+        end
+        output.write(']]')
+        leave_container(value)
+      end
+
+      def process_hash_pair_complete(stack, value, iterator, pairs, depth, output, pair)
+        pairs << pair.to_s
+        stack << [:hash_next, value, iterator, pairs, depth, output]
+      end
+
+      def process_raw(_stack, output, bytes)
+        output.write(bytes)
+      end
+
+      def tagged_scalar(output, tag, value)
+        output.write(%([#{json_string(tag)},#{json_string_or_fingerprint(value)}]))
+      end
+
+      def write_float(output, value)
+        raise UnsupportedTypeError, 'Definition payload contains a non-finite float' unless value.finite?
+
+        tagged_scalar(output, 'float', value.to_s)
+      end
+
+      def payload_scalar(value)
+        case value
+        when String then json_string_or_fingerprint(value)
+        when Integer then value.to_s
+        else raise UnsupportedTypeError, "Unsupported definition identity component: #{value.class}"
+        end
+      end
+
+      def json_string_or_fingerprint(value)
+        checked_string(value)
+        JSON.generate(value)
+      rescue JSON::GeneratorError, EncodingError
+        JSON.generate(['invalid_string', value.encoding.name, value.bytesize, Digest::SHA256.hexdigest(value.b)])
+      end
+
+      def json_string(value)
+        JSON.generate(value)
+      end
+
+      def checked_string(value)
+        return value if value.bytesize <= max_scalar_bytes
+
+        raise LimitExceeded, "Canonical scalar exceeds maximum size #{max_scalar_bytes}"
+      end
+
+      def enter_container!(value)
+        object_id = value.object_id
+        raise CycleError, "Definition payload contains a cycle at #{value.class}" if active_containers.key?(object_id)
+
+        active_containers[object_id] = true
+      end
+
+      def leave_container(value)
+        active_containers.delete(value.object_id)
+      end
+
+      def check_depth!(depth)
+        return if depth <= max_depth
+
+        raise LimitExceeded, "Canonical payload exceeds maximum depth #{max_depth}"
+      end
+
+      def count_item!
+        @items += 1
+        raise LimitExceeded, "Canonical payload exceeds maximum item count #{max_items}" if @items > max_items
+      end
+
+      def positive_limit(value, name)
+        value = Integer(value)
+        raise ArgumentError, "#{name} must be positive" unless value.positive?
+
+        value
+      end
+
+      class DigestOutput
+        def initialize(budget)
+          @budget = budget
+          @digest = Digest::SHA256.new
+        end
+
+        def write(value)
+          @budget.consume(value.bytesize)
+          @digest.update(value.b)
+        end
+
+        def hexdigest
+          @digest.hexdigest
+        end
+      end
+
+      class StringOutput
+        def initialize(budget)
+          @budget = budget
+          @value = String.new(encoding: Encoding::BINARY)
+        end
+
+        def write(value)
+          @budget.consume(value.bytesize)
+          @value << value.b
+        end
+
+        def to_s
+          @value
+        end
+      end
+
+      class ByteBudget
+        def initialize(limit)
+          @limit = limit
+          @bytes = 0
+        end
+
+        def consume(bytes)
+          @bytes += bytes
+          raise LimitExceeded, "Canonical payload exceeds maximum size #{@limit}" if @bytes > @limit
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/definition_identity.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'prism'
+require_relative 'definition_identity/canonical_digest'
+
+module Necropsy
+  module DefinitionIdentity
+    VERSION = 1
+    PREFIX = "def:v#{VERSION}".freeze
+    EXCLUDED_KEYS = %i[node_id location flags].freeze
+
+    module_function
+
+    def body_digest(node, **limits)
+      CanonicalDigest.new(**limits).hexdigest(node)
+    end
+
+    def definition_id(kind:, symbol_id:, relative_path:, body_digest:, ordinal:)
+      ordinal = Integer(ordinal)
+      raise ArgumentError, 'definition ordinal must be positive' unless ordinal.positive?
+
+      payload = [kind.to_s, symbol_id.to_s, relative_path.to_s, body_digest.to_s, ordinal]
+      "#{PREFIX}:#{CanonicalDigest.new.hexdigest_payload(payload)}"
+    end
+
+    def file_root_id(relative_path:)
+      payload = ['file_root', relative_path.to_s]
+      "#{PREFIX}:#{CanonicalDigest.new.hexdigest_payload(payload)}"
+    end
+
+    def excluded_key?(key)
+      name = key.to_s
+      EXCLUDED_KEYS.include?(key.to_sym) || name.end_with?('_loc', 'comment', 'comments')
+    end
+    private_class_method :excluded_key?
+  end
+end

data/lib/necropsy/diagnostics.rb

@@ -0,0 +1,342 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'shellwords'
+
+module Necropsy
+  class Diagnostics
+    FORMATS = %i[human json].freeze
+
+    def initialize(report)
+      raise Error, 'Reachability witnesses are unavailable for this report' unless report.reachability
+
+      @report = report
+    end
+
+    def why(node_id)
+      lookup = graph.nodes.lookup(node_id)
+      return with_source_incompleteness(ambiguous_payload(lookup)) if lookup.ambiguous?
+      return with_source_incompleteness(missing_payload(node_id)) if lookup.missing?
+
+      node = lookup.node
+
+      graph_id = node.graph_id
+      runtime_path = reachability.witness(graph_id)
+      return with_source_incompleteness(alive_payload(node, runtime_path, :runtime)) if runtime_path
+
+      external_path = reachability.witness(graph_id, kind: :external)
+      return with_source_incompleteness(alive_payload(node, external_path, :external)) if external_path
+
+      finding = finding_for(graph_id)
+      return with_source_incompleteness(dead_payload(node, finding)) if finding&.classification == :blocked
+
+      test_path = reachability.witness(graph_id, kind: :test)
+      return with_source_incompleteness(alive_payload(node, test_path, :test)) if test_path
+
+      with_source_incompleteness(dead_payload(node, finding))
+    end
+
+    def explain(node_id)
+      lookup = graph.nodes.lookup(node_id)
+      return with_source_incompleteness(ambiguous_payload(lookup)) if lookup.ambiguous?
+      return with_source_incompleteness(missing_payload(node_id)) if lookup.missing?
+
+      node = lookup.node
+
+      finding = finding_for(node.graph_id)
+      return with_source_incompleteness({ 'status' => 'alive', 'node' => node.to_h }) unless finding
+
+      payload = {
+        'status' => 'finding',
+        'node' => node.to_h,
+        'classification' => finding.classification.to_s,
+        'confidence' => finding.confidence.to_s,
+        'score' => finding.score,
+        'components' => finding.score_components.map(&:to_h),
+        'reasons' => finding.reasons,
+        'blockers' => finding.blockers.map(&:to_h)
+      }
+      with_source_incompleteness(payload)
+    end
+
+    def why_not(node_id)
+      lookup = graph.nodes.lookup(node_id)
+      return with_source_incompleteness(ambiguous_payload(lookup)) if lookup.ambiguous?
+      return with_source_incompleteness(missing_payload(node_id)) if lookup.missing?
+
+      WhyNotExplanation.new(report).call(lookup.node)
+    end
+
+    def render(payload, format: :human)
+      normalized = format.to_sym
+      raise Error, "Diagnostics support only human or json output, not #{format}" unless FORMATS.include?(normalized)
+
+      return JSON.pretty_generate(payload) if normalized == :json
+
+      render_human(payload)
+    end
+
+    private
+
+    attr_reader :report
+
+    def graph
+      report.graph
+    end
+
+    def reachability
+      report.reachability
+    end
+
+    def alive_payload(node, path, kind)
+      {
+        'status' => 'alive',
+        'kind' => kind.to_s,
+        'node' => node.to_h,
+        'path' => path.each_with_index.map do |node_id, index|
+          caller_id = index.positive? ? path[index - 1] : nil
+          path_step(node_id, caller_id, kind)
+        end
+      }
+    end
+
+    def path_step(node_id, caller_id, root_domain)
+      step = { 'node' => graph.nodes.fetch(node_id).to_h }
+      roots = graph.entry_points.select do |candidate|
+        candidate.node_id == node_id && candidate.domain == root_domain
+      end.sort_by { |candidate| [candidate.reason.to_s, BoundedCanonicalizer.dump(candidate.evidence)] }
+      if roots.any?
+        step['entry_reason'] = roots.first.reason.to_s
+        step['root_domain'] = root_domain.to_s
+        step['root_provenance'] = roots.map(&:to_h)
+      end
+      return step unless caller_id
+
+      evidences = graph.edges_from(caller_id).fetch(node_id, [])
+      step['edge'] = {
+        'caller_id' => caller_id,
+        'callee_id' => node_id,
+        'caller' => graph.nodes.fetch(caller_id).to_h,
+        'callee' => graph.nodes.fetch(node_id).to_h,
+        'evidences' => evidences.map(&:to_h)
+      }
+      step
+    end
+
+    def dead_payload(node, finding = finding_for(node.graph_id))
+      blockers = finding ? finding.blockers : graph.matching_blockers(node)
+      {
+        'status' => finding&.classification == :blocked ? 'blocked' : 'dead',
+        'node' => node.to_h,
+        'classification' => finding&.classification&.to_s,
+        'nearest_alive' => nearest_alive(node.graph_id),
+        'uncertainties' => graph.uncertainties(node.graph_id),
+        'blockers' => blockers.map(&:to_h)
+      }
+    end
+
+    def finding_for(node_id)
+      report.findings.find { |candidate| candidate.node.graph_id == node_id }
+    end
+
+    def nearest_alive(node_id)
+      kinds = reachability.runtime_alive.to_h { |id| [id, 'runtime'] }
+      reachability.external_alive.each { |id| kinds[id] ||= 'external' }
+      reachability.test_alive.each { |id| kinds[id] ||= 'test' }
+      visited = { node_id => 0 }
+      queue = [node_id]
+
+      until queue.empty?
+        current = queue.shift
+        neighbors(current).each do |neighbor|
+          next if visited.key?(neighbor)
+
+          distance = visited.fetch(current) + 1
+          if kinds.key?(neighbor)
+            node = graph.nodes.fetch(neighbor)
+            return {
+              'node_id' => neighbor,
+              'symbol_id' => node.symbol_id,
+              'definition_id' => node.definition_id,
+              'node' => node.to_h,
+              'kind' => kinds.fetch(neighbor),
+              'distance' => distance
+            }
+          end
+
+          visited[neighbor] = distance
+          queue << neighbor
+        end
+      end
+      nil
+    end
+
+    def ambiguous_payload(lookup)
+      {
+        'status' => 'ambiguous',
+        'node_id' => lookup.identifier,
+        'definitions' => lookup.definitions.map do |definition|
+          {
+            'symbol_id' => definition.symbol_id,
+            'definition_id' => definition.definition_id,
+            'file' => definition.file,
+            'line' => definition.line,
+            'commands' => {
+              'why' => diagnostic_command('why', definition.definition_id),
+              'why_not' => diagnostic_command('why-not', definition.definition_id),
+              'explain' => diagnostic_command('explain', definition.definition_id)
+            }
+          }
+        end
+      }
+    end
+
+    def diagnostic_command(command, definition_id)
+      "bundle exec necropsy #{command} #{Shellwords.escape(definition_id)} --root #{Shellwords.escape(report.root)}"
+    end
+
+    def neighbors(node_id)
+      outgoing = graph.edges_from(node_id).keys
+      incoming = graph.incoming_edges(node_id).map(&:caller_id)
+      (outgoing + incoming).uniq.sort
+    end
+
+    def missing_payload(node_id)
+      terms = [node_id, node_id.split(/[.#]/).last].compact.map(&:downcase).reject { |term| term.length < 2 }
+      suggestions = graph.nodes.values.map(&:symbol_id).uniq.select do |candidate|
+        terms.any? { |term| candidate.downcase[term] }
+      end.sort.first(10)
+      { 'status' => 'not_found', 'node_id' => node_id, 'suggestions' => suggestions }
+    end
+
+    def render_human(payload)
+      rendered = case payload.fetch('status')
+                 when 'alive' then render_alive(payload)
+                 when 'dead', 'blocked' then render_dead(payload)
+                 when 'finding' then render_explanation(payload)
+                 when 'why_not' then WhyNotRenderer.new(payload).render
+                 when 'ambiguous' then render_ambiguous(payload)
+                 when 'not_found' then render_missing(payload)
+                 else "#{node_reference(payload['node'])} is alive and has no dead-code finding."
+                 end
+      append_source_incompleteness(rendered, payload)
+    end
+
+    def render_alive(payload)
+      lines = ["Alive (#{payload.fetch('kind')}): #{node_reference(payload.fetch('node'))}"]
+      payload.fetch('path').each_with_index do |step, index|
+        suffix = step['entry_reason'] ? " entry=#{step['entry_reason']}" : ''
+        suffix += " domain=#{step['root_domain']}" if step['root_domain']
+        lines << "  #{index}. #{node_reference(step.fetch('node'))}#{suffix}"
+        Array(step.dig('edge', 'evidences')).each { |evidence| lines << render_evidence(evidence) }
+      end
+      lines.join("\n")
+    end
+
+    def render_evidence(evidence)
+      metadata = evidence.fetch('metadata', {})
+      location = [metadata['file'], metadata['line']].compact.join(':')
+      suffix = location.empty? ? '' : " at #{location}"
+      "     via #{evidence['analyzer']} weight=#{evidence['weight']}#{suffix}: #{evidence['details']}"
+    end
+
+    def render_dead(payload)
+      heading = payload['status'] == 'blocked' ? 'Blocked' : 'Dead'
+      lines = ["#{heading}: #{node_reference(payload.fetch('node'))}"]
+      lines << "Classification: #{payload['classification']}" if payload['classification']
+      nearest = payload['nearest_alive']
+      lines << if nearest
+                 "Nearest alive: #{node_reference(nearest['node'])} " \
+                   "(#{nearest['kind']}, distance #{nearest['distance']})"
+               else
+                 'Nearest alive: none'
+               end
+      lines << 'Uncertainties: none' if payload.fetch('uncertainties').empty?
+      payload.fetch('uncertainties').each { |message| lines << "Uncertainty: #{message}" }
+      append_blockers(lines, payload.fetch('blockers', []))
+      lines.join("\n")
+    end
+
+    def render_explanation(payload)
+      lines = [
+        "#{node_reference(payload.fetch('node'))}: #{payload['classification']}",
+        "Confidence: #{payload['confidence']} (score #{format('%.2f', payload['score'])})"
+      ]
+      payload.fetch('components').each do |component|
+        lines << format(
+          '  %<name>-28s %<value>+0.2f  %<details>s',
+          name: component['name'], value: component['value'], details: component['details']
+        )
+      end
+      lines << format('  %<name>-28s  %<value>0.2f', name: 'total', value: payload['score'])
+      append_blockers(lines, payload.fetch('blockers', []))
+      lines.join("\n")
+    end
+
+    def render_ambiguous(payload)
+      definitions = payload.fetch('definitions')
+      lines = [
+        "Ambiguous symbol ID: #{payload['node_id']}",
+        "Matched #{definitions.length} physical definitions:"
+      ]
+      definitions.each do |definition|
+        lines << "  #{definition['symbol_id']} [#{definition['definition_id']}] " \
+                 "#{definition['file']}:#{definition['line']}"
+        lines << "    why: #{definition.dig('commands', 'why')}"
+        lines << "    why-not: #{definition.dig('commands', 'why_not')}"
+        lines << "    explain: #{definition.dig('commands', 'explain')}"
+      end
+      lines.join("\n")
+    end
+
+    def append_blockers(lines, blockers)
+      blockers.each do |blocker|
+        metadata = blocker.fetch('metadata', {})
+        location = [metadata['file'], metadata['line']].compact.join(':')
+        caller = metadata['caller_id'] ? " caller=#{metadata['caller_id']}" : ''
+        lines << "Blocker: #{blocker['kind']} at #{location}#{caller}"
+        lines << "  Scope: #{blocker['scope_kind']}=#{blocker['scope_value'].inspect} message=#{metadata['message']}"
+        lines << "  Reason: #{blocker['reason']}"
+        lines << "  Match: #{metadata['snippet']}" if metadata['snippet']
+      end
+    end
+
+    def render_missing(payload)
+      lines = ["Node not found: #{payload['node_id']}"]
+      lines << "Suggestions: #{payload['suggestions'].join(', ')}" unless payload['suggestions'].empty?
+      lines.join("\n")
+    end
+
+    def node_reference(node)
+      return 'unknown definition' unless node
+
+      symbol_id = node['symbol_id'] || node['id']
+      definition_id = node['definition_id'] || node['id']
+      "#{symbol_id} [#{definition_id}]"
+    end
+
+    def with_source_incompleteness(payload)
+      return payload if graph.incomplete_files.empty?
+
+      payload.merge('source_incompleteness' => graph.source_incompleteness)
+    end
+
+    def append_source_incompleteness(rendered, payload)
+      diagnostic = payload['source_incompleteness']
+      return rendered unless diagnostic
+
+      lines = [rendered, "Incomplete source files: #{diagnostic['incomplete_files']}"]
+      diagnostic.fetch('files').each do |file|
+        errors = file.fetch('errors')
+        if errors.empty?
+          lines << "  #{file['file']}:1 [#{file['status']}]"
+        else
+          errors.each do |error|
+            lines << "  #{error['file']}:#{error['line']} [#{error['type']}] #{error['message']}"
+          end
+        end
+      end
+      lines.join("\n")
+    end
+  end
+end