necropsy 0.2.0 → 0.3.0

data/lib/necropsy/graph/definition_index.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Necropsy
+  class DefinitionIndex
+    include Enumerable
+
+    Lookup = Data.define(:identifier, :status, :definitions) do
+      def exact?
+        status == :exact
+      end
+
+      def unique?
+        status == :unique
+      end
+
+      def ambiguous?
+        status == :ambiguous
+      end
+
+      def missing?
+        status == :missing
+      end
+
+      def node
+        definitions.first if exact? || unique?
+      end
+    end
+
+    class AmbiguousDefinitionError < KeyError
+      attr_reader :lookup
+
+      def initialize(lookup)
+        @lookup = lookup
+        super("#{lookup.identifier.inspect} matches #{lookup.definitions.length} physical definitions")
+      end
+    end
+
+    UNDEFINED = Object.new.freeze
+    private_constant :UNDEFINED
+
+    def initialize(definitions = [])
+      @by_definition_id = {}
+      @by_symbol_id = Hash.new { |hash, key| hash[key] = [] }
+      definitions.each { |definition| add(definition) }
+    end
+
+    def add(definition)
+      existing = @by_definition_id[definition.graph_id]
+      return existing if existing == definition
+      raise ArgumentError, "duplicate physical definition ID: #{definition.graph_id}" if existing
+
+      @by_definition_id[definition.graph_id] = definition
+      @by_symbol_id[definition.symbol_id] << definition
+      definition
+    end
+
+    def exact(definition_id)
+      @by_definition_id[definition_id]
+    end
+
+    def definitions_for(symbol_id)
+      sorted(@by_symbol_id.fetch(symbol_id, [])).freeze
+    end
+
+    def lookup(identifier)
+      definitions = definitions_for(identifier)
+      unless definitions.empty?
+        status = definitions.one? ? :unique : :ambiguous
+        return Lookup.new(identifier: identifier, status: status, definitions: definitions)
+      end
+
+      exact_match = exact(identifier)
+      return Lookup.new(identifier: identifier, status: :exact, definitions: [exact_match].freeze) if exact_match
+
+      Lookup.new(identifier: identifier, status: :missing, definitions: definitions)
+    end
+
+    def [](identifier)
+      result = lookup(identifier)
+      raise AmbiguousDefinitionError, result if result.ambiguous?
+
+      result.node
+    end
+
+    def resolve(identifier)
+      lookup(identifier).definitions
+    end
+
+    def fetch(identifier, default = UNDEFINED)
+      result = lookup(identifier)
+      raise AmbiguousDefinitionError, result if result.ambiguous?
+      return result.node unless result.missing?
+      return yield(identifier) if block_given?
+      return default unless default.equal?(UNDEFINED)
+
+      raise KeyError, "key not found: #{identifier.inspect}"
+    end
+
+    def key?(identifier)
+      !lookup(identifier).missing?
+    end
+
+    def each
+      return enum_for(:each) unless block_given?
+
+      values.each { |definition| yield(definition.graph_id, definition) }
+      self
+    end
+
+    def each_key
+      return enum_for(:each_key) unless block_given?
+
+      values.each { |definition| yield definition.graph_id }
+      self
+    end
+
+    def each_value(&block)
+      return enum_for(:each_value) unless block
+
+      values.each(&block)
+      self
+    end
+
+    def keys
+      values.map(&:graph_id)
+    end
+
+    def values
+      sorted(@by_definition_id.values)
+    end
+
+    def length
+      @by_definition_id.length
+    end
+    alias size length
+
+    def empty?
+      @by_definition_id.empty?
+    end
+
+    private
+
+    def sorted(definitions)
+      definitions.sort_by do |definition|
+        [definition.symbol_id.to_s, definition.file.to_s, definition.line.to_i, definition.graph_id.to_s]
+      end
+    end
+  end
+end

data/lib/necropsy/graph/dynamic_evidence_tracking.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module DynamicEvidenceTracking
+    SAMPLE_LIMIT = 5
+    RESOLUTION_STATUSES = %w[exact unique ambiguous missing].freeze
+
+    def dynamic_observation?
+      observation.key?('dynamic_evidence')
+    end
+
+    def dynamic_evidence_diagnostic
+      observation['dynamic_evidence']
+    end
+
+    private
+
+    def dynamic_result?(result)
+      return true unless result.alive_evidences.empty?
+
+      dynamic_profiles = profiles.select { |profile| profile.kind == :dynamic }.map { |profile| profile.name.to_s }
+      evidence_analyzers = result.edge_evidences.map { |edge| edge.evidence.analyzer.to_s }
+      return true if dynamic_profiles.intersect?(evidence_analyzers)
+
+      observation_keys = result.observation.keys.map(&:to_s)
+      observation_keys.intersect?(dynamic_profiles) || observation_keys.intersect?(%w[coverage coverband trace_point])
+    end
+
+    def record_dynamic_evidence(result, alive_matches, edge_matches)
+      diagnostic = observation['dynamic_evidence'] ||= empty_dynamic_evidence_diagnostic
+      unmatched_nodes = result.alive_evidences.zip(alive_matches).filter_map do |_alive, resolution|
+        dynamic_reference_label(resolution) if missing_dynamic_resolution?(resolution)
+      end
+      incomplete_nodes = alive_matches.filter_map do |resolution|
+        dynamic_reference_label(resolution) unless precise_dynamic_resolution?(resolution)
+      end
+      incomplete_edges = edge_matches.filter_map { |match| incomplete_edge_sample(match) }
+
+      update_dynamic_node_counts(diagnostic, alive_matches)
+      update_dynamic_edge_counts(diagnostic, edge_matches)
+      update_dynamic_resolution_counts(diagnostic, alive_matches, edge_matches)
+      append_dynamic_samples(diagnostic, 'nodes', unmatched_nodes)
+      append_dynamic_samples(diagnostic, 'edges', incomplete_edges)
+      append_resolution_samples(diagnostic, alive_matches, edge_matches)
+      warn_unmatched_dynamic_evidence(alive_matches, incomplete_nodes)
+      warn_unmatched_dynamic_edges(edge_matches, incomplete_edges)
+    end
+
+    def empty_dynamic_evidence_diagnostic
+      {
+        'policy' => 'positive_only',
+        'runtime_unobserved' => 'informational_only',
+        'attempted' => { 'nodes' => 0, 'edges' => 0 },
+        'matched' => { 'nodes' => 0, 'edges' => 0 },
+        'partially_matched' => { 'nodes' => 0, 'edges' => 0 },
+        'unmatched' => { 'nodes' => 0, 'edges' => 0 },
+        'unmatched_samples' => { 'nodes' => [], 'edges' => [] },
+        'resolution' => {
+          'nodes' => empty_resolution_counts,
+          'edge_endpoints' => empty_resolution_counts
+        },
+        'resolution_samples' => { 'nodes' => [], 'edge_endpoints' => [] }
+      }
+    end
+
+    def empty_resolution_counts
+      RESOLUTION_STATUSES.to_h { |status| [status, 0] }
+    end
+
+    def update_dynamic_node_counts(diagnostic, resolutions)
+      diagnostic['attempted']['nodes'] += resolutions.length
+      diagnostic['matched']['nodes'] += resolutions.count { |resolution| precise_dynamic_resolution?(resolution) }
+      diagnostic['partially_matched']['nodes'] += resolutions.count { |resolution| ambiguous_dynamic_resolution?(resolution) }
+      diagnostic['unmatched']['nodes'] += resolutions.count { |resolution| missing_dynamic_resolution?(resolution) }
+    end
+
+    def update_dynamic_edge_counts(diagnostic, matches)
+      diagnostic['attempted']['edges'] += matches.length
+      diagnostic['matched']['edges'] += matches.count { |match| fully_matched_edge?(match) }
+      diagnostic['partially_matched']['edges'] += matches.count { |match| partially_matched_edge?(match) }
+      diagnostic['unmatched']['edges'] += matches.count { |match| unmatched_edge?(match) }
+    end
+
+    def update_dynamic_resolution_counts(diagnostic, nodes, edges)
+      nodes.each { |resolution| increment_resolution_count(diagnostic, 'nodes', resolution) }
+      edges.each do |match|
+        increment_resolution_count(diagnostic, 'edge_endpoints', match.fetch(:caller))
+        increment_resolution_count(diagnostic, 'edge_endpoints', match.fetch(:callee))
+      end
+    end
+
+    def increment_resolution_count(diagnostic, kind, resolution)
+      diagnostic['resolution'][kind][resolution.fetch(:status).to_s] += 1
+    end
+
+    def fully_matched_edge?(match)
+      match.fetch(:materialized)
+    end
+
+    def partially_matched_edge?(match)
+      !fully_matched_edge?(match) && !unmatched_edge?(match)
+    end
+
+    def unmatched_edge?(match)
+      missing_dynamic_resolution?(match.fetch(:caller)) && missing_dynamic_resolution?(match.fetch(:callee))
+    end
+
+    def ambiguous_dynamic_resolution?(resolution)
+      resolution.fetch(:status) == :ambiguous
+    end
+
+    def missing_dynamic_resolution?(resolution)
+      resolution.fetch(:status) == :missing
+    end
+
+    def incomplete_edge_sample(match)
+      return if fully_matched_edge?(match)
+
+      caller = match.fetch(:caller)
+      callee = match.fetch(:callee)
+      incomplete = []
+      missing = dynamic_endpoints_with_status(caller, callee, :missing)
+      ambiguous = dynamic_endpoints_with_status(caller, callee, :ambiguous)
+      incomplete << "unmatched #{missing.join(', ')}" unless missing.empty?
+      incomplete << "ambiguous #{ambiguous.join(', ')}" unless ambiguous.empty?
+      "#{dynamic_reference_label(caller)} -> #{dynamic_reference_label(callee)} (#{incomplete.join(', ')})"
+    end
+
+    def dynamic_endpoints_with_status(caller, callee, status)
+      { 'caller' => caller, 'callee' => callee }.filter_map do |endpoint, resolution|
+        "#{endpoint}: #{dynamic_reference_label(resolution)}" if resolution.fetch(:status) == status
+      end
+    end
+
+    def dynamic_reference_label(resolution)
+      reference = resolution.fetch(:reference)
+      return reference.fetch('identifier') if reference.key?('identifier')
+
+      identifier = reference['definition_id'] || reference['symbol_id'] || '<invalid>'
+      location = [reference['file'], reference['line']].compact.join(':')
+      location.empty? ? identifier : "#{identifier} @ #{location}"
+    end
+
+    def append_dynamic_samples(diagnostic, kind, samples)
+      combined = diagnostic['unmatched_samples'][kind] + samples
+      diagnostic['unmatched_samples'][kind] = combined.uniq.sort.first(SAMPLE_LIMIT)
+    end
+
+    def append_resolution_samples(diagnostic, nodes, edges)
+      node_samples = nodes.map { |resolution| resolution_sample(resolution) }
+      endpoint_samples = edges.flat_map do |match|
+        %i[caller callee].map do |endpoint|
+          resolution_sample(match.fetch(endpoint)).merge('endpoint' => endpoint.to_s)
+        end
+      end
+      append_bounded_resolution_samples(diagnostic, 'nodes', node_samples)
+      append_bounded_resolution_samples(diagnostic, 'edge_endpoints', endpoint_samples)
+    end
+
+    def resolution_sample(resolution)
+      {
+        'reference' => resolution.fetch(:reference),
+        'status' => resolution.fetch(:status).to_s,
+        'definition_ids' => resolution.fetch(:definitions).map(&:graph_id).sort
+      }
+    end
+
+    def append_bounded_resolution_samples(diagnostic, kind, samples)
+      combined = diagnostic['resolution_samples'][kind] + samples
+      diagnostic['resolution_samples'][kind] = combined.uniq.sort_by do |sample|
+        canonical_dynamic_sample(sample)
+      end.first(SAMPLE_LIMIT)
+    end
+
+    def canonical_dynamic_sample(value)
+      case value
+      when Hash
+        value.keys.sort.map { |key| [key, canonical_dynamic_sample(value.fetch(key))] }
+      when Array
+        value.map { |item| canonical_dynamic_sample(item) }
+      else
+        value.to_s
+      end
+    end
+
+    def warn_unmatched_dynamic_evidence(matches, samples)
+      return if samples.empty?
+
+      precise = matches.count { |resolution| precise_dynamic_resolution?(resolution) }
+      ambiguous = matches.count { |resolution| ambiguous_dynamic_resolution?(resolution) }
+      missing = matches.count { |resolution| missing_dynamic_resolution?(resolution) }
+      warn "Necropsy matched #{precise} of #{matches.length} dynamic node IDs precisely; ambiguous #{ambiguous}; " \
+           "missing #{missing}; unresolved: #{samples.first(SAMPLE_LIMIT).join(', ')}"
+    end
+
+    def warn_unmatched_dynamic_edges(matches, samples)
+      return if samples.empty?
+
+      full = matches.count { |match| fully_matched_edge?(match) }
+      partial = matches.count { |match| partially_matched_edge?(match) }
+      unmatched = matches.count { |match| unmatched_edge?(match) }
+      warn "Necropsy fully matched #{full} of #{matches.length} dynamic edges; partially matched #{partial}; " \
+           "unmatched #{unmatched}; unmatched endpoints: #{samples.first(SAMPLE_LIMIT).join(', ')}"
+    end
+  end
+end

data/lib/necropsy/graph/evidence_store.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Necropsy
+  module EvidenceStore
+    PROJECTIONS = %i[exact conservative observed].freeze
+
+    def self.normalize_projection(projection)
+      normalized = projection.to_sym if projection.respond_to?(:to_sym)
+      return normalized if PROJECTIONS.include?(normalized)
+
+      raise ArgumentError, "projection must be one of: #{PROJECTIONS.join(', ')}"
+    end
+
+    def evidence_records
+      @evidence_records_by_id.values.sort_by(&:evidence_id).freeze
+    end
+
+    def evidence_record(evidence_id)
+      @evidence_records_by_id[evidence_id.to_s]
+    end
+
+    def evidence_collisions
+      @evidence_collision_payloads.keys.sort.map do |evidence_id|
+        {
+          'evidence_id' => evidence_id,
+          'payload_sha256s' => @evidence_collision_payloads.fetch(evidence_id).to_a.sort,
+          'domains' => @evidence_domains.fetch(evidence_id, Set.new).map(&:to_s).sort
+        }
+      end.freeze
+    end
+
+    private
+
+    def initialize_evidence_store
+      @evidence_records_by_id = {}
+      @evidence_payloads_by_id = {}
+      @evidence_payloads_by_object = {}.compare_by_identity
+      @evidence_domains = Hash.new { |hash, key| hash[key] = Set.new }
+      @quarantined_evidence_ids = Set.new
+      @evidence_collision_payloads = Hash.new { |hash, key| hash[key] = Set.new }
+    end
+
+    def register_evidence(evidence, domain: :runtime, canonical_payload: nil)
+      record = evidence_with_identity(evidence)
+      evidence_id = record.evidence_id
+      payload = canonical_payload || canonical_evidence_payload(record)
+      @evidence_payloads_by_object[record] = [evidence_id, payload]
+      @evidence_domains[evidence_id] << domain.to_sym
+
+      if @quarantined_evidence_ids.include?(evidence_id)
+        @evidence_collision_payloads[evidence_id] << Digest::SHA256.hexdigest(payload)
+        rebuild_evidence_collision_state
+        return
+      end
+
+      existing_payload = @evidence_payloads_by_id[evidence_id]
+      if existing_payload && existing_payload != payload
+        @evidence_collision_payloads[evidence_id].merge(
+          [Digest::SHA256.hexdigest(existing_payload), Digest::SHA256.hexdigest(payload)]
+        )
+        quarantine_evidence(evidence_id)
+        return
+      end
+
+      @evidence_records_by_id[evidence_id] ||= record
+      @evidence_payloads_by_id[evidence_id] ||= payload
+      refresh_evidence_store_observation if observation.key?('evidence_store')
+      evidence_id
+    end
+
+    def evidence_with_identity(evidence)
+      return evidence if evidence.evidence_id
+
+      evidence.with(evidence_id: EvidenceIdentity.generate(evidence.to_h.except('evidence_id')))
+    end
+
+    def canonical_evidence_payload(evidence)
+      BoundedCanonicalizer.dump(evidence.to_h.except('evidence_id'))
+    end
+
+    def quarantine_evidence(evidence_id)
+      @quarantined_evidence_ids << evidence_id
+      @evidence_records_by_id.delete(evidence_id)
+      @evidence_payloads_by_id.delete(evidence_id)
+      remove_evidence_references(evidence_id)
+      rebuild_evidence_collision_state
+    end
+
+    def remove_evidence_references(evidence_id)
+      @edges.each_value do |callees|
+        callees.each_value { |evidence_ids| evidence_ids.delete(evidence_id) }
+        callees.delete_if { |_callee_id, evidence_ids| evidence_ids.empty? }
+      end
+      @edges.delete_if { |_caller_id, callees| callees.empty? }
+      @dynamic_alive.each_value { |evidence_ids| evidence_ids.delete(evidence_id) }
+      @dynamic_alive.delete_if { |_node_id, evidence_ids| evidence_ids.empty? }
+      rebuild_incoming_edges
+    end
+
+    def rebuild_evidence_collision_state
+      remove_blockers_matching do |blocker|
+        blocker.kind == :evidence_collision && blocker.source == :evidence_store
+      end
+      @evidence_collision_payloads.keys.sort.each do |evidence_id|
+        @evidence_domains.fetch(evidence_id).to_a.sort.each do |domain|
+          add_blocker(evidence_collision_blocker(evidence_id, domain))
+        end
+      end
+      refresh_evidence_store_observation
+    end
+
+    def refresh_evidence_store_observation
+      observation['evidence_store'] = {
+        'record_count' => @evidence_records_by_id.length,
+        'collision_count' => @evidence_collision_payloads.length,
+        'collisions' => evidence_collisions.first(5)
+      }
+    end
+
+    def evidence_collision_blocker(evidence_id, domain)
+      Blocker.new(
+        kind: :evidence_collision,
+        scope_kind: :global,
+        scope_value: '*',
+        source: :evidence_store,
+        reason: 'One evidence ID was supplied with conflicting payloads',
+        suggested_action: :fix_analyzer,
+        metadata: {
+          'caller_domain' => domain.to_s,
+          'receiver_kind' => 'implicit',
+          'evidence_id' => evidence_id,
+          'evidence_store' => true
+        }
+      )
+    end
+
+    def projected_evidence_records(evidence_ids, projection:, scope: nil)
+      projection = normalize_projection(projection)
+      evidence_ids.filter_map { |evidence_id| evidence_record(evidence_id) }
+                  .select { |evidence| evidence_in_projection?(evidence, projection, scope) }
+                  .sort_by(&:evidence_id)
+    end
+
+    def projected_evidence_ids(evidence_ids, projection:, scope: nil)
+      projected_evidence_records(evidence_ids, projection: projection, scope: scope).map(&:evidence_id).freeze
+    end
+
+    def normalize_projection(projection)
+      EvidenceStore.normalize_projection(projection)
+    end
+
+    def evidence_payload_registered?(evidence, canonical_payload: nil)
+      evidence_id = evidence.evidence_id
+      cached = @evidence_payloads_by_object[evidence]
+      return true if cached && cached[0] == evidence_id && @evidence_payloads_by_id[evidence_id] == cached[1]
+
+      payload = canonical_payload || canonical_evidence_payload(evidence)
+      return true if @evidence_payloads_by_id[evidence_id] == payload
+      return false unless @quarantined_evidence_ids.include?(evidence_id)
+
+      @evidence_collision_payloads.fetch(evidence_id).include?(Digest::SHA256.hexdigest(payload))
+    end
+
+    def evidence_in_projection?(evidence, projection, requested_scope)
+      grade = evidence.grade&.to_sym
+      case projection
+      when :conservative
+        true
+      when :exact
+        grade == :exact || (grade == :observed && verified_observed_scope?(evidence.scope, requested_scope))
+      when :observed
+        grade == :observed && evidence_scope_matches?(evidence.scope, requested_scope)
+      end
+    end
+
+    def verified_observed_scope?(actual_scope, requested_scope)
+      return false unless requested_scope.is_a?(Hash)
+
+      actual_status = actual_scope.is_a?(Hash) && actual_scope['source_revision_status']
+      return false if %w[mismatch stale].include?(actual_status.to_s)
+
+      revision = requested_scope['revision'] || requested_scope[:revision]
+      revision && evidence_scope_matches?(actual_scope, requested_scope)
+    end
+
+    def evidence_scope_matches?(actual_scope, requested_scope)
+      return true if requested_scope.nil?
+
+      actual_scope = actual_scope.to_h if actual_scope.respond_to?(:to_h)
+      requested_scope = requested_scope.to_h if requested_scope.respond_to?(:to_h)
+      return canonical_scope(actual_scope) == canonical_scope(requested_scope) unless requested_scope.is_a?(Hash)
+      return false unless actual_scope.is_a?(Hash)
+
+      requested_scope.all? do |key, expected|
+        actual_key = [key, key.to_s, key.respond_to?(:to_sym) && key.to_sym].find do |candidate|
+          candidate && actual_scope.key?(candidate)
+        end
+        next false unless actual_key
+
+        actual = actual_scope.fetch(actual_key)
+        evidence_scope_matches?(actual, expected)
+      end
+    rescue KeyError, NoMethodError, BoundedCanonicalizer::Error
+      false
+    end
+
+    def canonical_scope(value)
+      BoundedCanonicalizer.dump(value)
+    end
+  end
+end