necropsy 0.1.0 → 0.3.0

data/lib/necropsy/bounded_canonicalizer.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Necropsy
+  class BoundedCanonicalizer
+    MAX_DEPTH = 256
+    MAX_ITEMS = 1_000_000
+    MAX_STRING_BYTES = 64 * 1024 * 1024
+    MAX_TOTAL_BYTES = 256 * 1024 * 1024
+
+    class Error < StandardError; end
+    class CycleError < Error; end
+    class LimitError < Error; end
+    class UnsupportedTypeError < Error; end
+
+    def self.dump(value, **limits)
+      new(**limits).dump(value)
+    end
+
+    def initialize(max_depth: MAX_DEPTH, max_items: MAX_ITEMS, max_string_bytes: MAX_STRING_BYTES,
+                   max_total_bytes: MAX_TOTAL_BYTES)
+      @max_depth = positive_limit(max_depth, :max_depth)
+      @max_items = positive_limit(max_items, :max_items)
+      @max_string_bytes = positive_limit(max_string_bytes, :max_string_bytes)
+      @max_total_bytes = positive_limit(max_total_bytes, :max_total_bytes)
+    end
+
+    # Kept for callers that need the canonical Ruby representation rather than bytes.
+    def call(value)
+      JSON.parse(dump(value), max_nesting: false)
+    end
+
+    def dump(value)
+      reset_state
+      output = binary_string
+      process([[:value, value, 0, output]])
+      output
+    rescue JSON::GeneratorError => e
+      raise Error, "Could not encode canonical payload: #{e.message}"
+    end
+
+    private
+
+    attr_reader :max_depth, :max_items, :max_string_bytes, :max_total_bytes, :active_containers
+
+    def reset_state
+      @active_containers = {}
+      @items = 0
+      @written_bytes = 0
+    end
+
+    def process(stack)
+      until stack.empty?
+        action, *arguments = stack.pop
+        send("process_#{action}", stack, *arguments)
+      end
+    rescue SystemStackError => e
+      raise CycleError, "Could not canonicalize payload: #{e.message}"
+    end
+
+    def process_value(stack, value, depth, output)
+      count_item!
+      check_depth!(depth)
+
+      case value
+      when Hash then start_hash(stack, value, depth, output)
+      when Array then start_array(stack, value, depth, output)
+      when Symbol then append(output, %(["symbol",#{encoded_string(canonical_string(value.to_s))}]))
+      when String then append(output, %(["string",#{encoded_string(canonical_string(value))}]))
+      when Integer then append(output, %(["integer",#{json_string(value.to_s)}]))
+      when Float then append(output, %(["float",#{json_string(canonical_float(value))}]))
+      when true, false then append(output, %(["boolean",#{value}]))
+      when nil then append(output, '["nil"]')
+      else start_object(stack, value, depth, output)
+      end
+    end
+
+    def start_array(stack, value, depth, output)
+      enter_container!(value)
+      append(output, '["array",[')
+      stack << [:array_next, value, 0, depth, output]
+    end
+
+    def process_array_next(stack, value, index, depth, output)
+      if index >= value.length
+        append(output, ']]')
+        leave_container(value)
+        return
+      end
+
+      append(output, ',') if index.positive?
+      stack << [:array_next, value, index + 1, depth, output]
+      stack << [:value, value.fetch(index), depth + 1, output]
+    end
+
+    def start_hash(stack, value, depth, output)
+      enter_container!(value)
+      stack << [:hash_next, value, value.each_pair, [], depth, output]
+    end
+
+    def process_hash_next(stack, value, iterator, pairs, depth, output)
+      key, item = iterator.next
+      pair = binary_string
+      append(pair, '[')
+      stack << [:hash_pair_complete, value, iterator, pairs, depth, output, pair]
+      stack << [:raw, pair, ']']
+      stack << [:value, item, depth + 1, pair]
+      stack << [:raw, pair, ',']
+      stack << [:value, key, depth + 1, pair]
+    rescue StopIteration
+      append(output, '["hash",[')
+      pairs.sort.each_with_index do |pair, index|
+        append(output, ',') if index.positive?
+        append(output, pair)
+      end
+      append(output, ']]')
+      leave_container(value)
+    end
+
+    def process_hash_pair_complete(stack, value, iterator, pairs, depth, output, pair)
+      pairs << pair
+      stack << [:hash_next, value, iterator, pairs, depth, output]
+    end
+
+    def start_object(stack, value, depth, output)
+      raise UnsupportedTypeError, "Unsupported canonical payload type: #{value.class}" unless value.respond_to?(:to_h)
+
+      enter_container!(value)
+      converted = value.to_h
+      append(output, %(["object",#{json_string(value.class.name.to_s)},))
+      stack << [:object_complete, value, output]
+      stack << [:value, converted, depth + 1, output]
+    rescue Error
+      raise
+    rescue StandardError => e
+      leave_container(value)
+      raise UnsupportedTypeError, "Could not convert #{value.class} to a canonical payload: #{e.class}"
+    end
+
+    def process_object_complete(_stack, value, output)
+      append(output, ']')
+      leave_container(value)
+    end
+
+    def process_raw(_stack, output, bytes)
+      append(output, bytes)
+    end
+
+    def encoded_string(value)
+      %([#{json_string(value.fetch(0))},#{json_string(value.fetch(1))}])
+    end
+
+    def canonical_string(value)
+      check_scalar_size!(value)
+      normalized = value.encode(Encoding::UTF_8)
+      ['UTF-8', normalized.b.unpack1('H*')]
+    rescue EncodingError
+      [value.encoding.name, value.b.unpack1('H*')]
+    end
+
+    def canonical_float(value)
+      raise UnsupportedTypeError, 'Canonical payload contains a non-finite float' unless value.finite?
+
+      value.to_s
+    end
+
+    def json_string(value)
+      JSON.generate(value)
+    end
+
+    def append(output, bytes)
+      @written_bytes += bytes.bytesize
+      raise LimitError, "Canonical payload exceeds maximum size #{max_total_bytes}" if @written_bytes > max_total_bytes
+
+      output << bytes.b
+    end
+
+    def enter_container!(value)
+      object_id = value.object_id
+      raise CycleError, "Canonical payload contains a cycle at #{value.class}" if active_containers.key?(object_id)
+
+      active_containers[object_id] = true
+    end
+
+    def leave_container(value)
+      active_containers.delete(value.object_id)
+    end
+
+    def check_scalar_size!(value)
+      return if value.bytesize <= max_string_bytes
+
+      raise LimitError, "Canonical string exceeds maximum size #{max_string_bytes}"
+    end
+
+    def check_depth!(depth)
+      return if depth <= max_depth
+
+      raise LimitError, "Canonical payload exceeds maximum depth #{max_depth}"
+    end
+
+    def count_item!
+      @items += 1
+      raise LimitError, "Canonical payload exceeds maximum item count #{max_items}" if @items > max_items
+    end
+
+    def positive_limit(value, name)
+      value = Integer(value)
+      raise ArgumentError, "#{name} must be positive" unless value.positive?
+
+      value
+    end
+
+    def binary_string
+      String.new(encoding: Encoding::BINARY)
+    end
+  end
+end

data/lib/necropsy/cache/scan_cache.rb

@@ -1,12 +1,13 @@
 # frozen_string_literal: true
 
 require 'fileutils'
-require 'yaml'
+require 'json'
+require 'digest'
 
 module Necropsy
   module Cache
     class ScanCache
-      VERSION = 3
+      VERSION = 13
 
       def initialize(project:)
         @project = project
@@ -15,21 +16,28 @@ module Necropsy
       def fetch(files)
         return yield unless project.config.cache_enabled?
 
-        metadata = cache_metadata(files)
+        metadata = safe_cache_metadata(files)
+        return yield unless metadata
+
         cached = read(metadata)
         return cached if cached
 
         result = yield
         write(metadata, result)
         result
-      rescue SystemCallError, Psych::Exception
-        yield
       end
 
       private
 
       attr_reader :project
 
+      def safe_cache_metadata(files)
+        cache_metadata(files)
+      rescue SystemCallError, JSON::ParserError => e
+        warn_cache("Cache unavailable: #{e.message}")
+        nil
+      end
+
       def read(metadata)
         payload = load_payload
         return nil unless payload
@@ -37,24 +45,31 @@ module Necropsy
         return nil unless payload['metadata'] == metadata
 
         deserialize_scan_result(payload.fetch('scan_result'))
-      rescue StandardError
+      rescue StandardError => e
+        warn_cache("Ignoring invalid cache: #{e.message}")
         nil
       end
 
       def load_payload
         return nil unless File.exist?(path)
 
-        YAML.load_file(path)
+        JSON.parse(File.read(path))
       end
 
       def write(metadata, result)
         FileUtils.mkdir_p(File.dirname(path))
-        File.write(path, {
+        payload = JSON.generate(
           'version' => VERSION,
           'metadata' => metadata,
           'scan_result' => serialize_scan_result(result)
-        }.to_yaml)
-      rescue StandardError
+        )
+        temporary = "#{path}.tmp-#{Process.pid}-#{Thread.current.object_id}"
+        File.write(temporary, payload)
+        File.rename(temporary, path)
+      rescue StandardError => e
+        warn_cache("Could not write cache: #{e.message}")
+      ensure
+        FileUtils.rm_f(temporary) if temporary && File.exist?(temporary)
         nil
       end
 
@@ -67,7 +82,8 @@ module Necropsy
           stat = File.stat(file)
           metadata[project.relative_path(file)] = {
             'size' => stat.size,
-            'mtime' => "#{stat.mtime.to_i}.#{stat.mtime.nsec}"
+            'mtime' => "#{stat.mtime.to_i}.#{stat.mtime.nsec}",
+            'content_sha256' => Digest::SHA256.file(file).hexdigest
           }
         end
       end
@@ -75,7 +91,16 @@ module Necropsy
       def cache_metadata(files)
         {
           'files' => file_metadata(files),
-          'configuration' => project.config.scan_cache_key
+          'inventory' => project.scan_inventory_key,
+          'configuration' => project.config.scan_cache_key,
+          'environment' => {
+            'necropsy_version' => Necropsy::VERSION,
+            'ruby_engine' => RUBY_ENGINE,
+            'ruby_version' => RUBY_VERSION,
+            'prism_version' => Prism::VERSION,
+            'definition_identity_version' => DefinitionIdentity::VERSION,
+            'call_site_identity_version' => CallSiteIdentity::VERSION
+          }
         }
       end
 
@@ -86,7 +111,13 @@ module Necropsy
           'instantiated_classes' => result.instantiated_classes.to_a.sort,
           'uncertainties' => result.uncertainties.transform_values { |messages| Array(messages).map(&:to_s) },
           'class_infos' => result.class_infos.map(&:to_h),
-          'entrypoint_hints' => result.entrypoint_hints.map(&:to_h)
+          'entrypoint_hints' => result.entrypoint_hints.map(&:to_h),
+          'file_statuses' => result.file_statuses.transform_values(&:to_s),
+          'source_errors' => result.source_errors.map(&:to_h),
+          'source_domains' => result.source_domains.transform_values(&:to_s),
+          'scope_diagnostics' => result.scope_diagnostics,
+          'method_signatures' => result.method_signatures,
+          'semantic_blockers' => result.semantic_blockers.map(&:to_h)
         }
       end
 
@@ -97,13 +128,45 @@ module Necropsy
           instantiated_classes: Set.new(Array(data['instantiated_classes'])),
           uncertainties: deserialize_uncertainties(data['uncertainties']),
           class_infos: Array(data['class_infos']).map { |info| deserialize_class_info(info) },
-          entrypoint_hints: Array(data['entrypoint_hints']).map { |entry| deserialize_entry_point(entry) }
+          entrypoint_hints: Array(data['entrypoint_hints']).map { |entry| deserialize_entry_point(entry) },
+          file_statuses: (data['file_statuses'] || {}).transform_values(&:to_sym),
+          source_errors: Array(data['source_errors']).map { |error| deserialize_source_error(error) },
+          source_domains: (data['source_domains'] || {}).transform_values(&:to_sym),
+          scope_diagnostics: data['scope_diagnostics'] || {},
+          method_signatures: data['method_signatures'] || {},
+          semantic_blockers: Array(data['semantic_blockers']).map { |blocker| deserialize_blocker(blocker) }
+        )
+      end
+
+      def deserialize_blocker(data)
+        Blocker.new(
+          kind: data.fetch('kind').to_sym,
+          scope_kind: data.fetch('scope_kind').to_sym,
+          scope_value: data['scope_value'],
+          source: data['source'],
+          reason: data.fetch('reason'),
+          suggested_action: data.fetch('suggested_action', 'review').to_sym,
+          metadata: data['metadata'] || {}
+        )
+      end
+
+      def deserialize_source_error(data)
+        SourceError.new(
+          file: data['file'],
+          line: data['line']&.to_i || 1,
+          message: data['message'],
+          type: data['type'].to_sym
         )
       end
 
       def deserialize_node(data)
+        id = data.fetch('id')
         Node.new(
-          id: data['id'],
+          id: id,
+          symbol_id: data['symbol_id'] || id,
+          definition_id: data['definition_id'] || id,
+          body_digest: data['body_digest'],
+          ordinal: data['ordinal'].to_i,
           kind: data['kind'].to_sym,
           file: data['file'],
           line: data['line'].to_i,
@@ -111,13 +174,15 @@ module Necropsy
           defined_via: data['defined_via'].to_sym,
           owner: data['owner'],
           name: data['name'],
-          test: data['test']
+          test: data['test'],
+          visibility: (data['visibility'] || 'public').to_sym
         )
       end
 
       def deserialize_call_site(data)
         CallSite.new(
-          caller_id: data['caller_id'],
+          call_site_id: data.fetch('call_site_id'),
+          caller_id: data['caller_definition_id'] || data.fetch('caller_id'),
           message: data['message'],
           receiver_kind: data['receiver_kind'].to_sym,
           receiver_name: data['receiver_name'],
@@ -140,12 +205,19 @@ module Necropsy
           includes: Array(data['includes']),
           prepends: Array(data['prepends']),
           extends: Array(data['extends']),
+          singleton_includes: Array(data['singleton_includes']),
+          singleton_prepends: Array(data['singleton_prepends']),
           dynamic: data['dynamic']
         )
       end
 
       def deserialize_entry_point(data)
-        EntryPoint.new(node_id: data['node_id'], reason: data['reason'].to_sym)
+        Root.new(
+          definition_id: data['definition_id'] || data['node_id'],
+          domain: data['domain'],
+          reason: data['reason'].to_sym,
+          evidence: data['evidence']
+        )
       end
 
       def deserialize_uncertainties(data)
@@ -153,6 +225,10 @@ module Necropsy
           (data || {}).each { |node_id, messages| uncertainties[node_id] = Array(messages) }
         end
       end
+
+      def warn_cache(message)
+        warn "Necropsy: #{message}" if project.config.verbose?
+      end
     end
   end
 end

data/lib/necropsy/call_site_identity.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'digest'
+require_relative 'bounded_canonicalizer'
+
+module Necropsy
+  module CallSiteIdentity
+    VERSION = 1
+    PREFIX = "call:v#{VERSION}".freeze
+    SOURCE_ROLES = %i[
+      call initialize symbol_reference symbol_to_proc super alias_target delegate_target delegate_message
+    ].freeze
+    DERIVATIONS = %i[module_function rta_implicit].freeze
+
+    module_function
+
+    def source_id(caller_definition_id:, relative_path:, role:, message:, structural_digest:, ordinal:)
+      role = role.to_sym
+      raise ArgumentError, "invalid call site role: #{role.inspect}" unless SOURCE_ROLES.include?(role)
+
+      ordinal = Integer(ordinal)
+      raise ArgumentError, 'call site ordinal must be positive' unless ordinal.positive?
+
+      identifier(
+        'source', caller_definition_id.to_s, relative_path.to_s, role.to_s, message.to_s,
+        structural_digest.to_s, ordinal
+      )
+    end
+
+    def derived_id(parent_call_site_id:, derivation:, caller_definition_id:, message:, discriminator: nil)
+      derivation = derivation.to_sym
+      raise ArgumentError, "invalid call site derivation: #{derivation.inspect}" unless DERIVATIONS.include?(derivation)
+
+      payload = [
+        'derived', derivation.to_s, parent_call_site_id.to_s, caller_definition_id.to_s, message.to_s
+      ]
+      payload << discriminator.to_s unless discriminator.nil?
+      identifier(*payload)
+    end
+
+    def legacy_id(caller_definition_id:, message:, receiver_kind:, receiver_name:, file:, line:, test:, dynamic:,
+                  metadata:)
+      identifier(
+        'legacy', caller_definition_id.to_s, message.to_s, receiver_kind.to_s, receiver_name&.to_s,
+        file.to_s, line, test, dynamic, metadata
+      )
+    end
+
+    def identifier(*payload)
+      "#{PREFIX}:#{Digest::SHA256.hexdigest(BoundedCanonicalizer.dump(payload))}"
+    end
+    private_class_method :identifier
+  end
+end