necropsy 0.1.0 → 0.3.0

data/lib/necropsy/analyzers/dynamic/coverband_payload_set.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      class CoverbandPayloadSet
+        PAYLOAD_KEYS = %w[files coverage runtime production nodes executed observation].freeze
+
+        def initialize(limits:, deadline:)
+          @limits = limits
+          @deadline = deadline
+        end
+
+        def merge(payloads)
+          payloads.compact.reduce({}) do |merged, payload|
+            validate_payload_hash!(payload)
+            deadline.check!
+            limits.validate_payload!(merge_payload(merged, payload))
+          end
+        end
+
+        def from_hash_entries(entries, &)
+          domain_error('Malformed Redis hash response') unless entries.is_a?(Array) && entries.length.even?
+
+          hash = entries.each_slice(2).to_h
+          return coverband_hash_payload(hash) if coverband_hash_payload?(hash)
+
+          generic_hash_payload(hash, &)
+        end
+
+        def payload_shape?(value)
+          value.is_a?(Hash) && value.keys.any? { |key| PAYLOAD_KEYS.include?(key.to_s) }
+        end
+
+        private
+
+        attr_reader :limits, :deadline
+
+        def generic_hash_payload(hash)
+          payloads = []
+          files = {}
+          hash.each do |field, value|
+            deadline.check!
+            parsed = yield(value)
+            if payload_shape?(parsed)
+              payloads << parsed
+            elsif parsed.is_a?(Hash) || parsed.is_a?(Array)
+              files[field.to_s] = parsed
+            else
+              domain_error('Invalid Redis coverage payload value')
+            end
+          end
+
+          payloads << { 'files' => files } unless files.empty?
+          merge(payloads)
+        end
+
+        def coverband_hash_payload?(hash)
+          hash.key?('file') && hash.key?('file_length')
+        end
+
+        def coverband_hash_payload(hash)
+          payload = {
+            'files' => { hash.fetch('file') => coverband_line_counts(hash) },
+            'observation' => coverband_observation(hash)
+          }
+          limits.validate_payload!(payload)
+        end
+
+        def coverband_line_counts(hash)
+          length = Integer(hash.fetch('file_length'), exception: false)
+          domain_error('Invalid Coverband file length') unless length && length >= 0
+          domain_error('Coverband file length exceeds configured limit') if length > limits.max_array_elements
+
+          Array.new(length) do |index|
+            deadline.check!
+            count = hash[index.to_s]
+            count&.to_i
+          end
+        end
+
+        def coverband_observation(hash)
+          %w[first_updated_at last_updated_at file_hash].each_with_object({}) do |key, observation|
+            observation[key] = hash[key] if hash[key]
+          end
+        end
+
+        def validate_payload_hash!(payload)
+          domain_error('Redis coverage payload must be a mapping') unless payload.is_a?(Hash)
+
+          limits.validate_payload!(payload)
+        end
+
+        def merge_payload(merged, payload)
+          merged.merge(payload) do |key, left, right|
+            case key.to_s
+            when 'files', 'coverage', 'runtime', 'production' then merge_files(left, right)
+            when 'nodes', 'executed' then (Array(left) + Array(right)).uniq
+            when 'edges' then Array(left) + Array(right)
+            when 'observation' then ObservationPolicy.compatible_merge(left, right)
+            else right
+            end
+          end
+        end
+
+        def merge_files(left, right)
+          return right unless left.is_a?(Hash)
+          return left unless right.is_a?(Hash)
+
+          left.merge(right) do |_file, left_lines, right_lines|
+            merge_file_coverage(left_lines, right_lines)
+          end
+        end
+
+        def merge_file_coverage(left, right)
+          return merge_count_hashes(left, right) if left.is_a?(Hash) && right.is_a?(Hash)
+          return merge_count_arrays(left, right) if count_array?(left) || count_array?(right)
+
+          (Array(left) + Array(right)).uniq
+        end
+
+        def merge_count_hashes(left, right)
+          left.merge(right) { |_line, left_count, right_count| left_count.to_i + right_count.to_i }
+        end
+
+        def merge_count_arrays(left, right)
+          length = [Array(left).length, Array(right).length].max
+          domain_error('Merged Redis coverage exceeds configured collection size') if length > limits.max_array_elements
+
+          Array.new(length) do |index|
+            left_value = Array(left)[index]
+            right_value = Array(right)[index]
+            next if left_value.nil? && right_value.nil?
+
+            left_value.to_i + right_value.to_i
+          end
+        end
+
+        def count_array?(value)
+          value.is_a?(Array) && value.any? { |item| item.nil? || item.to_i.zero? }
+        end
+
+        def domain_error(message)
+          raise Error, message
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/analyzers/dynamic/observation_policy.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      module ObservationPolicy
+        module_function
+
+        def metadata(payload = nil, expected_revision: nil, **keyword_payload)
+          payload ||= keyword_payload
+          observation = payload['observation']
+          observation = {} unless observation.is_a?(Hash)
+          schema_version = Integer(payload.fetch('schema_version', 1), exception: false)
+          raise Error, 'Dynamic observation schema version must be 1 or 2' unless [1, 2].include?(schema_version)
+
+          revision = source_revision(payload, observation)
+
+          status = source_revision_status(payload, observation, revision, expected_revision)
+          normalized = observation.merge(
+            'schema_version' => schema_version,
+            'positive_evidence_policy' => 'alive_only',
+            'source_revision_status' => status,
+            'source_revision_policy' => 'accepted_for_liveness_only'
+          )
+          normalized = normalized.merge('source_revision' => revision) if revision
+          normalized = normalized.merge('expected_source_revision' => expected_revision) if expected_revision
+          return normalized unless schema_version == 2
+
+          collector = payload['collector']
+          collector = {} unless collector.is_a?(Hash)
+          source = payload['source']
+          source = {} unless source.is_a?(Hash)
+          scope = payload['scope']
+          scope = {} unless scope.is_a?(Hash)
+          quality = payload['quality']
+          quality = {} unless quality.is_a?(Hash)
+          normalized.merge(
+            'collector_name' => collector['name'] || payload['collector_name'],
+            'collector_version' => collector['version'] || payload['collector_version'],
+            'source_metadata' => source,
+            'environment' => scope['environment'] || observation['environment'],
+            'sample_unit' => scope['sample_unit'] || observation['sample_unit'],
+            'sample_rate' => scope['sample_rate'] || observation['sample_rate'],
+            'quality' => {
+              'dropped_events' => quality.fetch('dropped_events', 0),
+              'overflowed' => quality.fetch('overflowed', false)
+            }
+          ).compact
+        end
+
+        def compatible_merge(left, right)
+          return left.merge(right) unless left.is_a?(Hash) && right.is_a?(Hash)
+
+          %w[source_revision source_revision_status environment collector_name collector_version].each do |key|
+            next if left[key].nil? || right[key].nil? || left[key] == right[key]
+
+            raise Error, "Dynamic observation #{key} is incompatible"
+          end
+          left.merge(right)
+        end
+
+        def evidence_scope(observation)
+          declared_scope = observation['scope']
+          declared_scope = {} unless declared_scope.is_a?(Hash)
+          declared_scope = declared_scope.reject do |key, _value|
+            %w[revision source_revision].include?(key.to_s)
+          end
+          selectors = {
+            'revision' => observation['source_revision'],
+            'source_revision_status' => observation['source_revision_status'],
+            'environment' => observation['environment'],
+            'collector' => observation['collector']
+          }.compact
+          declared_scope.merge(selectors)
+        end
+
+        def source_revision(payload, observation)
+          source = payload['source']
+          source_revision = source['git_sha'] if source.is_a?(Hash)
+          source_revision || payload['source_revision'] || observation['source_revision']
+        end
+        private_class_method :source_revision
+
+        def source_revision_status(payload, observation, revision, expected_revision)
+          stale = payload['stale'] || observation['stale'] || payload['source_revision_status'] == 'stale'
+          return 'stale' if stale
+          return 'unknown' unless revision
+          return 'provided_unverified' unless expected_revision
+
+          revision.to_s == expected_revision.to_s ? 'match' : 'mismatch'
+        end
+        private_class_method :source_revision_status
+      end
+    end
+  end
+end

data/lib/necropsy/analyzers/dynamic/redis_input_limits.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      class RedisInputLimits
+        DEFAULTS = {
+          'connect_timeout' => 5.0,
+          'read_timeout' => 5.0,
+          'total_timeout' => 15.0,
+          'max_response_bytes' => 16 * 1024 * 1024,
+          'max_bulk_bytes' => 8 * 1024 * 1024,
+          'max_array_elements' => 100_000,
+          'max_resp_depth' => 16,
+          'max_keys' => 1_000,
+          'max_payload_depth' => 64
+        }.freeze
+        TIMEOUT_KEYS = %w[connect_timeout read_timeout total_timeout].freeze
+        INTEGER_KEYS = (DEFAULTS.keys - TIMEOUT_KEYS).freeze
+
+        attr_reader(*DEFAULTS.keys.map(&:to_sym))
+
+        def initialize(config)
+          TIMEOUT_KEYS.each { |key| instance_variable_set("@#{key}", positive_float(config, key)) }
+          INTEGER_KEYS.each { |key| instance_variable_set("@#{key}", positive_integer(config, key)) }
+        end
+
+        def validate_payload!(payload)
+          stack = [[payload, 0]]
+          until stack.empty?
+            value, depth = stack.pop
+            raise Error, 'Redis coverage payload exceeds maximum nesting depth' if depth > max_payload_depth
+            if (value.is_a?(Hash) || value.is_a?(Array)) && value.length > max_array_elements
+              raise Error, 'Redis coverage payload exceeds maximum collection size'
+            end
+
+            children = value.is_a?(Hash) ? value.flat_map { |key, child| [key, child] } : Array(value)
+            stack.concat(children.map { |child| [child, depth + 1] }) if value.is_a?(Hash) || value.is_a?(Array)
+          end
+          payload
+        end
+
+        private
+
+        def positive_float(config, key)
+          value = Float(config.fetch(key, DEFAULTS.fetch(key)))
+          return value if value.positive? && value.finite?
+
+          invalid_limit!(key)
+        rescue ArgumentError, TypeError
+          invalid_limit!(key)
+        end
+
+        def positive_integer(config, key)
+          value = Integer(config.fetch(key, DEFAULTS.fetch(key)))
+          return value if value.positive?
+
+          invalid_limit!(key)
+        rescue ArgumentError, TypeError
+          invalid_limit!(key)
+        end
+
+        def invalid_limit!(key)
+          raise Error, "analyzers.dynamic.coverband.#{key} must be a finite positive number"
+        end
+      end
+
+      class RedisDeadline
+        def initialize(seconds)
+          @expires_at = monotonic_time + seconds
+        end
+
+        def remaining
+          seconds = @expires_at - monotonic_time
+          raise Error, 'Redis total timeout exceeded' unless seconds.positive?
+
+          seconds
+        end
+
+        def check!
+          remaining
+          nil
+        end
+
+        private
+
+        def monotonic_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+
+      class RedisCredentialRedactor
+        REDACTED = '[REDACTED]'
+
+        def initialize(source)
+          @source = source.to_s
+          @safe_source = @source.sub(%r{(?<=://)[^/@\s]+@}, "#{REDACTED}@")
+          @secrets = credentials(@source)
+        end
+
+        def redact(message)
+          sanitized = message.to_s.gsub(@source, @safe_source)
+          @secrets.reduce(sanitized) { |text, secret| text.gsub(secret, REDACTED) }
+        end
+
+        private
+
+        def credentials(source)
+          uri = URI(source)
+          [uri.user, uri.password].compact.flat_map do |credential|
+            [credential, URI.decode_www_form_component(credential)]
+          end.reject(&:empty?).uniq.sort_by { |value| -value.length }
+        rescue URI::Error, ArgumentError
+          []
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/analyzers/dynamic/redis_nonblocking_io.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'io/wait'
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      module RedisNonblockingIO
+        private
+
+        def finish_tls_connect(ssl_socket)
+          expires_at = monotonic_time + limits.connect_timeout
+          loop do
+            status = ssl_socket.connect_nonblock(exception: false)
+            return unless wait_status?(status)
+
+            wait_for_io(status, io: ssl_socket.to_io, expires_at: expires_at,
+                                label: 'Redis TLS handshake timeout exceeded')
+          rescue IO::WaitReadable
+            wait_for_io(:wait_readable, io: ssl_socket.to_io, expires_at: expires_at,
+                                        label: 'Redis TLS handshake timeout exceeded')
+          rescue IO::WaitWritable
+            wait_for_io(:wait_writable, io: ssl_socket.to_io, expires_at: expires_at,
+                                        label: 'Redis TLS handshake timeout exceeded')
+          end
+        end
+
+        def write_all(value)
+          offset = 0
+          expires_at = monotonic_time + limits.read_timeout
+          while offset < value.bytesize
+            check_io_deadline!(expires_at, 'Redis write timeout exceeded')
+            begin
+              written = socket.write_nonblock(value.byteslice(offset, value.bytesize - offset), exception: false)
+              if wait_status?(written)
+                wait_for_io(written, io: socket.to_io, expires_at: expires_at, label: 'Redis write timeout exceeded')
+              elsif written&.positive?
+                offset += written
+              else
+                domain_error('Redis closed the connection while writing a request')
+              end
+            rescue IO::WaitReadable
+              wait_for_io(:wait_readable, io: socket.to_io, expires_at: expires_at,
+                                          label: 'Redis write timeout exceeded')
+            rescue IO::WaitWritable
+              wait_for_io(:wait_writable, io: socket.to_io, expires_at: expires_at,
+                                          label: 'Redis write timeout exceeded')
+            end
+          end
+          check_io_deadline!(expires_at, 'Redis write timeout exceeded')
+        end
+
+        def read_nonblock(length)
+          expires_at = monotonic_time + limits.read_timeout
+          loop do
+            value = socket.read_nonblock(length, exception: false)
+            return value if value.is_a?(String) && !value.empty?
+
+            domain_error('Redis closed the connection while reading a response') if value.nil? || value == ''
+            wait_for_io(value, io: socket.to_io, expires_at: expires_at, label: 'Redis read timeout exceeded')
+          rescue IO::WaitReadable
+            wait_for_io(:wait_readable, io: socket.to_io, expires_at: expires_at, label: 'Redis read timeout exceeded')
+          rescue IO::WaitWritable
+            wait_for_io(:wait_writable, io: socket.to_io, expires_at: expires_at, label: 'Redis read timeout exceeded')
+          end
+        end
+
+        def wait_status?(value)
+          %i[wait_readable wait_writable].include?(value)
+        end
+
+        def check_io_deadline!(expires_at, label)
+          deadline.check!
+          domain_error(label) unless (expires_at - monotonic_time).positive?
+        end
+
+        def wait_for_io(status, io:, expires_at:, label:)
+          timeout = [expires_at - monotonic_time, deadline.remaining].min
+          domain_error(label) unless timeout.positive?
+
+          ready = status == :wait_readable ? io.wait_readable(timeout) : io.wait_writable(timeout)
+          return if ready
+
+          deadline.check!
+          domain_error(label)
+        end
+
+        def monotonic_time
+          Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/analyzers/dynamic/redis_payload_loader.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'uri'
+require 'yaml'
+
+require_relative 'coverband_payload_set'
+require_relative 'redis_input_limits'
+require_relative 'redis_transport'
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      class RedisPayloadLoader
+        DEFAULT_PATTERN = 'coverband*'
+
+        def initialize(source:, config:)
+          @redactor = RedisCredentialRedactor.new(source)
+          @uri = parse_uri(source)
+          @config = config
+          @limits = RedisInputLimits.new(config)
+        end
+
+        def load
+          start_session
+          transport.connect
+          authenticate
+          select_database
+          payload_processor.merge(keys.filter_map { |key| payload_for_key(key) })
+        rescue RedisCommandError, Error => e
+          raise Error, redactor.redact(e.message), cause: nil
+        rescue StandardError => e
+          raise Error, redactor.redact("Could not load Redis coverage: #{e.message}"), cause: nil
+        ensure
+          transport&.close
+        end
+
+        private
+
+        attr_reader :uri, :config, :limits, :redactor, :transport, :deadline, :payload_processor
+
+        def parse_uri(source)
+          parsed = URI(source)
+          return parsed if %w[redis rediss].include?(parsed.scheme) && parsed.host
+
+          raise Error, 'Invalid Redis source URI'
+        rescue URI::Error, ArgumentError
+          raise Error, 'Invalid Redis source URI', cause: nil
+        end
+
+        def start_session
+          @deadline = RedisDeadline.new(limits.total_timeout)
+          @payload_processor = CoverbandPayloadSet.new(limits: limits, deadline: deadline)
+          @transport = RedisTransport.new(uri: uri, limits: limits, deadline: deadline, redactor: redactor)
+        end
+
+        def authenticate
+          return unless uri.password
+
+          password = URI.decode_www_form_component(uri.password)
+          return command('AUTH', password) unless uri.user
+
+          command('AUTH', URI.decode_www_form_component(uri.user), password)
+        end
+
+        def select_database
+          database = uri.path.to_s.delete_prefix('/')
+          command('SELECT', database) unless database.empty?
+        end
+
+        def keys
+          configured = Array(config['keys'] || config['key'] || query['key']).compact
+          return validate_key_count!(configured) unless configured.empty?
+
+          scan(pattern: config['pattern'] || query['pattern'] || DEFAULT_PATTERN)
+        end
+
+        def query
+          @query ||= URI.decode_www_form(uri.query.to_s).to_h
+        end
+
+        def scan(pattern:)
+          cursor = '0'
+          found = []
+          loop do
+            deadline.check!
+            cursor, batch = scan_page(cursor, pattern)
+            append_keys!(found, batch)
+            break if cursor == '0'
+          end
+          found
+        end
+
+        def scan_page(cursor, pattern)
+          response = command('SCAN', cursor, 'MATCH', pattern)
+          valid = response.is_a?(Array) && response.length == 2 && response.last.is_a?(Array)
+          raise Error, 'Malformed Redis SCAN response' unless valid
+
+          next_cursor = response.first.to_s
+          raise Error, 'Malformed Redis SCAN cursor' unless next_cursor.match?(/\A\d+\z/)
+
+          [next_cursor, response.last]
+        end
+
+        def append_keys!(found, batch)
+          raise Error, 'Redis key count exceeds configured limit' if found.length + batch.length > limits.max_keys
+
+          found.concat(batch)
+        end
+
+        def validate_key_count!(keys)
+          raise Error, 'Redis key count exceeds configured limit' if keys.length > limits.max_keys
+
+          keys
+        end
+
+        def payload_for_key(key)
+          value = command('GET', key)
+          return nil if value.nil? || value.empty?
+
+          payload = parse_payload(value)
+          raise Error, 'Redis coverage payload must be a mapping' unless payload.is_a?(Hash)
+
+          limits.validate_payload!(payload)
+        rescue RedisCommandError => e
+          raise unless e.message.start_with?('WRONGTYPE')
+
+          parse_hash_payload(command('HGETALL', key))
+        end
+
+        def command(*parts)
+          transport.command(*parts)
+        end
+
+        def parse_payload(value)
+          return nil if value.nil? || value.empty?
+
+          parsed = begin
+            JSON.parse(value, max_nesting: limits.max_payload_depth)
+          rescue JSON::ParserError
+            parse_yaml_payload(value)
+          end
+          limits.validate_payload!(parsed)
+        end
+
+        def parse_yaml_payload(value)
+          YAML.safe_load(value, aliases: false)
+        rescue Psych::Exception, ArgumentError, EncodingError, SystemStackError => e
+          raise Error, redactor.redact("Invalid Redis coverage payload: #{e.message}"), cause: nil
+        end
+
+        def parse_hash_payload(entries)
+          active_payload_processor.from_hash_entries(entries) { |value| parse_payload(value) }
+        end
+
+        def active_payload_processor
+          @active_payload_processor ||= CoverbandPayloadSet.new(
+            limits: limits,
+            deadline: (@deadline ||= RedisDeadline.new(limits.total_timeout))
+          )
+        end
+      end
+    end
+  end
+end