necropsy 0.1.0 → 0.3.0

data/lib/necropsy/analyzers/dynamic/redis_transport.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require 'openssl'
+require 'socket'
+require 'timeout'
+
+require_relative 'redis_nonblocking_io'
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      class RedisCommandError < StandardError; end
+
+      class RedisTransport
+        include RedisNonblockingIO
+
+        CRLF = "\r\n"
+
+        def initialize(uri:, limits:, deadline:, redactor:)
+          @uri = uri
+          @limits = limits
+          @deadline = deadline
+          @redactor = redactor
+          @response_bytes = 0
+          @read_buffer = +''
+        end
+
+        def connect
+          @tcp_socket = connect_tcp
+          @socket = uri.scheme == 'rediss' ? connect_tls : tcp_socket
+          deadline.check!
+          self
+        rescue Error
+          close
+          raise
+        rescue SystemCallError, SocketError, IO::TimeoutError, OpenSSL::SSL::SSLError => e
+          close
+          domain_error("Could not connect securely to Redis at #{target}: #{e.message}")
+        end
+
+        def close
+          socket&.close
+          tcp_socket&.close unless socket.equal?(tcp_socket)
+        rescue IOError, SystemCallError
+          nil
+        ensure
+          @socket = nil
+          @tcp_socket = nil
+        end
+
+        def command(*parts)
+          deadline.check!
+          write_all(redis_command(parts))
+          response = read_response
+          deadline.check!
+          response
+        rescue RedisCommandError
+          raise
+        rescue IOError, SystemCallError, OpenSSL::SSL::SSLError => e
+          domain_error("Redis connection failed: #{e.message}")
+        end
+
+        private
+
+        attr_reader :uri, :limits, :deadline, :redactor, :socket, :tcp_socket
+
+        def connect_tcp
+          expires_at = monotonic_time + limits.connect_timeout
+          last_error = nil
+          addresses(expires_at).each do |address|
+            candidate = Socket.new(address.afamily, address.socktype, address.protocol)
+            begin
+              finish_tcp_connect(candidate, address, expires_at)
+              return candidate
+            rescue SystemCallError, SocketError => e
+              candidate.close
+              last_error = e
+            rescue Error
+              candidate.close
+              raise
+            end
+          end
+          raise(last_error || SocketError.new("No Redis address found for #{uri.host}"))
+        end
+
+        def addresses(expires_at)
+          timeout = [expires_at - monotonic_time, deadline.remaining].min
+          domain_error('Redis address resolution timeout exceeded') unless timeout.positive?
+
+          # Limit asynchronous interruption to pure resolution, before any socket resource exists.
+          resolved = Timeout.timeout(timeout) do
+            Addrinfo.getaddrinfo(uri.host, uri.port || 6379, nil, :STREAM)
+          end
+          deadline.check!
+          domain_error('Redis address resolution timeout exceeded') unless (expires_at - monotonic_time).positive?
+          resolved
+        rescue Timeout::Error
+          deadline.check!
+          domain_error('Redis address resolution timeout exceeded')
+        end
+
+        def finish_tcp_connect(candidate, address, expires_at)
+          status = candidate.connect_nonblock(address, exception: false)
+          return unless wait_status?(status)
+
+          wait_for_io(status, io: candidate, expires_at: expires_at, label: 'Redis connect timeout exceeded')
+          error = candidate.getsockopt(Socket::SOL_SOCKET, Socket::SO_ERROR).int
+          raise SystemCallError.new("connect(2) for #{target}", error) unless error.zero?
+        rescue Errno::EISCONN
+          nil
+        end
+
+        def connect_tls
+          context = OpenSSL::SSL::SSLContext.new
+          store = OpenSSL::X509::Store.new
+          store.set_default_paths
+          context.cert_store = store
+          context.verify_mode = OpenSSL::SSL::VERIFY_PEER
+          context.verify_hostname = true if context.respond_to?(:verify_hostname=)
+
+          ssl_socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, context)
+          ssl_socket.sync_close = true
+          ssl_socket.hostname = uri.host
+          finish_tls_connect(ssl_socket)
+          ssl_socket.post_connection_check(uri.host)
+          ssl_socket
+        rescue StandardError
+          ssl_socket&.close
+          raise
+        end
+
+        def target
+          "#{uri.host}:#{uri.port || 6379}"
+        end
+
+        def redis_command(parts)
+          encoded = parts.flat_map { |part| ["$#{part.to_s.bytesize}", part.to_s] }
+          (["*#{parts.length}"] + encoded).join(CRLF) + CRLF
+        end
+
+        def read_response(depth = 0)
+          prefix = read_exact(1)
+          case prefix
+          when '+' then read_line
+          when '-' then raise RedisCommandError, redactor.redact(read_line)
+          when ':' then read_integer('integer')
+          when '$' then read_bulk_string
+          when '*' then read_array(depth)
+          else domain_error("Malformed Redis response prefix #{prefix.inspect}")
+          end
+        end
+
+        def read_bulk_string
+          length = read_integer('bulk length', nullable: true)
+          return nil if length == -1
+
+          domain_error('Invalid negative Redis bulk length') if length.negative?
+          domain_error('Redis bulk response exceeds configured limit') if length > limits.max_bulk_bytes
+
+          value = read_exact(length)
+          domain_error('Malformed Redis bulk response terminator') unless read_exact(2) == CRLF
+          value
+        end
+
+        def read_array(depth)
+          length = read_integer('array length', nullable: true)
+          return nil if length == -1
+
+          domain_error('Invalid negative Redis array length') if length.negative?
+          domain_error('Redis array response exceeds configured element limit') if length > limits.max_array_elements
+
+          child_depth = depth + 1
+          domain_error('Redis response exceeds configured nesting depth') if child_depth > limits.max_resp_depth
+          Array.new(length) { read_response(child_depth) }
+        end
+
+        def read_integer(label, nullable: false)
+          text = read_line
+          domain_error("Malformed Redis #{label}") unless text.match?(/\A-?\d+\z/)
+
+          value = Integer(text, 10)
+          domain_error("Invalid negative Redis #{label}") if value.negative? && (!nullable || value < -1)
+          value
+        end
+
+        def read_line
+          loop do
+            if (terminator = @read_buffer.index(CRLF))
+              return @read_buffer.slice!(0, terminator + CRLF.bytesize).delete_suffix(CRLF)
+            end
+
+            fill_buffer
+          end
+        end
+
+        def read_exact(length)
+          fill_buffer while @read_buffer.bytesize < length
+          @read_buffer.slice!(0, length)
+        end
+
+        def fill_buffer
+          deadline.check!
+          remaining = limits.max_response_bytes - @response_bytes
+          domain_error('Redis response exceeds configured byte limit') unless remaining.positive?
+
+          chunk = read_nonblock([4096, remaining].min)
+          @response_bytes += chunk.bytesize
+          @read_buffer << chunk
+        end
+
+        def domain_error(message)
+          raise Error, redactor.redact(message), cause: nil
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/analyzers/dynamic/runtime_reference.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Analyzers
+    module Dynamic
+      module RuntimeReference
+        module_function
+
+        def build(symbol_id:, file: nil, line: nil, definition_id: nil)
+          {
+            'definition_id' => definition_id,
+            'symbol_id' => symbol_id,
+            'file' => file,
+            'line' => line
+          }.compact
+        end
+
+        def normalize(value)
+          return value if value.is_a?(String)
+          return unless value.is_a?(Hash)
+
+          data = value.to_h { |key, item| [key.to_s, item] }
+          symbol_id = data['symbol_id']
+          return unless symbol_id.is_a?(String) && !symbol_id.empty?
+          return unless valid_optional_string?(data, 'definition_id') && valid_optional_string?(data, 'file')
+
+          line = normalize_line(data['line'])
+          return if data.key?('line') && !data['line'].nil? && line.nil?
+
+          build(
+            definition_id: optional_string(data['definition_id']),
+            symbol_id: symbol_id,
+            file: optional_string(data['file']),
+            line: line
+          )
+        end
+
+        def sort_key(reference)
+          normalized = normalize(reference)
+          return ['', reference.to_s, '', 0] unless normalized.is_a?(Hash)
+
+          [
+            normalized.fetch('definition_id', ''),
+            normalized.fetch('symbol_id'),
+            normalized.fetch('file', ''),
+            normalized.fetch('line', 0)
+          ]
+        end
+
+        def key(reference)
+          sort_key(reference)
+        end
+
+        def preferred(structured:, legacy:)
+          return Array(legacy) if structured.nil?
+
+          structured_values = Array(structured)
+          structured_symbols = structured_values.filter_map do |reference|
+            normalize(reference).then { |normalized| normalized['symbol_id'] if normalized.is_a?(Hash) }
+          end.to_set
+          structured_values + Array(legacy).reject { |symbol_id| structured_symbols.include?(symbol_id) }
+        end
+
+        def malformed?(reference)
+          !reference.is_a?(String) && normalize(reference).nil?
+        end
+
+        def relative_file(root, path)
+          expanded = File.expand_path(path)
+          prefix = "#{File.expand_path(root)}/"
+          return unless expanded.start_with?(prefix)
+
+          expanded.delete_prefix(prefix)
+        end
+
+        def optional_string(value)
+          value if value.is_a?(String) && !value.empty?
+        end
+        private_class_method :optional_string
+
+        def valid_optional_string?(data, key)
+          !data.key?(key) || data[key].nil? || (data[key].is_a?(String) && !data[key].empty?)
+        end
+        private_class_method :valid_optional_string?
+
+        def normalize_line(value)
+          return if value.nil?
+
+          line = Integer(value, exception: false)
+          line if line&.positive?
+        end
+        private_class_method :normalize_line
+      end
+    end
+  end
+end

data/lib/necropsy/analyzers/dynamic/trace_point_collector.rb

@@ -1,57 +1,124 @@
 # frozen_string_literal: true
 
 require 'securerandom'
+require 'fileutils'
 require 'time'
 require 'yaml'
+require_relative '../../clock'
+require_relative 'runtime_reference'
+require_relative 'observation_policy'
 
 module Necropsy
   module Analyzers
     module Dynamic
       class TracePointCollector
-        def self.record(root:, output:, sample_rate: 1.0, &)
-          new(root: root, output: output, sample_rate: sample_rate).record(&)
+        def self.record(root:, output:, sample_rate: 1.0, clock: nil, random: nil, &)
+          new(root: root, output: output, sample_rate: sample_rate, clock: clock, random: random).record(&)
         end
 
-        def initialize(root:, output:, sample_rate: 1.0)
+        def self.install_at_exit(root:, output:, sample_rate: 1.0, merge: false, run_id: nil, clock: nil, random: nil)
+          new(
+            root: root, output: output, sample_rate: sample_rate, merge: merge, run_id: run_id,
+            clock: clock, random: random
+          ).install_at_exit
+        end
+
+        def initialize(root:, output:, sample_rate: 1.0, merge: false, run_id: nil, clock: nil, random: nil)
           @root = File.expand_path(root)
           @output = output
           @sample_rate = sample_rate.to_f
+          raise Error, 'sample_rate must be between 0.0 and 1.0' unless @sample_rate.between?(0.0, 1.0)
+
           @nodes = {}
+          @node_references = {}
           @edges = {}
-          @stack = []
+          @edge_references = {}
+          @stacks = {}.compare_by_identity
+          @lock = Mutex.new
+          @merge = merge
+          @run_id = run_id
+          @clock = clock || -> { Clock.new.time }
+          @random = random || default_random
         end
 
         def record
-          started_at = Time.now.utc
+          started_at = current_time
           tracer = TracePoint.new(:call, :return) { |event| capture(event) }
           tracer.enable
           yield
         ensure
           tracer&.disable
-          write_payload(started_at: started_at, finished_at: Time.now.utc)
+          write_payload(started_at: started_at, finished_at: current_time)
+        end
+
+        def install_at_exit
+          started_at = current_time
+          tracer = TracePoint.new(:call, :return) { |event| capture(event) }
+          tracer.enable
+          at_exit do
+            tracer.disable
+            write_payload(started_at: started_at, finished_at: current_time)
+          rescue StandardError => e
+            warn "Necropsy TracePoint collector failed: #{e.message}"
+          end
         end
 
         private
 
-        attr_reader :root, :output, :sample_rate, :nodes, :edges, :stack
+        attr_reader :root, :output, :sample_rate, :nodes, :node_references, :edges, :edge_references, :stacks, :lock,
+                    :run_id, :clock, :random
+
+        def merge?
+          @merge
+        end
 
         def capture(event)
           return unless project_path?(event.path)
-          return if sample_rate < 1.0 && SecureRandom.random_number > sample_rate
 
-          node_id = node_id_for(event)
-          return unless node_id
+          reference = node_reference_for(event)
+          return unless reference
+
+          lock.synchronize do
+            stack = stacks[Thread.current] ||= []
+            if event.event == :return
+              unwind_stack(stack, reference)
+              stacks.delete(Thread.current) if stack.empty?
+              return
+            end
 
-          if event.event == :return
-            stack.pop if stack.last == node_id
-            return
+            sampled = sample_rate >= 1.0 || (sample_rate.positive? && random.call < sample_rate)
+            caller_reference = stack.last&.first if stack.last&.last
+            if sampled
+              record_node(reference)
+              record_edge(caller_reference, reference) if caller_reference
+            end
+            stack << [reference, sampled]
           end
+        end
+
+        def current_time
+          clock.call.utc
+        end
 
-          caller_id = stack.last
-          nodes[node_id] = true
-          edges[[caller_id, node_id]] = true if caller_id
+        def default_random
+          epoch = ENV.fetch(Clock::SOURCE_DATE_EPOCH, nil)
+          return SecureRandom.method(:random_number) if epoch.nil? || epoch.empty?
 
-          stack << node_id
+          Random.new(Integer(epoch)).method(:rand)
+        rescue ArgumentError
+          SecureRandom.method(:random_number)
+        end
+
+        def unwind_stack(stack, reference)
+          reference_key = invocation_key(reference)
+          index = stack.rindex { |frame| invocation_key(frame.first) == reference_key }
+          return unless index
+
+          stack.slice!(index..)
+        end
+
+        def invocation_key(reference)
+          reference.values_at('definition_id', 'symbol_id', 'file')
         end
 
         def project_path?(path)
@@ -67,6 +134,28 @@ module Necropsy
           "#{owner}#{separator}#{event.method_id}"
         end
 
+        def node_reference_for(event)
+          symbol_id = node_id_for(event)
+          return unless symbol_id
+
+          RuntimeReference.build(
+            symbol_id: symbol_id,
+            file: RuntimeReference.relative_file(root, event.path),
+            line: event.respond_to?(:lineno) ? event.lineno : nil
+          )
+        end
+
+        def record_node(reference)
+          nodes[reference.fetch('symbol_id')] = true
+          node_references[RuntimeReference.key(reference)] = reference
+        end
+
+        def record_edge(caller, callee)
+          edges[[caller.fetch('symbol_id'), callee.fetch('symbol_id')]] = true
+          key = [RuntimeReference.key(caller), RuntimeReference.key(callee)]
+          edge_references[key] = { 'caller_id' => caller.dup, 'callee_id' => callee.dup }
+        end
+
         def owner_and_separator(defined_class)
           name = defined_class.name
           return [name, '#'] if name
@@ -79,16 +168,132 @@ module Necropsy
 
         def write_payload(started_at:, finished_at:)
           payload = {
+            'schema_version' => 2,
+            'collector' => { 'name' => 'necropsy-trace-point', 'version' => Necropsy::VERSION },
+            'scope' => { 'sample_unit' => 'event', 'sample_rate' => sample_rate },
+            'quality' => { 'dropped_events' => 0, 'overflowed' => false },
             'nodes' => nodes.keys.sort,
-            'edges' => edges.keys.map { |caller_id, callee_id| { 'caller_id' => caller_id, 'callee_id' => callee_id } },
+            'node_references' => node_references.values.sort_by { |reference| RuntimeReference.sort_key(reference) },
+            'edges' => legacy_edges,
+            'edge_references' => structured_edges,
             'observation' => {
               'started_at' => started_at.iso8601,
               'finished_at' => finished_at.iso8601,
               'days' => [((finished_at - started_at) / 86_400.0).ceil, 1].max,
-              'sample_rate' => sample_rate
+              'sample_rate' => sample_rate,
+              'collector_overhead' => {
+                'wall_time_seconds' => [finished_at - started_at, 0].max.round(6),
+                'observed_nodes' => nodes.length,
+                'observed_edges' => edges.length
+              }
             }
           }
-          File.write(output, payload.to_yaml)
+          payload['observation']['run_id'] = run_id if run_id
+          FileUtils.mkdir_p(File.dirname(output))
+          merge? ? write_merged_payload(payload) : File.write(output, payload.to_yaml)
+        end
+
+        def write_merged_payload(payload)
+          File.open(output, File::RDWR | File::CREAT, 0o644) do |file|
+            file.flock(File::LOCK_EX)
+            existing = trace_payload_for_current_run(read_payload(file), payload)
+            merged = merge_payload(existing, payload)
+            file.rewind
+            file.truncate(0)
+            file.write(merged.to_yaml)
+            file.flush
+          end
+        end
+
+        def read_payload(file)
+          file.rewind
+          content = file.read
+          return {} if content.empty?
+
+          YAML.safe_load(content, aliases: false) || {}
+        rescue Psych::Exception
+          {}
+        end
+
+        def trace_payload_for_current_run(existing, payload)
+          current_run = payload.dig('observation', 'run_id')
+          return existing unless current_run
+          return existing if existing.empty? || existing.dig('observation', 'run_id') == current_run
+
+          {}
+        end
+
+        def merge_payload(left, right)
+          edges = (Array(left['edges']) + Array(right['edges'])).uniq do |edge|
+            [edge['caller_id'], edge['callee_id']]
+          end
+          left_observation = left.fetch('observation', {})
+          right_observation = right.fetch('observation', {})
+          observation = ObservationPolicy.compatible_merge(left_observation, right_observation)
+          observation['started_at'] = [left_observation['started_at'], right_observation['started_at']].compact.min
+          observation['finished_at'] = [left_observation['finished_at'], right_observation['finished_at']].compact.max
+          observation['days'] = [left_observation['days'].to_i, right_observation['days'].to_i, 1].max
+          observation['collector_overhead'] = merge_overhead(left_observation, right_observation)
+          observation['processes'] = process_count(left_observation) + process_count(right_observation)
+          {
+            'schema_version' => [left['schema_version'], right['schema_version'], 2].compact.max,
+            'collector' => right['collector'] || left['collector'],
+            'scope' => right['scope'] || left['scope'],
+            'quality' => right['quality'] || left['quality'],
+            'nodes' => (Array(left['nodes']) + Array(right['nodes'])).uniq.sort,
+            'node_references' => merge_node_references(left['node_references'], right['node_references']),
+            'edges' => edges,
+            'edge_references' => merge_edge_references(left['edge_references'], right['edge_references']),
+            'observation' => observation
+          }
+        end
+
+        def legacy_edges
+          edges.keys.sort.map { |caller_id, callee_id| { 'caller_id' => caller_id, 'callee_id' => callee_id } }
+        end
+
+        def structured_edges
+          edge_references.values.sort_by do |edge|
+            [RuntimeReference.sort_key(edge.fetch('caller_id')), RuntimeReference.sort_key(edge.fetch('callee_id'))]
+          end
+        end
+
+        def merge_node_references(left, right)
+          (Array(left) + Array(right))
+            .filter_map { |reference| RuntimeReference.normalize(reference) }
+            .uniq { |reference| RuntimeReference.key(reference) }
+            .sort_by { |reference| RuntimeReference.sort_key(reference) }
+        end
+
+        def merge_edge_references(left, right)
+          (Array(left) + Array(right)).filter_map do |edge|
+            next unless edge.is_a?(Hash)
+
+            caller = RuntimeReference.normalize(edge['caller_id'] || edge[:caller_id])
+            callee = RuntimeReference.normalize(edge['callee_id'] || edge[:callee_id])
+            { 'caller_id' => caller, 'callee_id' => callee } if caller && callee
+          end.uniq do |edge|
+            [RuntimeReference.key(edge.fetch('caller_id')), RuntimeReference.key(edge.fetch('callee_id'))]
+          end.sort_by do |edge|
+            [RuntimeReference.sort_key(edge.fetch('caller_id')), RuntimeReference.sort_key(edge.fetch('callee_id'))]
+          end
+        end
+
+        def process_count(observation)
+          return observation['processes'].to_i if observation['processes']
+          return 0 if observation.empty?
+
+          1
+        end
+
+        def merge_overhead(left, right)
+          left_overhead = left.fetch('collector_overhead', {})
+          right_overhead = right.fetch('collector_overhead', {})
+          {
+            'wall_time_seconds' => left_overhead['wall_time_seconds'].to_f + right_overhead['wall_time_seconds'].to_f,
+            'observed_nodes' => left_overhead['observed_nodes'].to_i + right_overhead['observed_nodes'].to_i,
+            'observed_edges' => left_overhead['observed_edges'].to_i + right_overhead['observed_edges'].to_i
+          }
         end
       end
     end

data/lib/necropsy/analyzers/dynamic/trace_point_importer.rb

@@ -9,7 +9,9 @@ module Necropsy
             name: :trace_point,
             kind: :dynamic,
             soundness: :observational,
-            description: 'Imports method execution and caller-callee edges captured by Necropsy TracePoint recording.'
+            description: 'Imports method execution and caller-callee edges captured by Necropsy TracePoint recording.',
+            version: Necropsy::VERSION,
+            assumptions: %w[positive_observations_only tracepoint_call_events]
           )
         end
       end