vivarium 0.3.0 → 0.3.2

data/lib/vivarium/http_decoder.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+begin
+  require "http/2"
+rescue LoadError
+  # http/2 gem is optional; without it we can still parse frame headers but not HPACK-decompress HEADERS.
+end
+
+module Vivarium
+  # Decodes payloads captured from OpenSSL `SSL_write` into a human-readable
+  # one-liner. Auto-detects HTTP/1.x request/response lines and HTTP/2 binary
+  # frames; HPACK-decompresses HEADERS / CONTINUATION when the `http-2` gem
+  # is available, otherwise reports frame types only.
+  #
+  # HPACK decompressor state is kept per pid. This is sufficient for the
+  # common "one HTTPS connection per process" case; with multiple concurrent
+  # TLS connections per pid the HPACK table can diverge and decoding may
+  # fail — in that case the decompressor for that pid is reset on the next
+  # decode error.
+  class HttpDecoder
+    HTTP1_METHODS = %w[GET POST PUT PATCH DELETE HEAD OPTIONS TRACE CONNECT].freeze
+    HTTP2_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n".b
+    H2_FRAME_HEADER_SIZE = 9
+    H2_FLAG_END_HEADERS = 0x04
+    H2_FLAG_PADDED = 0x08
+    H2_FLAG_PRIORITY = 0x20
+    FRAME_TYPE_NAMES = {
+      0x0 => "DATA",
+      0x1 => "HEADERS",
+      0x2 => "PRIORITY",
+      0x3 => "RST_STREAM",
+      0x4 => "SETTINGS",
+      0x5 => "PUSH_PROMISE",
+      0x6 => "PING",
+      0x7 => "GOAWAY",
+      0x8 => "WINDOW_UPDATE",
+      0x9 => "CONTINUATION"
+    }.freeze
+
+    def initialize
+      @hpack_available = load_http2_gem
+      @decompressors = {}
+      @continuation = {}
+    end
+
+    def hpack_available?
+      @hpack_available
+    end
+
+    def render(pid:, data:, data_len:)
+      data = data.to_s.b
+      data_len = data_len.to_i
+
+      return "data_len=0" if data_len <= 0
+      return "len=#{data_len} <no-capture>" if data.empty?
+
+      if (summary = http1_summary(data))
+        kind, line = summary
+        return "http/1.x #{kind}: #{line}#{truncation_note(data, data_len)}"
+      end
+
+      rest = data
+      preface_note = ""
+      if rest.start_with?(HTTP2_PREFACE)
+        preface_note = "preface "
+        rest = rest.byteslice(HTTP2_PREFACE.bytesize..) || "".b
+      end
+
+      frames = parse_h2_frames(rest)
+      if frames.empty?
+        if !preface_note.empty?
+          return "h2 preface only#{truncation_note(data, data_len)}"
+        end
+        return "binary len=#{data_len}#{truncation_note(data, data_len)}"
+      end
+
+      rendered = frames.map { |f| render_h2_frame(pid, f) }.join(" | ")
+      "h2 #{preface_note}#{rendered}#{truncation_note(data, data_len)}"
+    end
+
+    private
+
+    def truncation_note(data, data_len)
+      return "" if data.bytesize >= data_len
+
+      " (captured #{data.bytesize}/#{data_len}B)"
+    end
+
+    def load_http2_gem
+      require "http/2"
+      true
+    rescue LoadError
+      false
+    end
+
+    def http1_summary(data)
+      head = data.byteslice(0, 512).to_s
+      first_line = head.split("\r\n", 2).first
+      return nil if first_line.nil? || first_line.empty?
+
+      first_line = first_line.dup.force_encoding(Encoding::UTF_8)
+      return nil unless first_line.valid_encoding?
+
+      if HTTP1_METHODS.any? { |m| first_line.start_with?("#{m} ") }
+        return ["request", first_line]
+      end
+
+      if first_line.start_with?("HTTP/1.1 ") || first_line.start_with?("HTTP/1.0 ")
+        return ["response", first_line]
+      end
+
+      nil
+    end
+
+    # @return [Array<Array(Integer, Integer, Integer, String, Boolean)>]
+    #   each entry: [frame_type, flags, stream_id, frame_payload, truncated?]
+    def parse_h2_frames(payload)
+      frames = []
+      i = 0
+      total = payload.bytesize
+
+      while i + H2_FRAME_HEADER_SIZE <= total
+        length = (payload.getbyte(i) << 16) |
+                 (payload.getbyte(i + 1) << 8) |
+                 payload.getbyte(i + 2)
+        frame_type = payload.getbyte(i + 3)
+        flags = payload.getbyte(i + 4)
+        stream_id = payload.byteslice(i + 5, 4).unpack1("N") & 0x7fff_ffff
+        i += H2_FRAME_HEADER_SIZE
+
+        if i + length > total
+          remaining = payload.byteslice(i, total - i) || "".b
+          frames << [frame_type, flags, stream_id, remaining, true]
+          break
+        end
+
+        frame_payload = payload.byteslice(i, length) || "".b
+        i += length
+        frames << [frame_type, flags, stream_id, frame_payload, false]
+      end
+
+      # Heuristic: if the very first "frame" doesn't look like a valid HTTP/2
+      # frame, refuse the whole parse so we fall back to "binary".
+      first_type = frames.first && frames.first[0]
+      return [] if first_type && !FRAME_TYPE_NAMES.key?(first_type)
+
+      frames
+    end
+
+    def render_h2_frame(pid, frame)
+      frame_type, flags, stream_id, frame_payload, truncated = frame
+      frame_name = FRAME_TYPE_NAMES.fetch(frame_type, "TYPE0x#{format('%02x', frame_type)}")
+      header = "#{frame_name} stream=#{stream_id} flags=0x#{format('%02x', flags)} len=#{frame_payload.bytesize}#{truncated ? '*' : ''}"
+
+      case frame_type
+      when 0x1 # HEADERS
+        fragment = headers_fragment(flags, frame_payload)
+        return "#{header} <bad_payload>" if fragment.nil?
+
+        if (flags & H2_FLAG_END_HEADERS) != 0
+          pseudo = decode_hpack(pid, fragment)
+          "#{header}#{format_pseudo(pseudo)}"
+        else
+          @continuation[[pid, stream_id]] = fragment.dup
+          "#{header} <collecting>"
+        end
+      when 0x9 # CONTINUATION
+        key = [pid, stream_id]
+        unless @continuation.key?(key)
+          return "#{header} <orphan>"
+        end
+
+        @continuation[key] << frame_payload
+        if (flags & H2_FLAG_END_HEADERS) == 0
+          "#{header} <collecting>"
+        else
+          buf = @continuation.delete(key)
+          pseudo = decode_hpack(pid, buf)
+          "#{header}#{format_pseudo(pseudo)}"
+        end
+      else
+        header
+      end
+    end
+
+    def headers_fragment(flags, frame_payload)
+      start_idx = 0
+      end_idx = frame_payload.bytesize
+
+      if (flags & H2_FLAG_PADDED) != 0
+        return nil if end_idx.zero?
+
+        pad_len = frame_payload.getbyte(0)
+        start_idx += 1
+        end_idx = [start_idx, end_idx - pad_len].max
+      end
+
+      if (flags & H2_FLAG_PRIORITY) != 0
+        return nil if start_idx + 5 > end_idx
+
+        start_idx += 5
+      end
+
+      frame_payload.byteslice(start_idx, end_idx - start_idx) || "".b
+    end
+
+    def decompressor_for(pid)
+      return nil unless @hpack_available
+
+      @decompressors[pid] ||= HTTP2::Header::Decompressor.new
+    end
+
+    def decode_hpack(pid, header_block)
+      dec = decompressor_for(pid)
+      return { ":error" => "hpack-unavailable" } unless dec
+
+      pairs = dec.decode(header_block.b)
+      pairs.each_with_object({}) { |(k, v), h| h[k] = v }
+    rescue StandardError => e
+      @decompressors.delete(pid)
+      { ":error" => "#{e.class}: #{e.message}" }
+    end
+
+    def format_pseudo(pseudo)
+      return " <error: #{pseudo[':error']}>" if pseudo.key?(":error")
+
+      parts = []
+      parts << ":method=#{pseudo[':method']}" if pseudo[':method']
+      parts << ":path=#{pseudo[':path']}" if pseudo[':path']
+      parts << ":authority=#{pseudo[':authority']}" if pseudo[':authority']
+      parts << ":status=#{pseudo[':status']}" if pseudo[':status']
+      return "" if parts.empty?
+
+      " #{parts.join(' ')}"
+    end
+  end
+end

data/lib/vivarium/tree_renderer.rb

@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
 require "set"
+require_relative "http_decoder"
 
 module Vivarium
   class TreeRenderer
     SPAN_EVENT_NAMES = %w[span_start span_stop].to_set.freeze
     FORK_EVENT_NAME = "proc_fork"
     EXEC_EVENT_NAME = "proc_exec"
+    SSL_WRITE_EVENT_NAME = "ssl_write"
 
     LSM_EVENT_NAMES = %w[
       path_open sock_connect odd_socket
@@ -19,6 +21,9 @@ module Vivarium
       dns_req proc_exec file_getdents proc_fork
     ].to_set.freeze
 
+    UPROBE_EVENT_NAMES = %w[ssl_write].to_set.freeze
+    DL_EVENT_NAMES = %w[dlopen mmap_exec].to_set.freeze
+
     SYNTHETIC_SPAN_NAME = "<no-span>"
     UNRESOLVED_METHOD_PREFIX = "<method_id="
 
@@ -49,7 +54,7 @@ module Vivarium
 
     def initialize(events:, method_table:, observer_pid:, main_tid:,
                    session_start_iso:, session_start_ktime:,
-                   session_stop_iso:, session_stop_ktime:, dest:)
+                   session_stop_iso:, session_stop_ktime:, filter: nil, dest:)
       @events = events
       @method_table = method_table
       @observer_pid = observer_pid
@@ -58,6 +63,7 @@ module Vivarium
       @session_start_ktime = session_start_ktime
       @session_stop_iso = session_stop_iso
       @session_stop_ktime = session_stop_ktime
+      @display_filter = Vivarium::DisplayFilter.compile(filter)
       @dest = dest
 
       @pid_comm = { observer_pid => "ruby" }
@@ -265,6 +271,7 @@ module Vivarium
       @dest.puts "[PROC pid=#{@observer_pid} comm=#{@pid_comm[@observer_pid] || 'ruby'}]"
       children = spans.reject { |s| s.synthetic && s.events.empty? }
                       .reject { |s| @child_span_set.include?(s) }
+                      .select { |s| span_visible?(s) }
       children.each_with_index do |span, idx|
         print_span(span, prefix: "", is_last: idx == children.size - 1)
       end
@@ -318,8 +325,17 @@ module Vivarium
     def build_span_children(span)
       proc_node_by_pid = {}
       root_children = []
+      prev_event_ktime = span.start_ktime
 
       span.events.each do |ev|
+        target_text = nil
+        if @display_filter.needs_payload?
+          target_text = render_target(ev)
+          next unless event_visible?(ev, span, target_text)
+        else
+          next unless event_visible?(ev, span)
+        end
+
         if ev.event_name == FORK_EVENT_NAME
           child_pid = read_proc_fork_child_pid(ev.payload)
           child_node = ProcNode.new(
@@ -333,28 +349,28 @@ module Vivarium
           ev_node = EventNode.new(
             kind: kind_for(ev),
             name: ev.event_name,
-            target: render_target(ev),
+            target: target_text || render_target(ev),
             offset_ns: ev.ktime_ns - span.start_ktime,
             child_proc: child_node
           )
 
           parent_container = container_for_pid(ev.pid, span, proc_node_by_pid, root_children)
+          maybe_inject_drop_node(parent_container, ev, span, prev_event_ktime)
           append_event(parent_container, ev_node)
         else
           ev_node = EventNode.new(
             kind: kind_for(ev),
             name: ev.event_name,
-            target: render_target(ev),
+            target: target_text || render_target(ev),
             offset_ns: ev.ktime_ns - span.start_ktime,
             child_proc: nil
           )
 
-          if ev.pid == @observer_pid && ev.tid == span.tid
-            append_event(root_children, ev_node)
+          container = if ev.pid == @observer_pid && ev.tid == span.tid
+            root_children
           elsif (node = proc_node_by_pid[ev.pid])
-            append_event(node.children, ev_node)
+            node.children
           else
-            # event from a descendant pid we haven't materialized — synthesize a stub PROC node
             stub = ProcNode.new(
               pid: ev.pid,
               comm: @pid_comm[ev.pid] || "?",
@@ -362,14 +378,19 @@ module Vivarium
               children: []
             )
             proc_node_by_pid[ev.pid] = stub
-            append_event(stub.children, ev_node)
             root_children << stub
+            stub.children
           end
 
+          maybe_inject_drop_node(container, ev, span, prev_event_ktime)
+          append_event(container, ev_node)
+
           if ev.event_name == EXEC_EVENT_NAME && (node = proc_node_by_pid[ev.pid])
             node.comm = @pid_comm[ev.pid] || node.comm
           end
         end
+
+        prev_event_ktime = ev.ktime_ns
       end
 
       # Interleave child spans by start time among the event/proc nodes
@@ -415,9 +436,30 @@ module Vivarium
       container << ev_node
     end
 
+    def maybe_inject_drop_node(container, ev, span, prev_event_ktime = nil)
+      n = ev.dropped_since_last.to_i
+      return if n.zero?
+
+      # Show the start of the drop window (= time of last good event).
+      # The end is implicitly ev.ktime_ns (shown on the following event line).
+      drop_start_ns = prev_event_ktime ? (prev_event_ktime - span.start_ktime) : nil
+
+      container << EventNode.new(
+        kind: "DROP",
+        name: "dropped_events",
+        target: "#{n} event(s) lost (ringbuf overflow)",
+        offset_ns: drop_start_ns,
+        child_proc: nil
+      )
+    end
+
     def print_nodes(nodes, prefix)
-      nodes.each_with_index do |node, idx|
-        is_last = idx == nodes.size - 1
+      visible_nodes = nodes.select do |node|
+        !node.is_a?(Span) || span_visible?(node)
+      end
+
+      visible_nodes.each_with_index do |node, idx|
+        is_last = idx == visible_nodes.size - 1
         case node
         when EventNode
           print_event_node(node, prefix: prefix, is_last: is_last)
@@ -429,6 +471,21 @@ module Vivarium
       end
     end
 
+    def span_visible?(span)
+      @display_filter.allow_span_name?(span_display_name(span))
+    end
+
+    def event_visible?(ev, span, target_text = nil)
+      @display_filter.allow_event?(
+        event_name: ev.event_name,
+        severity: Vivarium.event_severity(ev.event_name),
+        span_name: span_display_name(span),
+        payload: target_text,
+        pid: ev.pid,
+        tid: ev.tid
+      )
+    end
+
     def print_event_node(node, prefix:, is_last:)
       marker = is_last && node.child_proc.nil? ? "└─ " : "├─ "
       marker = "└─ " if is_last && node.child_proc.nil?
@@ -457,6 +514,8 @@ module Vivarium
     def kind_for(ev)
       return "EXCP" if ev.event_name == "span_raise"
       return "USDT" if SPAN_EVENT_NAMES.include?(ev.event_name)
+      return "SSL" if ev.event_name == SSL_WRITE_EVENT_NAME
+      return "DL" if DL_EVENT_NAMES.include?(ev.event_name)
       return "LSM" if LSM_EVENT_NAMES.include?(ev.event_name)
       return "TP" if TP_EVENT_NAMES.include?(ev.event_name)
 
@@ -465,12 +524,28 @@ module Vivarium
 
     def render_target(ev)
       return render_raise_target(ev) if ev.event_name == "span_raise"
+      return render_ssl_write_target(ev) if ev.event_name == SSL_WRITE_EVENT_NAME
 
       text = Vivarium.render_event_payload(ev).to_s
       text = text.gsub(/\s+/, " ").strip
       text.empty? ? "-" : text
     end
 
+    def render_ssl_write_target(ev)
+      decoded = Vivarium.decode_ssl_write_payload(ev.payload)
+      http_decoder.render(
+        pid: ev.pid,
+        data: decoded[:data],
+        data_len: decoded[:data_len]
+      )
+    rescue StandardError => e
+      "ssl_write <decode-error: #{e.class}: #{e.message}>"
+    end
+
+    def http_decoder
+      @http_decoder ||= Vivarium::HttpDecoder.new
+    end
+
     def render_raise_target(ev)
       bytes = ev.payload.to_s.b
       return "-" if bytes.bytesize < 8

data/lib/vivarium/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vivarium
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end