vivarium 0.5.2 → 0.6.0

data/lib/vivarium/otel_stream.rb

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module Vivarium
+  # Background OTLP/HTTP(JSON) sender. Completed spans are enqueued and flushed
+  # in batches by a worker thread to {endpoint}/v1/traces. Send failures are
+  # logged and the batch is dropped so a resident observation never stalls.
+  class OtelHttpExporter
+    def initialize(endpoint:, flush_interval: 2.0, max_batch: 256, max_queue: 10_000)
+      @uri = build_uri(endpoint)
+      @flush_interval = flush_interval
+      @max_batch = max_batch
+      @max_queue = max_queue
+
+      @queue = []
+      @mutex = Mutex.new
+      @cond = ConditionVariable.new
+      @stop = false
+      @dropped = 0
+      @thread = nil
+    end
+
+    def start
+      return self if @thread
+
+      @thread = Thread.new { worker }
+      self
+    end
+
+    def enqueue(span)
+      @mutex.synchronize do
+        if @queue.size >= @max_queue
+          @dropped += 1
+        else
+          @queue << span
+          @cond.signal
+        end
+      end
+    end
+
+    def shutdown
+      @mutex.synchronize do
+        @stop = true
+        @cond.signal
+      end
+      @thread&.join
+      warn "[vivarium] otel: dropped #{@dropped} span(s) (queue overflow)" if @dropped.positive?
+    end
+
+    private
+
+    def worker
+      until stop_and_drained?
+        batch = take_batch
+        post_batch(batch) unless batch.empty?
+      end
+    rescue StandardError => e
+      warn "[vivarium] otel worker error: #{e.class}: #{e.message}"
+    end
+
+    def stop_and_drained?
+      @mutex.synchronize { @stop && @queue.empty? }
+    end
+
+    def take_batch
+      @mutex.synchronize do
+        @cond.wait(@mutex, @flush_interval) if @queue.empty? && !@stop
+        @queue.shift(@max_batch)
+      end
+    end
+
+    def post_batch(spans)
+      body = JSON.generate(Vivarium::OtelExporter.wrap_document(spans))
+      req = Net::HTTP::Post.new(@uri)
+      req["Content-Type"] = "application/json"
+      req.body = body
+
+      http = Net::HTTP.new(@uri.host, @uri.port)
+      http.use_ssl = (@uri.scheme == "https")
+      http.open_timeout = 5
+      http.read_timeout = 10
+      res = http.request(req)
+      return if res.code.to_s.start_with?("2")
+
+      warn "[vivarium] otel: collector returned HTTP #{res.code} for #{spans.size} span(s)"
+    rescue StandardError => e
+      warn "[vivarium] otel: POST failed (#{e.class}: #{e.message}); dropped #{spans.size} span(s)"
+    end
+
+    def build_uri(endpoint)
+      base = endpoint.to_s.strip.chomp("/")
+      base += "/v1/traces" unless base.end_with?("/v1/traces")
+      URI.parse(base)
+    end
+  end
+
+  # Reconstructs method-call spans live from the event stream and enqueues each
+  # completed span to an OtelHttpExporter. Each observation session becomes one
+  # trace with a "vivarium session" root span, per-thread child spans, and method
+  # spans nested under the current thread/method span.
+  class OtelSpanStreamer
+    SESSION_ROOT_SALT = 0x766976617269756d
+    THREAD_ROOT_SALT = 0x7468726561640000
+    PROCESS_EXIT_EVENT_NAME = "proc_exit"
+
+    def initialize(exporter:, session_start_iso:, session_start_ktime:, observer_pid:, main_tid:)
+      @exporter = exporter
+      start_unix = Vivarium::OtelExporter.iso_to_unix_ns(session_start_iso)
+      start_ktime = session_start_ktime.to_i
+      @to_unix = ->(k) { (start_unix + (k.to_i - start_ktime)).to_s }
+      @session_start_ktime = start_ktime
+      @observer_pid = observer_pid
+      @main_tid = main_tid
+      @trace_hi, @trace_lo = Vivarium.synth_trace_id(observer_pid, main_tid, SESSION_ROOT_SALT, start_ktime)
+      @session_span_id = Vivarium.synth_span_id(@trace_hi, @trace_lo, observer_pid, start_ktime ^ SESSION_ROOT_SALT)
+      @stacks = Hash.new { |h, k| h[k] = [] }
+      @thread_spans = {}
+      @bpf_thread_span_ids = {}
+    end
+
+    def on_event(ev)
+      return if Vivarium::OtelExporter.internal_comm?(ev.comm)
+
+      case ev.event_name
+      when "span_start" then handle_start(ev)
+      when "span_stop" then handle_stop(ev)
+      else handle_event(ev)
+      end
+    end
+
+    # Close any still-open spans (dangling) at end of observation.
+    def finalize(stop_ktime:)
+      @stacks.each_value do |stack|
+        emit_method_span(stack.pop, stop_ktime) until stack.empty?
+      end
+      @thread_spans.each_value { |rec| emit_thread_span(rec, stop_ktime) unless rec[:emitted] }
+      emit_session_span(stop_ktime)
+    end
+
+    private
+
+    def handle_start(ev)
+      thread = touch_thread_span(ev)
+      stack = @stacks[ev.tid]
+      parent = stack.empty? ? thread[:span_id] : stack.last[:span_id]
+
+      name, file, lineno = Vivarium::OtelExporter.read_span_payload(ev.payload)
+      stack.push(
+        tid: ev.tid, pid: ev.pid, trace_hi: @trace_hi, trace_lo: @trace_lo,
+        span_id: Vivarium.synth_span_id(ev.trace_hi.to_i, ev.trace_lo.to_i, ev.tid, ev.ktime_ns),
+        parent: parent, name: (name.nil? || name.empty? ? "<anonymous>" : name),
+        file: file, lineno: lineno, start_k: ev.ktime_ns, events: []
+      )
+    end
+
+    def handle_stop(ev)
+      touch_thread_span(ev)
+      rec = @stacks[ev.tid].pop
+      emit_method_span(rec, ev.ktime_ns) if rec
+    end
+
+    def handle_event(ev)
+      thread = touch_thread_span(ev)
+      if ev.event_name == PROCESS_EXIT_EVENT_NAME
+        emit_thread_span(thread, ev.ktime_ns) unless thread[:root]
+        return
+      end
+
+      stack = @stacks[ev.tid]
+      if stack.empty?
+        thread[:events] << Vivarium::OtelExporter.build_span_event(ev, @to_unix)
+      else
+        stack.last[:events] << Vivarium::OtelExporter.build_span_event(ev, @to_unix)
+      end
+    end
+
+    def touch_thread_span(ev)
+      rec = (@thread_spans[ev.tid] ||= new_thread_span(ev))
+      remember_bpf_thread_span(ev, rec)
+      rec[:comm] = ev.comm.to_s unless ev.comm.to_s.empty?
+      rec[:min_k] = ev.ktime_ns if ev.ktime_ns < rec[:min_k]
+      rec[:max_k] = ev.ktime_ns if ev.ktime_ns > rec[:max_k]
+      rec
+    end
+
+    def new_thread_span(ev)
+      {
+        tid: ev.tid, pid: ev.pid, comm: ev.comm.to_s,
+        span_id: thread_span_id(ev),
+        bpf_parent_span_id: ev.parent_span_id.to_i,
+        parent: ev.tid == @main_tid ? @session_span_id : nil,
+        min_k: ev.ktime_ns, max_k: ev.ktime_ns,
+        root: ev.tid == @main_tid,
+        events: [], emitted: false
+      }
+    end
+
+    def thread_span_id(ev)
+      span_id = ev.span_id.to_i
+      return span_id unless span_id.zero?
+
+      Vivarium.synth_span_id(@trace_hi ^ THREAD_ROOT_SALT, @trace_lo, ev.tid, ev.ktime_ns)
+    end
+
+    def remember_bpf_thread_span(ev, rec)
+      bpf_span_id = ev.span_id.to_i
+      @bpf_thread_span_ids[bpf_span_id] = rec[:span_id] unless bpf_span_id.zero?
+
+      bpf_parent_span_id = ev.parent_span_id.to_i
+      rec[:bpf_parent_span_id] = bpf_parent_span_id unless bpf_parent_span_id.zero?
+    end
+
+    def thread_parent_span_id(rec)
+      parent = rec[:parent]
+      return parent if parent
+
+      mapped = @bpf_thread_span_ids[rec[:bpf_parent_span_id]]
+      mapped && mapped != rec[:span_id] ? mapped : @session_span_id
+    end
+
+    def emit_method_span(rec, end_k)
+      attrs = [
+        Vivarium::OtelExporter.int_attr("thread.id", rec[:tid]),
+        Vivarium::OtelExporter.int_attr("process.pid", rec[:pid])
+      ]
+      attrs << Vivarium::OtelExporter.str_attr("code.filepath", rec[:file]) if rec[:file] && !rec[:file].empty?
+      attrs << Vivarium::OtelExporter.int_attr("code.lineno", rec[:lineno]) if rec[:lineno] && rec[:lineno] > 0
+
+      @exporter.enqueue(
+        Vivarium::OtelExporter.span_hash(
+          trace_hi: rec[:trace_hi], trace_lo: rec[:trace_lo], span_id: rec[:span_id],
+          parent: rec[:parent], name: rec[:name], start_k: rec[:start_k], stop_k: end_k,
+          to_unix: @to_unix, attributes: attrs, events: rec[:events]
+        )
+      )
+    end
+
+    def emit_thread_span(rec, stop_ktime)
+      return if rec[:emitted]
+
+      start_k = rec[:root] ? @session_start_ktime : rec[:min_k]
+      stop_k = rec[:root] ? stop_ktime : [rec[:max_k], stop_ktime].compact.max
+      name = rec[:comm].empty? ? "tid=#{rec[:tid]}" : rec[:comm]
+      attrs = [
+        Vivarium::OtelExporter.int_attr("thread.id", rec[:tid]),
+        Vivarium::OtelExporter.int_attr("process.pid", rec[:pid])
+      ]
+      attrs << Vivarium::OtelExporter.str_attr("process.command", rec[:comm]) unless rec[:comm].empty?
+      @exporter.enqueue(
+        Vivarium::OtelExporter.span_hash(
+          trace_hi: @trace_hi, trace_lo: @trace_lo, span_id: rec[:span_id], parent: thread_parent_span_id(rec),
+          name: name, start_k: start_k, stop_k: stop_k,
+          to_unix: @to_unix, attributes: attrs, events: rec[:events]
+        )
+      )
+      rec[:emitted] = true
+    end
+
+    def emit_session_span(stop_ktime)
+      @exporter.enqueue(
+        Vivarium::OtelExporter.span_hash(
+          trace_hi: @trace_hi, trace_lo: @trace_lo, span_id: @session_span_id, parent: 0,
+          name: "vivarium session", start_k: @session_start_ktime, stop_k: stop_ktime,
+          to_unix: @to_unix,
+          attributes: [
+            Vivarium::OtelExporter.int_attr("process.pid", @observer_pid),
+            Vivarium::OtelExporter.int_attr("thread.id", @main_tid)
+          ],
+          events: []
+        )
+      )
+    end
+  end
+end

data/lib/vivarium/raw_store.rb

@@ -4,7 +4,9 @@ require "json"
 
 module Vivarium
   RawEvent = Struct.new(
-    :ktime_ns, :pid, :tid, :event_name, :payload, :dropped_since_last,
+    :ktime_ns, :pid, :tid, :uid, :gid,
+    :trace_hi, :trace_lo, :span_id, :parent_span_id, :comm,
+    :event_name, :payload, :dropped_since_last,
     keyword_init: true
   )
 
@@ -16,12 +18,14 @@ module Vivarium
     class FormatError < StandardError; end
 
     FORMAT = "vivarium-raw"
-    VERSION = 1
-    PACK_FMT = "Q<L<L<a16a256Q<" # struct event_t (296B)
+    VERSION = 2
+    PACK_FMT = "Q<L<L<L<L<Q<Q<Q<Q<a16a16a256Q<" # struct event_t (352B)
 
     def self.pack_record(ev)
       [
-        ev.ktime_ns, ev.pid, ev.tid,
+        ev.ktime_ns, ev.pid, ev.tid, ev.uid.to_i, ev.gid.to_i,
+        ev.trace_hi.to_i, ev.trace_lo.to_i, ev.span_id.to_i, ev.parent_span_id.to_i,
+        ev.comm.to_s.b.ljust(EVENT_COMM_SIZE, "\x00")[0, EVENT_COMM_SIZE],
         ev.event_name.to_s.b.ljust(EVENT_NAME_SIZE, "\x00")[0, EVENT_NAME_SIZE],
         ev.payload.to_s.b.ljust(EVENT_PAYLOAD_SIZE, "\x00")[0, EVENT_PAYLOAD_SIZE],
         ev.dropped_since_last
@@ -33,12 +37,19 @@ module Vivarium
       bytes = bytes.ljust(EVENT_STRUCT_SIZE, "\x00") if bytes.bytesize < EVENT_STRUCT_SIZE
 
       RawEvent.new(
-        ktime_ns:           bytes[EVENT_TS_OFFSET,      EVENT_TS_SIZE].unpack1("Q<"),
-        pid:                bytes[EVENT_PID_OFFSET,      4].unpack1("L<"),
-        tid:                bytes[EVENT_TID_OFFSET,      4].unpack1("L<"),
+        ktime_ns:           bytes[EVENT_TS_OFFSET,         EVENT_TS_SIZE].unpack1("Q<"),
+        pid:                bytes[EVENT_PID_OFFSET,         4].unpack1("L<"),
+        tid:                bytes[EVENT_TID_OFFSET,         4].unpack1("L<"),
+        uid:                bytes[EVENT_UID_OFFSET,         4].unpack1("L<"),
+        gid:                bytes[EVENT_GID_OFFSET,         4].unpack1("L<"),
+        trace_hi:           bytes[EVENT_TRACE_HI_OFFSET,    8].unpack1("Q<"),
+        trace_lo:           bytes[EVENT_TRACE_LO_OFFSET,    8].unpack1("Q<"),
+        span_id:            bytes[EVENT_SPAN_OFFSET,        8].unpack1("Q<"),
+        parent_span_id:     bytes[EVENT_PARENT_SPAN_OFFSET, 8].unpack1("Q<"),
+        comm:               Vivarium.c_string(bytes[EVENT_COMM_OFFSET, EVENT_COMM_SIZE]),
         event_name:         Vivarium.c_string(bytes[EVENT_NAME_OFFSET, EVENT_NAME_SIZE]),
-        payload:            bytes[EVENT_PAYLOAD_OFFSET,  EVENT_PAYLOAD_SIZE].to_s.b,
-        dropped_since_last: bytes[EVENT_DROPPED_OFFSET,  8].unpack1("Q<")
+        payload:            bytes[EVENT_PAYLOAD_OFFSET,     EVENT_PAYLOAD_SIZE].to_s.b,
+        dropped_since_last: bytes[EVENT_DROPPED_OFFSET,     8].unpack1("Q<")
       )
     end
 
@@ -70,6 +81,13 @@ module Vivarium
         raise FormatError, "format=#{meta[:format].inspect} (expected #{FORMAT.inspect})"
       end
 
+      size = meta[:event_struct_size]
+      if size && size != EVENT_STRUCT_SIZE
+        raise FormatError,
+              "event_struct_size=#{size} (expected #{EVENT_STRUCT_SIZE}); " \
+              "incompatible capture (likely an older vivarium-raw v1 file)"
+      end
+
       events = []
       while (rec = io.read(EVENT_STRUCT_SIZE))
         break if rec.bytesize < EVENT_STRUCT_SIZE

data/lib/vivarium/tree_renderer.rb

@@ -24,6 +24,11 @@ module Vivarium
     UPROBE_EVENT_NAMES = %w[ssl_write].to_set.freeze
     DL_EVENT_NAMES = %w[dlopen mmap_exec].to_set.freeze
 
+    # Events whose traced value repeats heavily (same file/lib/env key opened over
+    # and over). With dedup_values on, each (event_name, value) pair is rendered
+    # only on its first occurrence in the session; later repeats are suppressed.
+    DEDUP_EVENT_NAMES = %w[path_open mmap_exec dlopen env_caccess].to_set.freeze
+
     SYNTHETIC_SPAN_NAME = "<no-span>"
     UNRESOLVED_METHOD_PREFIX = "<method_id="
 
@@ -67,6 +72,7 @@ module Vivarium
 
       @pid_comm = { observer_pid => "ruby" }
       @pid_parent = {}
+      @dedup_seen = Set.new
     end
 
     def render
@@ -366,6 +372,8 @@ module Vivarium
           next unless event_visible?(ev, span)
         end
 
+        next if dedup_suppressed?(ev, target_text)
+
         if ev.event_name == FORK_EVENT_NAME
           child_pid = read_proc_fork_child_pid(ev.payload)
           child_node = ProcNode.new(
@@ -505,6 +513,17 @@ module Vivarium
       @display_filter.allow_span_name?(span_display_name(span))
     end
 
+    # True when dedup_values is on and this (event_name, value) pair was already
+    # rendered earlier in the session. Only visible events reach this point, so a
+    # suppressed-by-filter event never consumes the "first occurrence" slot.
+    def dedup_suppressed?(ev, target_text)
+      return false unless @display_filter.dedup_values
+      return false unless DEDUP_EVENT_NAMES.include?(ev.event_name)
+
+      value = target_text || render_target(ev)
+      !@dedup_seen.add?([ev.event_name, value])
+    end
+
     def event_visible?(ev, span, target_text = nil)
       @display_filter.allow_event?(
         event_name: ev.event_name,

data/lib/vivarium/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vivarium
-  VERSION = "0.5.2"
+  VERSION = "0.6.0"
 end

data/lib/vivarium.rb

@@ -6,6 +6,7 @@ require "net/http"
 require "optparse"
 require "pathname"
 require "rbbcc"
+require "securerandom"
 require "set"
 require "socket"
 if defined?(Ruby) && defined?(Ruby::Box) && Ruby::Box.enabled?
@@ -31,15 +32,23 @@ module Vivarium
   EVENT_NAME_SIZE = 16
   EVENT_PAYLOAD_SIZE = 256
   EVENT_TS_SIZE = 8
+  EVENT_COMM_SIZE = 16
   PROC_EXEC_SLOT_SIZE = 64
   PROC_EXEC_SLOT_COUNT = 4
-  EVENT_STRUCT_SIZE = 296
+  EVENT_STRUCT_SIZE = 352
   EVENT_TS_OFFSET = 0
   EVENT_PID_OFFSET = 8
   EVENT_TID_OFFSET = 12
-  EVENT_NAME_OFFSET = 16
-  EVENT_PAYLOAD_OFFSET = 32
-  EVENT_DROPPED_OFFSET = 288
+  EVENT_UID_OFFSET = 16
+  EVENT_GID_OFFSET = 20
+  EVENT_TRACE_HI_OFFSET = 24
+  EVENT_TRACE_LO_OFFSET = 32
+  EVENT_SPAN_OFFSET = 40
+  EVENT_PARENT_SPAN_OFFSET = 48
+  EVENT_COMM_OFFSET = 56
+  EVENT_NAME_OFFSET = 72
+  EVENT_PAYLOAD_OFFSET = 88
+  EVENT_DROPPED_OFFSET = 344
   EVENTS_RINGBUF_PAGES = 256
 
   SPAN_METHOD_SIZE     = 128
@@ -236,6 +245,36 @@ module Vivarium
     EVENT_SEVERITY_HIGH.include?(event_name.to_s) ? "high" : "medium"
   end
 
+  U64_MASK = 0xFFFFFFFFFFFFFFFF
+
+  # Deterministic 64-bit span id for a method-call span, derived by folding the
+  # trace id, tid, and span-start ktime through splitmix64. Non-zero, unique
+  # within a trace, and stable across re-runs. Shared by the report --dump-otel
+  # view and the OTLP exporter so both assign identical method span ids.
+  def self.synth_span_id(trace_hi, trace_lo, tid, start_ktime)
+    seed = mix64(trace_hi)
+    seed = mix64(seed ^ (trace_lo & U64_MASK))
+    seed = mix64(seed ^ (tid.to_i & U64_MASK))
+    seed = mix64(seed ^ (start_ktime.to_i & U64_MASK))
+    seed.zero? ? 1 : seed
+  end
+
+  def self.mix64(value)
+    x = (value.to_i + 0x9E3779B97F4A7C15) & U64_MASK
+    x = ((x ^ (x >> 30)) * 0xBF58476D1CE4E5B9) & U64_MASK
+    x = ((x ^ (x >> 27)) * 0x94D049BB133111EB) & U64_MASK
+    (x ^ (x >> 31)) & U64_MASK
+  end
+
+  # Deterministic 128-bit trace id (returned as [hi, lo], both non-zero) for a
+  # top-level method span in the streaming exporter, where each top span starts
+  # its own OTel trace. Folds the BPF trace id, tid, and span-start ktime.
+  def self.synth_trace_id(seed_hi, seed_lo, tid, start_ktime)
+    hi = synth_span_id(seed_hi, seed_lo, tid, start_ktime)
+    lo = synth_span_id(seed_lo ^ U64_MASK, seed_hi, tid, start_ktime)
+    [hi, lo]
+  end
+
   def self.decode_dns_qname(raw_payload)
     bytes = raw_payload.to_s.b.bytes
     labels = []
@@ -711,21 +750,47 @@ module Vivarium
         u16 transport_header;
       };
 
+      // trace_id is a 128-bit value carried as two u64 halves (hi/lo). They are
+      // kept as flat scalar fields (not a nested struct) because rbbcc/Fiddle's
+      // CParser cannot decode nested-struct members of a BPF map value type.
       struct event_t {
         u64 ktime_ns;
         u32 pid;
         u32 tid;
+        u32 uid;
+        u32 gid;
+        u64 trace_id_hi;
+        u64 trace_id_lo;
+        u64 span_id;
+        u64 parent_span_id;
+        char comm[#{EVENT_COMM_SIZE}];
         char event_name[16];
         char payload[#{EVENT_PAYLOAD_SIZE}];
         u64 dropped_since_last;
       };
 
+      // Per-thread OpenTelemetry context. trace_id (hi/lo) is issued by userspace
+      // at target registration and inherited by spawned children; span_id is
+      // re-issued per tid (root in userspace, children at fork).
+      struct otel_ctx_t {
+        u64 trace_id_hi;
+        u64 trace_id_lo;
+        u64 span_id;
+        u64 parent_span_id;
+      };
+
       BPF_HASH(config_root_targets, u32, u8, 1024);
       BPF_HASH(config_spawned_targets, u32, u8, 8192);
       BPF_HASH(dns_connected_tids, u32, u8, 8192);
+      BPF_HASH(otel_ctx, u32, struct otel_ctx_t, 8192);
       BPF_RINGBUF_OUTPUT(events, #{EVENTS_RINGBUF_PAGES});
       BPF_ARRAY(drop_counter, u64, 1);
 
+      static __always_inline u64 rand_span_id()
+      {
+        return ((u64)bpf_get_prandom_u32() << 32) | (u64)bpf_get_prandom_u32();
+      }
+
       static __always_inline int target_enabled(u32 pid, u32 tid)
       {
         u8 *enabled_root = config_root_targets.lookup(&pid);
@@ -783,6 +848,20 @@ module Vivarium
         ev->tid = (u32)bpf_get_current_pid_tgid();
         ev->dropped_since_last = 0;
 
+        u64 uid_gid = bpf_get_current_uid_gid();
+        ev->uid = (u32)uid_gid;
+        ev->gid = (u32)(uid_gid >> 32);
+        bpf_get_current_comm(&ev->comm, sizeof(ev->comm));
+
+        u32 ctid = (u32)bpf_get_current_pid_tgid();
+        struct otel_ctx_t *octx = otel_ctx.lookup(&ctid);
+        if (octx) {
+          ev->trace_id_hi = octx->trace_id_hi;
+          ev->trace_id_lo = octx->trace_id_lo;
+          ev->span_id = octx->span_id;
+          ev->parent_span_id = octx->parent_span_id;
+        }
+
         cnt = drop_counter.lookup(&key);
         if (cnt && *cnt > 0) {
           ev->dropped_since_last = __sync_lock_test_and_set(cnt, 0);
@@ -895,11 +974,26 @@ module Vivarium
 
         if (is_target) {
           u64 pid_tgid = bpf_get_current_pid_tgid();
+
+          // Re-issue a fresh span_id for the child, inheriting the parent's
+          // trace_id and linking the child's parent_span_id to the parent span.
+          u32 parent_tid = (u32)pid_tgid;
+          struct otel_ctx_t *pctx = otel_ctx.lookup(&parent_tid);
+          struct otel_ctx_t cctx = {};
+          u64 child_span = rand_span_id();
+          if (pctx) {
+            cctx.trace_id_hi = pctx->trace_id_hi;
+            cctx.trace_id_lo = pctx->trace_id_lo;
+            cctx.parent_span_id = pctx->span_id;
+          }
+          cctx.span_id = child_span;
+          otel_ctx.update(&child, &cctx);
+
           struct event_t ev = {};
           ev.pid = pid_tgid >> 32;
           __builtin_memcpy(ev.event_name, "proc_fork", 10);
           __builtin_memcpy(&ev.payload[0], &child, sizeof(child));
-          __builtin_memcpy(&ev.payload[4], &child, sizeof(child));
+          __builtin_memcpy(&ev.payload[8], &child_span, sizeof(child_span));
           submit_event(&ev);
         }
 
@@ -908,9 +1002,18 @@ module Vivarium
 
       TRACEPOINT_PROBE(sched, sched_process_exit)
       {
-        u32 tid = (u32)bpf_get_current_pid_tgid();
+        u64 pid_tgid = bpf_get_current_pid_tgid();
+        u32 pid = pid_tgid >> 32;
+        u32 tid = (u32)pid_tgid;
+        if (target_enabled(pid, tid)) {
+          struct event_t ev = {};
+          ev.pid = pid;
+          __builtin_memcpy(ev.event_name, "proc_exit", 10);
+          submit_event(&ev);
+        }
         config_spawned_targets.delete(&tid);
         dns_connected_tids.delete(&tid);
+        otel_ctx.delete(&tid);
         return 0;
       }
 
@@ -1779,16 +1882,18 @@ module Vivarium
 
       config_root_targets = bpf["config_root_targets"]
       config_spawned_targets = bpf["config_spawned_targets"]
+      otel_ctx = bpf["otel_ctx"]
       events_ringbuf = bpf["events"]
 
       config_spawned_targets.clear
+      otel_ctx.clear
 
       pin_map(config_root_targets, File.join(@pin_dir, "config_root_targets"))
       pin_map(config_spawned_targets, File.join(@pin_dir, "config_spawned_targets"))
       pin_map(events_ringbuf, File.join(@pin_dir, "events"))
 
       event_log = EventLog.new
-      registry = Registry.new(config_root_targets, config_spawned_targets)
+      registry = Registry.new(config_root_targets, config_spawned_targets, otel_ctx)
       start_ringbuf_poller(bpf, events_ringbuf, event_log)
 
       @api_server = ApiServer.new(
@@ -2180,15 +2285,19 @@ module Vivarium
     end
   end
 
-  def self.observe(socket_path: self.socket_path, dest: $stdout, filter: nil, save_raw: nil, &block)
+  def self.observe(socket_path: self.socket_path, dest: $stdout, filter: nil, save_raw: nil,
+                   otel_out: nil, otel_endpoint: nil, &block)
     if block_given?
-      return scoped_observe(socket_path: socket_path, dest: dest, filter: filter, save_raw: save_raw, &block)
+      return scoped_observe(socket_path: socket_path, dest: dest, filter: filter,
+                            save_raw: save_raw, otel_out: otel_out, otel_endpoint: otel_endpoint, &block)
     end
 
-    top_observe(socket_path: socket_path, dest: dest, filter: filter, save_raw: save_raw)
+    top_observe(socket_path: socket_path, dest: dest, filter: filter,
+                save_raw: save_raw, otel_out: otel_out, otel_endpoint: otel_endpoint)
   end
 
-  def self.top_observe(socket_path: self.socket_path, dest: $stdout, filter: nil, save_raw: nil)
+  def self.top_observe(socket_path: self.socket_path, dest: $stdout, filter: nil, save_raw: nil,
+                       otel_out: nil, otel_endpoint: nil)
     client = DaemonClient.new(socket_path: socket_path)
     pid = Process.pid
     main_tid = gettid
@@ -2199,7 +2308,9 @@ module Vivarium
       main_tid: main_tid,
       filter: filter,
       dest: dest,
-      save_raw: save_raw
+      save_raw: save_raw,
+      otel_out: otel_out,
+      otel_endpoint: otel_endpoint
     )
     correlator.start
     client.register(pid)
@@ -2214,7 +2325,8 @@ module Vivarium
     session
   end
 
-  def self.scoped_observe(socket_path: self.socket_path, dest:, filter: nil, save_raw: nil)
+  def self.scoped_observe(socket_path: self.socket_path, dest:, filter: nil, save_raw: nil,
+                          otel_out: nil, otel_endpoint: nil)
     client = DaemonClient.new(socket_path: socket_path)
     pid = Process.pid
     main_tid = gettid
@@ -2225,7 +2337,9 @@ module Vivarium
       main_tid: main_tid,
       filter: filter,
       dest: dest,
-      save_raw: save_raw
+      save_raw: save_raw,
+      otel_out: otel_out,
+      otel_endpoint: otel_endpoint
     )
     correlator.start
     client.register(pid)
@@ -2360,6 +2474,8 @@ end
 require_relative "vivarium/daemon_client"
 require_relative "vivarium/api_server"
 require_relative "vivarium/raw_store"
+require_relative "vivarium/otel_exporter"
+require_relative "vivarium/otel_stream"
 require_relative "vivarium/correlator"
 require_relative "vivarium/display_filter"
 require_relative "vivarium/tree_renderer"