vivarium 0.0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a7b939d2599fb7e7115ef398bcc83830be78d3b8576a05e5c6d7c1adec4003a1
-  data.tar.gz: 39a158bacb6b706a4b77144b47554f027822263662818a9175b1828109538406
+  metadata.gz: 1bc495e3c2b8f9d99a2b1c84706b5e00913cc08831b3103b0460db0d4027c77b
+  data.tar.gz: 1ee086243fdc153da5bac6a2bec8d0afe8e644ceac2fef2308b7723b89533338
 SHA512:
-  metadata.gz: 6b17e753a8ccaf72cb969e5fdb8b005d2e31fc2aa10f1d33984d93ba2b7c6a1e2d8e28560b39a4cb9f2860f453a498c3c9c974db9b4b316b4162b78b3e3841e6
-  data.tar.gz: 0bcd982884661f9e93ba27c3dc79cb1a952de8c68f715572022917cad708a771e4f4017af4bf12257f7f71ed38119535d14a5745d06666c580d6486c1d15b820
+  metadata.gz: a203dd1cbaeff9fe0f4bf464a01ff49f131a104ca9e1eb14b6c08dd4371f49acf31c493119eee70114cafacbc6aa8a42976a9e6c230b1d828fe442e738c57c54
+  data.tar.gz: a31a64e20f2f7d0956626658b5d6f91e0cd5333f97854ad3388c41cc0d3b0b8d233850c3bdaaf60663c9152d6cf35e4034355e37c50fc913687ad05295dffaba

data/README.md

@@ -1,5 +1,9 @@
 # Vivarium
 
+[![Gem Version](https://badge.fury.io/rb/vivarium.svg)](https://rubygems.org/gems/vivarium)
+
+RubyGems: https://rubygems.org/gems/vivarium
+
 Vivarium is an observation and sandbox helper for Ruby.
 
 It combines:
@@ -15,11 +19,13 @@ Implemented in this repository:
 
 - BPF LSM hook on `file_open`
 - Shared pinned maps on bpffs
-	- `config_targets` (PID -> 0/1)
+	- `config_root_targets` (root PID -> 0/1)
+	- `config_spawned_targets` (spawned TID -> 0/1)
 	- `event_invoked` (array length 64 with `event_t` records)
 	- `event_write_pos` (cursor for appending into `event_invoked`)
 - Ruby API `Vivarium.observe do ... end`
-	- Registers current PID to `config_targets`
+	- Registers current PID to `config_root_targets`
+	- eBPF tracks spawned descendants into `config_spawned_targets` via `sched_process_fork`
 	- On each `:return` / `:c_return`, drains `event_invoked`
 	- Prints stack trace + events
 	- Clears event slots and cursor
@@ -71,19 +77,34 @@ sudo bundle exec vivariumd
 require "vivarium"
 
 Vivarium.observe do
-	File.read("/etc/hosts")
+  File.read("/etc/passwd")
 end
 ```
 
+You can also start top-level observation without a block (it keeps observing until process exit):
+
+```ruby
+require "vivarium"
+
+observer = Vivarium.top_observe
+# or: Vivarium.observe
+# do anything ...
+observer.stop
+```
+
+By default, Vivarium excludes its own internal frames from stack output. Set `VIVARIUM_FILTER_INTERNAL_FRAMES=0` to disable this filter.
+
 You can override pin directory via `VIVARIUM_BPF_PIN_DIR` on both sides:
 
 ```bash
 VIVARIUM_BPF_PIN_DIR=/sys/fs/bpf/vivarium bundle exec vivariumd
 ```
 
+Use `Vivarium.bpf_pin_dir = "/sys/fs/bpf/..."` in Ruby code to set it programmatically.
+
 ```ruby
-ENV["VIVARIUM_BPF_PIN_DIR"] = "/sys/fs/bpf/vivarium"
 require "vivarium"
+Vivarium.bpf_pin_dir = "/sys/fs/bpf/vivarium"
 ```
 
 ## Development
@@ -108,6 +129,7 @@ bundle exec vivariumd --pin-dir /sys/fs/bpf/vivarium
 - Current output format is textual and intended for iteration.
 - `vivariumd` resolves `struct file::f_path` offset from `/sys/kernel/btf/vmlinux` at startup.
 - You can override offset manually with `VIVARIUM_FILE_F_PATH_OFFSET` if auto-detection fails.
+- `vivariumd` also prints `bpf_trace_printk` lines (`vivarium: pid=... path=...`) to its own logs.
 
 ## Contributing
 

data/lib/vivarium/logger.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Vivarium
+  class Logger
+    FORMATS = %i[human json].freeze
+
+    # dest: IO object or file path string
+    # format: :human or :json
+    # TODO: support flushing in bulk for performance
+    def initialize(dest: $stdout, format: :human)
+      @format = format.to_sym
+      raise ArgumentError, "unknown format: #{@format}; choose from #{FORMATS.join(', ')}" unless FORMATS.include?(@format)
+
+      if dest.is_a?(String)
+        @io = File.open(dest, "a")
+        @owned = true
+      else
+        @io = dest
+        @owned = false
+      end
+    end
+
+    def log(events, tp, stack)
+      case @format
+      when :human then log_human(events, tp, stack)
+      when :json  then log_json(events, tp, stack)
+      end
+      @io.flush
+    end
+
+    def info(message)
+      @io.puts("[vivarium] #{message}")
+      @io.flush
+    end
+
+    def close
+      @io.close if @owned
+    end
+
+    private
+
+    def log_human(events, tp, stack)
+      @io.puts "[vivarium] #{events.size} event(s) at #{tp.defined_class}##{tp.method_id} (#{tp.event})"
+      @io.puts "  location: #{tp.path}:#{tp.lineno}"
+      events.each do |event|
+        @io.puts "  pid=#{event.pid} #{event.event_name} payload=#{event.payload.inspect}"
+      end
+      @io.puts "  stack:"
+      stack.each do |loc|
+        @io.puts "    #{loc.path}:#{loc.lineno}:in #{loc.base_label}"
+      end
+    end
+
+    def log_json(events, tp, stack)
+      entry = {
+        at:     "#{tp.defined_class}##{tp.method_id}",
+        event:  tp.event.to_s,
+        path:   tp.path,
+        lineno: tp.lineno,
+        events: events.map { |e| { pid: e.pid, event_name: e.event_name, payload: e.payload } },
+        stack:  stack.map { |loc| "#{loc.path}:#{loc.lineno}:in #{loc.base_label}" }
+      }
+      @io.puts JSON.generate(entry)
+    end
+  end
+end

data/lib/vivarium/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vivarium
-  VERSION = "0.0.1"
+  VERSION = "0.1.1"
 end

data/lib/vivarium.rb

@@ -6,20 +6,33 @@ require "optparse"
 require "pathname"
 require "rbbcc"
 require_relative "vivarium/version"
+require_relative "vivarium/logger"
 
 module Vivarium
   class Error < StandardError; end
 
   PIN_DIR = ENV.fetch("VIVARIUM_BPF_PIN_DIR", "/sys/fs/bpf/vivarium")
-  CONFIG_TARGETS_PIN = File.join(PIN_DIR, "config_targets")
+  CONFIG_ROOT_TARGETS_PIN = File.join(PIN_DIR, "config_root_targets")
+  CONFIG_SPAWNED_TARGETS_PIN = File.join(PIN_DIR, "config_spawned_targets")
+  CONFIG_TARGETS_PIN = CONFIG_ROOT_TARGETS_PIN
   EVENT_INVOKED_PIN = File.join(PIN_DIR, "event_invoked")
   EVENT_WRITE_POS_PIN = File.join(PIN_DIR, "event_write_pos")
 
   EVENT_NAME_SIZE = 16
-  EVENT_PAYLOAD_SIZE = 64
+  EVENT_PAYLOAD_SIZE = 256
   EVENT_STRUCT_SIZE = 4 + EVENT_NAME_SIZE + EVENT_PAYLOAD_SIZE
   EVENT_CAPACITY = 64
 
+  @bpf_pin_dir = PIN_DIR
+
+  class << self
+    attr_writer :bpf_pin_dir
+
+    def bpf_pin_dir
+      @bpf_pin_dir || PIN_DIR
+    end
+  end
+
   Event = Struct.new(:pid, :event_name, :payload, keyword_init: true) do
     def empty?
       pid.to_i.zero? && event_name.to_s.empty? && payload.to_s.empty?
@@ -30,18 +43,33 @@ module Vivarium
       bytes = bytes.ljust(EVENT_STRUCT_SIZE, "\x00")
 
       pid = bytes[0, 4].unpack1("L<")
-      event_name = bytes[4, EVENT_NAME_SIZE].delete("\x00")
-      payload = bytes[4 + EVENT_NAME_SIZE, EVENT_PAYLOAD_SIZE].delete("\x00")
+      event_name = c_string(bytes[4, EVENT_NAME_SIZE])
+      payload = c_string(bytes[4 + EVENT_NAME_SIZE, EVENT_PAYLOAD_SIZE])
 
       new(pid: pid, event_name: event_name, payload: payload)
     end
+
+    def self.c_string(bytes)
+      str = bytes.to_s.b
+      nul = str.index("\x00")
+      return str if nul.nil?
+
+      str[0, nul]
+    end
   end
 
   class MapStore
-    def initialize(pin_dir: PIN_DIR)
+    def initialize(pin_dir: Vivarium.bpf_pin_dir)
       @pin_dir = pin_dir
-      @config_targets = RbBCC::HashTable.from_pin(
-        File.join(@pin_dir, "config_targets"),
+      @config_root_targets = RbBCC::HashTable.from_pin(
+        File.join(@pin_dir, "config_root_targets"),
+        "unsigned int",
+        "unsigned char",
+        keysize: 4,
+        leafsize: 1
+      )
+      @config_spawned_targets = RbBCC::HashTable.from_pin(
+        File.join(@pin_dir, "config_spawned_targets"),
         "unsigned int",
         "unsigned char",
         keysize: 4,
@@ -66,11 +94,12 @@ module Vivarium
     end
 
     def register_pid(pid)
-      @config_targets[pid] = 1
+      @config_root_targets[pid] = 1
     end
 
     def unregister_pid(pid)
-      @config_targets.delete(pid)
+      @config_root_targets.delete(pid)
+      @config_spawned_targets.clear
     rescue KeyError
       nil
     end
@@ -115,26 +144,63 @@ module Vivarium
       struct event_t {
         u32 pid;
         char event_name[16];
-        char payload[64];
+        char payload[#{EVENT_PAYLOAD_SIZE}];
       };
 
-      BPF_HASH(config_targets, u32, u32, 1024);
-      BPF_ARRAY(event_invoked, struct event_t, 64);
+      BPF_HASH(config_root_targets, u32, u8, 1024);
+      BPF_HASH(config_spawned_targets, u32, u8, 8192);
+      BPF_ARRAY(event_invoked, struct event_t, 1024);
       BPF_ARRAY(event_write_pos, u32, 1);
 
-      static __always_inline int target_enabled(u32 pid)
+      static __always_inline int target_enabled(u32 pid, u32 tid)
       {
-        u32 *enabled = config_targets.lookup(&pid);
-        if (!enabled) {
+        u8 *enabled_root = config_root_targets.lookup(&pid);
+        if (enabled_root && *enabled_root == 1) {
+          return 1;
+        }
+
+        u8 *enabled_spawned = config_spawned_targets.lookup(&tid);
+        if (enabled_spawned && *enabled_spawned == 1) {
+          return 1;
+        }
+
+        return 0;
+      }
+
+      TRACEPOINT_PROBE(sched, sched_process_fork)
+      {
+        u32 parent = args->parent_pid;
+        u32 child = args->child_pid;
+        u8 one = 1;
+
+        u8 *enabled_root = config_root_targets.lookup(&parent);
+        if (enabled_root && *enabled_root == 1) {
+          config_spawned_targets.update(&child, &one);
           return 0;
         }
-        return *enabled == 1;
+
+        u8 *enabled_spawned = config_spawned_targets.lookup(&parent);
+        if (enabled_spawned && *enabled_spawned == 1) {
+          config_spawned_targets.update(&child, &one);
+        }
+
+        return 0;
+      }
+
+      TRACEPOINT_PROBE(sched, sched_process_exit)
+      {
+        u32 tid = (u32)bpf_get_current_pid_tgid();
+        config_spawned_targets.delete(&tid);
+        return 0;
       }
 
       LSM_PROBE(file_open, struct file *file)
       {
-        u32 pid = bpf_get_current_pid_tgid() >> 32;
-        if (!target_enabled(pid)) {
+        u64 pid_tgid = bpf_get_current_pid_tgid();
+        u32 pid = pid_tgid >> 32;
+        u32 tid = (u32)pid_tgid;
+        bpf_trace_printk("vivarium: invoked pid=%d\\n", pid);
+        if (!target_enabled(pid, tid)) {
           return 0;
         }
 
@@ -144,7 +210,7 @@ module Vivarium
           return 0;
         }
 
-        u32 idx = *write_pos & 63;
+        u32 idx = *write_pos & 1023;
         __sync_fetch_and_add(write_pos, 1);
         struct event_t ev = {};
         int path_ret;
@@ -153,16 +219,20 @@ module Vivarium
 
         path_ret = bpf_d_path(&file->f_path, ev.payload, sizeof(ev.payload));
         if (path_ret < 0) {
-          __builtin_memcpy(ev.payload, "<path_error>", 13);
+          if (ev.payload[0] == 0) {
+            __builtin_memcpy(ev.payload, "<path_error>", 13);
+            bpf_trace_printk("vivarium: failed to obtain full path. pid=%d path=%s\\n", pid, ev.payload);
+          }
         }
 
+        bpf_trace_printk("vivarium: pid=%d path=%s\\n", pid, ev.payload);
         event_invoked.update(&idx, &ev);
 
         return 0;
       }
     CLANG
 
-    def initialize(pin_dir: PIN_DIR)
+    def initialize(pin_dir: Vivarium.bpf_pin_dir)
       @pin_dir = pin_dir
     end
 
@@ -174,27 +244,37 @@ module Vivarium
       program = BPF_PROGRAM_TEMPLATE.gsub("__VIVARIUM_F_PATH_OFFSET__", f_path_offset.to_s)
 
       bpf = RbBCC::BCC.new(text: program)
+      kprint_thread = start_kprint_logger(bpf)
 
-      config_targets = bpf["config_targets"]
+      config_root_targets = bpf["config_root_targets"]
+      config_spawned_targets = bpf["config_spawned_targets"]
       event_invoked = bpf["event_invoked"]
       event_write_pos = bpf["event_write_pos"]
 
       clear_event_slots(event_invoked)
       event_write_pos[0] = 0
+      config_spawned_targets.clear
 
-      pin_map(config_targets, File.join(@pin_dir, "config_targets"))
+      pin_map(config_root_targets, File.join(@pin_dir, "config_root_targets"))
+      pin_map(config_spawned_targets, File.join(@pin_dir, "config_spawned_targets"))
       pin_map(event_invoked, File.join(@pin_dir, "event_invoked"))
       pin_map(event_write_pos, File.join(@pin_dir, "event_write_pos"))
 
       puts "[vivariumd] started"
       puts "[vivariumd] pinned maps in #{@pin_dir}"
       puts "[vivariumd] watching LSM file_open (f_path offset=#{f_path_offset})"
+      puts "[vivariumd] kprint logger enabled"
 
       loop do
         sleep 1
       end
     rescue Interrupt
       puts "\n[vivariumd] stopping"
+    ensure
+      if kprint_thread
+        kprint_thread.kill
+        kprint_thread.join(0.2)
+      end
     end
 
     private
@@ -218,6 +298,26 @@ module Vivarium
       end
     end
 
+    def start_kprint_logger(bpf)
+      Thread.new do
+        begin
+          bpf.trace_fields do |_task, pid, _cpu, _flags, ts, msg|
+            line = msg.to_s.strip
+            next unless line.start_with?("vivarium:")
+
+            puts "[vivariumd:kprint #{ts} pid=#{pid}] #{line}"
+          end
+        rescue IOError, Errno::EINTR
+          nil
+        rescue StandardError => e
+          warn "[vivariumd] kprint stream stopped: #{e.class}: #{e.message}"
+        end
+      end
+    rescue StandardError => e
+      warn "[vivariumd] failed to start kprint logger: #{e.class}: #{e.message}"
+      nil
+    end
+
     def detect_f_path_offset
       env_offset = ENV["VIVARIUM_FILE_F_PATH_OFFSET"]
       return Integer(env_offset, 10) if env_offset
@@ -279,36 +379,80 @@ module Vivarium
     end
   end
 
-  def self.observe(pin_dir: PIN_DIR, out: $stdout)
-    raise ArgumentError, "block is required" unless block_given?
+  class ObservationSession
+    def initialize(store:, pid:, tracer:)
+      @store = store
+      @pid = pid
+      @tracer = tracer
+      @stopped = false
+    end
+
+    def stop
+      return if @stopped
+
+      @tracer.disable
+      @store.unregister_pid(@pid)
+      @stopped = true
+    end
+  end
 
+  def self.observe(pin_dir: bpf_pin_dir, logger: nil, dest: $stdout, format: :human)
+    return scoped_observe(pin_dir: pin_dir, logger: logger, dest: dest, format: format) { yield } if block_given?
+
+    top_observe(pin_dir: pin_dir, logger: logger, dest: dest, format: format)
+  end
+
+  def self.top_observe(pin_dir: bpf_pin_dir, logger: nil, dest: $stdout, format: :human)
+    logger ||= Logger.new(dest: dest, format: format)
     store = MapStore.new(pin_dir: pin_dir)
     pid = Process.pid
     store.register_pid(pid)
+    logger.info("top-level observing with pid=#{pid}")
 
-    tracer = TracePoint.new(:return, :c_return) do |tp|
-      events = store.drain_events
-      next if events.empty?
+    tracer = build_observe_tracepoint(store, logger)
+    tracer.enable
 
-      out.puts "[vivarium] #{events.size} event(s) at #{tp.defined_class}##{tp.method_id} (#{tp.event})"
-      events.each do |event|
-        out.puts "  pid=#{event.pid} #{event.event_name} payload=#{event.payload.inspect}"
-      end
-      out.puts "  stack:"
-      caller_locations(0, 12).each do |loc|
-        out.puts "    #{loc.path}:#{loc.lineno}:in #{loc.base_label}"
-      end
-    end
+    session = ObservationSession.new(store: store, pid: pid, tracer: tracer)
+    at_exit { session.stop }
+    session
+  end
+
+  def self.scoped_observe(pin_dir:, logger:, dest:, format:)
+    logger ||= Logger.new(dest: dest, format: format)
+    store = MapStore.new(pin_dir: pin_dir)
+    pid = Process.pid
+    store.register_pid(pid)
+    logger.info("scoped observing with pid=#{pid}")
 
+    tracer = build_observe_tracepoint(store, logger)
     tracer.enable
+
     yield
   ensure
     tracer&.disable
     store&.unregister_pid(pid)
   end
 
+  def self.build_observe_tracepoint(store, logger)
+    TracePoint.new(:return, :c_return) do |tp|
+      events = store.drain_events
+      next if events.empty?
+
+      stack = caller_locations(2, 16)
+      stack = stack.reject { |loc| loc.path.to_s.include?("vivarium") } if filter_internal_frames?
+      logger.log(events, tp, stack)
+    end
+  end
+
+  def self.filter_internal_frames?
+    value = ENV["VIVARIUM_FILTER_INTERNAL_FRAMES"]
+    return true if value.nil?
+
+    !%w[0 false off no].include?(value.strip.downcase)
+  end
+
   def self.run_daemon!(argv = ARGV)
-    options = { pin_dir: PIN_DIR }
+    options = { pin_dir: bpf_pin_dir }
     OptionParser.new do |opts|
       opts.banner = "Usage: vivariumd [--pin-dir PATH]"
       opts.on("--pin-dir PATH", "Pinned map directory") { |v| options[:pin_dir] = v }

data/sig/vivarium.rbs

@@ -21,12 +21,18 @@ module Vivarium
   end
 
   class Daemon
-    BPF_PROGRAM: String
+    BPF_PROGRAM_TEMPLATE: String
     def initialize: (?pin_dir: String pin_dir) -> void
     def run: () -> void
   end
 
+  class ObservationSession
+    def stop: () -> void
+  end
+
   PIN_DIR: String
+  CONFIG_ROOT_TARGETS_PIN: String
+  CONFIG_SPAWNED_TARGETS_PIN: String
   CONFIG_TARGETS_PIN: String
   EVENT_INVOKED_PIN: String
   EVENT_WRITE_POS_PIN: String
@@ -36,6 +42,10 @@ module Vivarium
   EVENT_STRUCT_SIZE: Integer
   EVENT_CAPACITY: Integer
 
-  def self.observe: (?pin_dir: String pin_dir, ?out: untyped out) { () -> untyped } -> untyped
+  def self.bpf_pin_dir: () -> String
+  def self.bpf_pin_dir=: (String dir) -> String
+  def self.observe: (?pin_dir: String pin_dir, ?logger: untyped logger, ?dest: untyped dest, ?format: Symbol format) { () -> untyped } -> untyped
+  def self.top_observe: (?pin_dir: String pin_dir, ?logger: untyped logger, ?dest: untyped dest, ?format: Symbol format) -> ObservationSession
+  def self.filter_internal_frames?: () -> bool
   def self.run_daemon!: (?Array[String] argv) -> void
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vivarium
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.1
 platform: ruby
 authors:
 - Uchio Kondo
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.2
+        version: 0.11.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.2
+        version: 0.11.3
 description: Vivarium visualizes low-level events such as file open paths and relates
   them to Ruby method boundaries by combining RbBCC (eBPF LSM) and TracePoint.
 email:
@@ -36,6 +36,7 @@ files:
 - Rakefile
 - exe/vivariumd
 - lib/vivarium.rb
+- lib/vivarium/logger.rb
 - lib/vivarium/version.rb
 - sig/vivarium.rbs
 homepage: https://github.com/udzura/vivarium