bpfql 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bd92fdeedd7501b48699058bf9a466287d511beb9ebebaa865dd53f171f4f0e7
+  data.tar.gz: 485ece1c48041ea20293838d9bfc6c5ee88fac2b6edfffccc6d696de6f425520
+SHA512:
+  metadata.gz: 852f267f0bc56295861c35eb7a18fce76c228f4a896589cf5f51ecf3e69e9b1d0ee19b78192e8e45a1a801b9b778e121af8dc43e9f0eabf20076efa935c28d8a
+  data.tar.gz: 1d013db8052f28175fde7f166e3afa350b74a840a2b4012ba4435d629140352a10a01da5c94787c5f1185a92a26805a91d867d416b2f7fd0c31a198bd2141165

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7.0
+before_install: gem install bundler -v 2.1.2

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in bpfql.gemspec
+gemspec
+
+gem "rake", "~> 12.0"
+gem "minitest", "~> 5.0"
+gem "pry"

data/Gemfile.lock

@@ -0,0 +1,29 @@
+PATH
+  remote: .
+  specs:
+    bpfql (0.1.0)
+      rbbcc
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.2)
+    method_source (0.9.2)
+    minitest (5.14.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (12.3.3)
+    rbbcc (0.3.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bpfql!
+  minitest (~> 5.0)
+  pry
+  rake (~> 12.0)
+
+BUNDLED WITH
+   2.0.2

data/README.md

@@ -0,0 +1,73 @@
+# BPFQL
+
+eBPF query runner. Choose a format in:
+
+* Ruby DSL
+* YAML
+* SQL-like query language (in the future)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'bpfql'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install bpfql
+
+## Usage
+
+```ruby
+BPFQL do
+  select "*"
+  from "tracepoint:random:urandom_read"
+  where "comm", is: "ruby"
+  _and  "pid", is: 12345
+end
+```
+
+```ruby
+BPFQL do
+  select "count()"
+  from "tracepoint:syscalls:sys_clone_enter"
+  group_by "comm"
+  interval "15s"
+end
+```
+
+### YAML format
+
+```yaml
+BPFQL:
+- select: count()
+  from: tracepoint:syscalls:sys_clone_enter
+  group_by: comm
+  stop_after: "30s"
+```
+
+```yaml
+BPFQL:
+- select: count()
+  from: tracepoint:syscalls:sys_clone_enter
+  where:
+    - comm is "ruby"
+    - pid is 12345
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/udzura/bpfql.
+

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "bpfql"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bpfql.gemspec

@@ -0,0 +1,24 @@
+require_relative 'lib/bpfql/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "bpfql"
+  spec.version       = Bpfql::VERSION
+  spec.authors       = ["Uchio Kondo"]
+  spec.email         = ["udzura@udzura.jp"]
+
+  spec.summary       = %q{eBPF query runner}
+  spec.description   = %q{eBPF query runner. Use Ruby DSL / yaml / or plane text}
+  spec.homepage      = "https://github.com/udzura/bpfql"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rbbcc"
+end

data/examples/random-simple.yml

@@ -0,0 +1,3 @@
+BPFQL:
+- select: [pid, ts, comm, got_bits]
+  from: tracepoint:random:urandom_read

data/exe/bpfql

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "bpfql/cli"
+
+Bpfql::Cli.run(ARGV)

data/lib/bpfql.rb

@@ -0,0 +1,6 @@
+require "bpfql/version"
+require "bpfql/query"
+require "bpfql/runner"
+
+module Bpfql
+end

data/lib/bpfql/cli.rb

@@ -0,0 +1,23 @@
+require 'bpfql'
+
+module Bpfql
+  module Cli
+    USAGE = <<~USAGE
+      #{File.basename $0} version #{Bpfql::VERSION}
+      usage:
+      \t#{File.basename $0} <YAML_FILE>
+    USAGE
+
+    def self.run(argv)
+      if argv.size != 1
+        $stderr.puts USAGE
+        exit 1
+      end
+
+      yaml_file = argv[0]
+      q = Bpfql::Query.parse_yaml(File.read yaml_file)
+      r = Bpfql::Runner.new(q[0])
+      r.run
+    end
+  end
+end

data/lib/bpfql/query.rb

@@ -0,0 +1,116 @@
+require 'yaml'
+
+module Bpfql
+  class Query
+    def self.parse_yaml(yaml)
+      data = YAML.load(yaml)
+      data['BPFQL'].map do |query|
+        Query.new do |builder|
+          builder.select = SelectOption.new(query['select'])
+          builder.from = ProbeOption.new(query['from'])
+          if query['where']
+            builder.where = FilterOption.parse(query['where'])
+          end
+          builder.group_by = query['group_by'] # Accepts nil
+          builder.interval = query['interval'] # Accepts nil
+          if query['stop_after']
+            builder.stop = StopOption.new(:after, query['stop_after'])
+          end
+        end
+      end
+    end
+
+    def initialize(&b)
+      b.call(self)
+    end
+    attr_accessor :select, :from, :where, :group_by, :interval, :stop
+    alias probe from
+
+    class SelectOption < Struct.new(:members, :type)
+      def initialize(query)
+        case query
+        when String, Symbol
+          if query.include? ','
+            self.members = query.split(',').map{|v| v.strip }
+          else
+            self.members = [query]
+          end
+        when Array
+          self.members = query
+        end
+      end
+    end
+
+    class ProbeOption < Struct.new(:type, :arg1, :arg2)
+      def initialize(probe)
+        # FIXME: complcated probe, e.g. uprobe and USDT has 4 sections
+        super(*probe.split(':'))
+      end
+
+      def tracepoint?
+        self.type == "tracepoint"
+      end
+
+      def kprobe?
+        self.type == "kprobe"
+      end
+
+      def uprobe?
+        self.type == "uprobe"
+      end
+
+      def usdt?
+        self.type == "usdt"
+      end
+
+      def to_s
+        [type, arg1, arg2].join ":"
+      end
+    end
+
+    class FilterOption < Struct.new(:lhs, :op, :rhs)
+      def self.parse(where)
+        where_list = Array(where)
+        where_list.map do |whr|
+          FilterOption.new(*whr)
+        end
+      end
+
+      def initialize(lhs, op=nil, rhs=nil)
+        if !rhs # args.size < 3
+          m = /^([^\s]+)\s+([^\s]+)\s+([^\s]+|"[^"]+"|'[^']+')$/.match(lhs)
+          unless m
+            raise "Failed to parse where clause: #{lhs}"
+          end
+          if m2 = /^['"](.+)['"]$/.match(m[3])
+            rhs = m2[1]
+          else
+            rhs = m[3]
+          end
+          super(m[1], m[2].to_sym, rhs)
+        else
+          super(lhs, op.to_sym, rhs)
+        end
+      end
+    end
+
+    class StopOption < Struct.new(:timing, :seconds)
+      def initialize(timing, secstr)
+        seconds = 0
+        m = /^(\d+)(\w+)?$/.match(secstr.to_s)
+        unless m
+          raise "Failed to parse stop option clause: #{secstr}"
+        end
+        case m[2]
+        when nil, /^s.*/
+          seconds = m[1].to_i
+        when /^m.*/
+          seconds = m[1].to_i * 60
+        when /^h.*/
+          seconds = m[1].to_i * 60 * 60
+        end
+        super(timing, seconds)
+      end
+    end
+  end
+end

data/lib/bpfql/runner.rb

@@ -0,0 +1,174 @@
+require 'rbbcc'
+
+module Bpfql
+  # TODO: REFACTOR ME!!
+  class Runner
+    def initialize(qobj)
+      @query_builder = qobj
+      @fmt = gen_fmt
+      @start_ts = 0
+    end
+
+    def run
+      @module = RbBCC::BCC.new(text: bpf_source)
+      puts(@fmt.gsub(/(\.\d+f|d)/, 's') % tracepoint_fields_sorted.map(&:upcase))
+
+      @module["events"].open_perf_buffer do |cpu, data, size|
+        event = @module["events"].event(data)
+        puts(@fmt % gen_extract_data(event))
+      end
+      loop {
+        begin
+          @module.perf_buffer_poll()
+        rescue Interrupt
+          break
+        end
+      }
+      puts "Exiting bpfql..."
+    end
+
+    def gen_fmt
+      fmt = []
+      fmt << '%-18.9f' if tracepoint_fields.include?("ts")
+      fmt << '%-16s' if tracepoint_fields.include?("comm")
+      fmt << '%-6d' if tracepoint_fields.include?("pid")
+      fields_noncommon.each do |f|
+        if field_maps[f] =~ /^char \w+\[\w+\]$/
+          fmt << '%-16s'
+        else
+          fmt << '%-8d'
+        end
+      end
+      fmt.join ' '
+    end
+
+    def gen_extract_data(event)
+      ret = []
+      if tracepoint_fields.include?("ts")
+        @start_ts = event.ts if @start_ts == 0
+        time_s = ((event.ts - @start_ts).to_f) / 1000000000
+        ret << time_s
+      end
+
+      tracepoint_fields_sorted.each do |f|
+        next if f == 'ts'
+        ret << event.send(f)
+      end
+      ret
+    end
+
+    def bpf_source
+      <<~SOURCE
+        #include <linux/sched.h>
+        #define BPFQL_ARY_MAX 64
+        #define BPFQL_STR_MAX 64
+        #{data_struct_source}
+
+        BPF_PERF_OUTPUT(events);
+
+        #{trace_func_source}
+      SOURCE
+    end
+
+    def data_struct_source
+      <<~STRUCT
+        struct data_t {
+          #{tracepoint_fields.map{|k| field_maps[k] + ";"}.join("\n  ")}
+        };
+      STRUCT
+    end
+
+    def trace_func_source
+      if qb.probe.tracepoint?
+        src = <<~FUNCTION
+          TRACEPOINT_PROBE(#{qb.probe.arg1}, #{qb.probe.arg2}) {
+            struct data_t data = {};
+            __ASSIGN_PID__
+            __ASSIGN_TS__
+            __ASSIGN_COMM__
+
+            __ASSIGN_FIELDS__
+            events.perf_submit(args, &data, sizeof(data));
+            return 0;
+          }
+        FUNCTION
+        src.sub!('__ASSIGN_PID__', tracepoint_fields.include?("pid") ? 'data.pid = bpf_get_current_pid_tgid();' : '')
+        src.sub!('__ASSIGN_TS__', tracepoint_fields.include?("ts") ? 'data.ts = bpf_ktime_get_ns();' : '')
+        src.sub!('__ASSIGN_COMM__', tracepoint_fields.include?("comm") ? 'bpf_get_current_comm(&data.comm, sizeof(data.comm));' : '')
+
+        if fields_noncommon.empty?
+          src.sub!('__ASSIGN_FIELDS__', '')
+        else
+          assigner = fields_noncommon.map { |field|
+            "data.#{field} = args->#{field};"
+          }.join("\n")
+          src.sub!('__ASSIGN_FIELDS__', assigner)
+        end
+        src
+      else
+        raise NotImplementedError, "unsupported probe: #{qb.probe.to_s}"
+      end
+    end
+
+    def tracepoint_fields
+      @fields ||= begin
+                    if qb.select.members[0] == '*'
+                      field_maps.keys
+                    else
+                      qb.select.members
+                    end
+                  end
+    end
+
+    def tracepoint_fields_sorted
+      [].tap do |a|
+        a << 'ts' if tracepoint_fields.include?("ts")
+        a << 'comm' if tracepoint_fields.include?("comm")
+        a << 'pid' if tracepoint_fields.include?("pid")
+        a.concat fields_noncommon
+      end
+    end
+
+    def fields_noncommon
+      tracepoint_fields - %w(pid ts comm)
+    end
+
+    def tracepoint_field_maps_from_format
+      @_tfmap ||= begin
+                    fmt = File.read "/sys/kernel/debug/tracing/events/#{qb.probe.arg1}/#{qb.probe.arg2}/format"
+                    dst = {}
+                    fmt.each_line do |l|
+                      next unless l.include?("field:")
+                      next if l.include?("common_")
+                      kv = l.split(";").map{|elm| elm.split(":").map(&:strip)}.reject{|e| e.size != 2 }
+                      kv = Hash[kv]
+                      field_name = kv['field'].split.last
+                      field_type = if kv['field'] =~ /^const char \* #{field_name}$/
+                                     "char #{field_name}[BPFQL_STR_MAX]"
+                                   elsif kv['field'] =~ /^const char \*const \* #{field_name}$/
+                                     warn("not yet fully unsupported field type")
+                                     "char #{field_name}[BPFQL_ARY_MAX][BPFQL_STR_MAX]"
+                                   else
+                                     kv['field']
+                                   end
+                      dst[field_name] = field_type
+                    end
+                    dst
+                  end
+    end
+
+    def field_maps
+      {
+        "pid"  => "u32 pid",
+        "ts"   => "u64 ts",
+        "comm" => "char comm[TASK_COMM_LEN]"
+      }.merge(tracepoint_field_maps_from_format)
+    end
+
+    private
+    def qb
+      @query_builder
+    end
+
+  end
+end

data/lib/bpfql/version.rb

@@ -0,0 +1,3 @@
+module Bpfql
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: bpfql
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Uchio Kondo
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rbbcc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: eBPF query runner. Use Ruby DSL / yaml / or plane text
+email:
+- udzura@udzura.jp
+executables:
+- bpfql
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- bpfql.gemspec
+- examples/random-simple.yml
+- exe/bpfql
+- lib/bpfql.rb
+- lib/bpfql/cli.rb
+- lib/bpfql/query.rb
+- lib/bpfql/runner.rb
+- lib/bpfql/version.rb
+homepage: https://github.com/udzura/bpfql
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: eBPF query runner
+test_files: []