immunio 0.15.2

data/lib/immunio/request.rb

@@ -0,0 +1,139 @@
+require_relative "vm"
+
+module Immunio
+  class Request
+    # Data sent to the server
+    attr_accessor :data, :vm
+
+    # We keep the VM and hook handlers in the request to ensure all hooks for this request
+    # use the same handlers.
+    attr_accessor :vm
+
+    def initialize(env)
+      @env = env
+      @data = {
+        "type" => "engine.http_request",
+        "request_id" => id,
+        "start_time" => Time.now.utc.iso8601(6),
+      }
+      @timings = Hash.new { |h, name| h[name] = { count: 0, total_duration: 0 } }
+      @paused_times = {}
+      @start_time = Time.now
+
+      Immunio.logger.debug "Created new Request #{id} with data #{@data}"
+    end
+
+    def id
+      @env["action_dispatch.request_id"] || internal_id
+    end
+
+    # Time the execution of a block and stores it in the request's `timings`.
+    def time(type, name)
+      Immunio.logger.debug { "Starting request time for #{type} #{name}" }
+      start = Time.now
+      key = {type: type, name: name}
+      nested = false
+      if @paused_times[key]
+        nested = true
+      else
+        @paused_times[key] = 0
+      end
+      yield
+    ensure
+        timing = @timings[key]
+        timing[:count] += 1
+        timing[:total_duration] += (Time.now - start) * 1000
+      if !nested
+        timing[:total_duration] -= @paused_times.delete(key)
+      end
+    end
+
+    # Pause execution timing while executing block
+    def pause(type, name)
+      key = {type: type, name: name}
+      start = Time.now
+      yield
+    ensure
+      @paused_times[key]+= (Time.now - start) * 1000
+    end
+
+    def timings
+      total_key = {type: "request", name: "total"}
+      timing = @timings[total_key]
+      timing[:count] = 1
+      timing[:total_duration] = (Time.now - @start_time) * 1000
+
+      split_timings = {}
+      @timings.each do |key, value|
+        # Round to microsecond precision
+        value[:total_duration] = value[:total_duration].round(3)
+
+        split_timings[key[:type]] ||= {}
+        split_timings[key[:type]][key[:name]] = value
+      end
+      split_timings
+    end
+
+    def should_report?
+      result = Immunio.run_hook "request", "should_report"
+      # If the hook doesn't exist, turn a nil report value into a false value
+      report = !!result["report"]
+      Immunio.logger.debug "Report request: #{report}"
+      report
+    end
+
+    # Encode the request data to send it to the channel.
+    def encode
+      if @data.is_a? Hash
+        @data.to_msgpack
+      else # Lua table
+        # Encode the data in Lua to avoid pulling a huge Lua table out of the VM, which is slow.
+        @vm.unsafe_call_by_name 'encode', @data
+      end
+    end
+
+    def self.current
+      Thread.current["immunio.request"]
+    end
+
+    def self.current=(request)
+      Thread.current["immunio.request"] = request
+      Immunio.logger.debug "Setting current request for thread #{Thread.current.object_id} to #{request && request.id || nil}"
+    end
+
+    # This shim exists to preserve the stack depth into hooks when self.time and self.pause are
+    # called. Otherwise the number of immunio frames will vary based on the state of the request object
+    def self.stack_shim()
+      yield
+    end
+
+    def self.time(type, name, &block)
+      if Request.current
+        Request.current.time(type, name, &block)
+      else
+        Immunio.logger.debug { "Starting request time for #{type} #{name}" }
+        self.stack_shim(&block)
+      end
+    end
+
+    def self.pause(type, name, &block)
+      if Request.current
+        Request.current.pause(type, name, &block)
+      else
+        self.stack_shim(&block)
+      end
+    end
+
+    private
+
+    def internal_id
+      return @internal_id if defined?(@internal_id)
+
+      if @env['HTTP_X_REQUEST_ID']
+        @internal_id = @env['HTTP_X_REQUEST_ID'].gsub(/[^\w\-]/, '').first(255)
+      else
+        @internal_id = SecureRandom.uuid
+      end
+    end
+  end
+end

data/lib/immunio/rufus_lua_ext/ref.rb

@@ -0,0 +1,27 @@
+module Rufus::Lua
+  # Auto-unref Lua ref when it goes out of scope in Ruby.
+  class Ref
+    def initialize_with_finalizer(*args)
+      initialize_without_finalizer(*args)
+
+      ObjectSpace.define_finalizer self, self.class.finalize(@pointer, @ref)
+    end
+    alias_method :initialize_without_finalizer, :initialize
+    alias_method :initialize, :initialize_with_finalizer
+
+    def self.finalize(pointer, ref)
+      # Hack to test if the Lua state was closed and de-refed.
+      original_pointer_address = pointer.read_pointer
+      proc do
+        Lib.luaL_unref(pointer, LUA_REGISTRYINDEX, ref) if original_pointer_address == pointer.read_pointer
+      end
+    end
+
+    undef :free
+    def free
+      # Noop. Taken care of by finalizer. Should not conflict.
+    end
+
+    public :load_onto_stack
+  end
+end

data/lib/immunio/rufus_lua_ext/state.rb

@@ -0,0 +1,157 @@
+module Rufus::Lua
+  class State
+    # Load the code into a function and return it.
+    # Safer than evaling some string.
+    def load(code, filename=nil)
+      err = Lib.luaL_loadbuffer(@pointer, code, code.size, filename)
+      fail_if_error('load:compile', err, nil, filename, nil)
+
+      function = stack_fetch
+      stack_pop
+
+      function
+    end
+  end
+
+  class HashWithNilKeyError < StandardError
+  end
+
+  module StateMixin
+    # Expose error_handler to set it manually on Functions since it is not inherited from State.
+    attr_accessor :error_handler
+
+    protected
+
+      # Fix stack_fetch so it doesn't pop a value off the top for tables,
+      # functions, and threads.
+      #
+      # FIXME: pos isn't honored for tables, functions, or threads.
+      undef :stack_fetch
+      def stack_fetch(pos=-1)
+        type, tname = stack_type_at(pos)
+
+        case type
+
+          when TNIL then nil
+
+          when TSTRING then
+            string = nil
+            FFI::MemoryPointer.new(:size_t) do |len|
+              ptr = Lib.lua_tolstring(@pointer, pos, len)
+              string = ptr.read_string(len.read_long)
+            end
+            string
+
+          when TBOOLEAN then (Lib.lua_toboolean(@pointer, pos) == 1)
+          when TNUMBER then Lib.lua_tonumber(@pointer, pos)
+
+          when TTABLE then
+            table = Table.new(@pointer)
+            table.load_onto_stack
+            table
+
+          when TFUNCTION then
+            func = Function.new(@pointer)
+            func.load_onto_stack
+            func
+
+          when TTHREAD then
+            coroutine = Coroutine.new(@pointer)
+            coroutine.load_onto_stack
+            coroutine
+
+          else tname
+        end
+      end
+
+      # Now that we've fixed popping in stack_fetch above, always remove the top
+      # item from the stack.
+      undef :stack_pop
+      def stack_pop
+        r = stack_fetch
+        stack_unstack
+        r
+      end
+
+      # This method is effectively a pop without retrieving the top value.
+      # Upstream papers over stack corruption by not letting the stack pointer
+      # go negative, but this makes our tests erroneously pass when they check
+      # for stack corruption. Instead, do the right thing, and catch problems in
+      # tests.
+      undef :stack_unstack
+      def stack_unstack
+        Lib.lua_settop(@pointer, -2)
+      end
+
+      # Patch `stack_push` to support passing Lua objects (functions, tables)
+      # back into the VM. Also, support conversion of Pathname objects.
+      def stack_push_with_ref(o)
+        case o
+        when Ref
+          o.load_onto_stack
+        when Pathname
+          stack_push_without_ref o.to_s
+        when File
+          stack_push_without_ref o.path
+        when ActiveSupport::Multibyte::Chars
+          stack_push_without_ref o.to_s
+        else
+          begin
+            stack_push_without_ref o
+          rescue ArgumentError
+            # Rufus-lua doesn't know how to handle it, so call inspect which
+            # should always return a string describing the object. Usually we
+            # just need a placeholder for such objects, so this will do.
+            stack_push_without_ref o.inspect
+          end
+        end
+      end
+      alias_method :stack_push_without_ref, :stack_push
+      alias_method :stack_push, :stack_push_with_ref
+
+      # Patch `stack_fetch` to set encoding to strings.
+      def stack_fetch_with_unicode(pos=-1)
+        type = stack_type_at(pos)
+
+        case type[0]
+        when TSTRING
+          string = nil
+          FFI::MemoryPointer.new(:size_t) do |len|
+            ptr = Lib.lua_tolstring(@pointer, pos, len)
+            string = ptr.read_string(len.read_long)
+          end
+          string.force_encoding(Encoding.default_internal)
+        else
+          stack_fetch_without_unicode pos
+        end
+      end
+      alias_method :stack_fetch_without_unicode, :stack_fetch
+      alias_method :stack_fetch, :stack_fetch_with_unicode
+
+      # Trying to push a hash with a nil key causes a Lua panic. Instead, raise
+      # an exception in Ruby which we can deal with.
+      def stack_push_hash_with_immunio(h)
+        if h.has_key? nil
+          raise HashWithNilKeyError.new "can't pass hash with nil key into Lua"
+        end
+        stack_push_hash_without_immunio h
+      end
+      alias_method :stack_push_hash_without_immunio, :stack_push_hash
+      alias_method :stack_push_hash, :stack_push_hash_with_immunio
+  end
+
+  class Function
+    # If an argument fails to be converted for Lua, we end up leaking everything
+    # that was already pushed onto the stack for the call. Make sure we reset
+    # our stack.
+    def call_with_stack_reset(*args)
+      begin
+        call_without_stack_reset(*args)
+      rescue
+        Lib.lua_settop(@pointer, 0)
+        raise
+      end
+    end
+    alias_method_chain :call, :stack_reset
+  end
+end

data/lib/immunio/rufus_lua_ext/table.rb

@@ -0,0 +1,137 @@
+module Rufus::Lua
+  class Table
+    # Method to behave like Hash#fetch.
+    def fetch(key, default=nil)
+      value = self[key]
+      if value.nil?
+        default
+      else
+        value
+      end
+    end
+
+    def has_key?(key)
+      self.fetch(key) != nil
+    end
+
+    # Fix memory leak
+    undef :"[]"
+    def [](k)
+      # Push table onto stack
+      load_onto_stack
+
+      # Push key onto stack
+      stack_push(k)
+
+      # Fetches the value from the table, replaces the key at the top of the
+      # stack with the value. At the end, the stack still has the table at -2.
+      Lib.lua_gettable(@pointer, -2)
+
+      # Pop value off stack
+      v = stack_pop
+
+      # Pop table off stack
+      stack_pop # <= Memory leak fixed here
+
+      v
+    end
+
+    # Fix memory leak
+    undef :"[]="
+    def []=(k, v)
+      # Push table onto stack
+      load_onto_stack
+
+      stack_push(k)
+      stack_push(v)
+
+      # For the VM heavy thread test, we need to cause out-of-order execution in
+      # methods modifying the Lua stack. Here we tell Ruby to swap out threads
+      # if we are in the thread test.
+      Thread.pass if $IMMUNIO_IN_THREAD_TEST
+
+      # Sets the key-value pair onto the table, then pops the key and value but
+      # leaves the table on the stack
+      Lib.lua_settable(@pointer, -3)
+
+      # Pop table off stack
+      stack_pop # <= Memory leak fixed here
+
+      v
+    end
+
+    # Fix memory leak
+    undef :objlen
+    def objlen
+      # Push table onto stack
+      load_onto_stack
+
+      # Gets the length of the table, but leaves the table on the stack
+      len = Lib.lua_objlen(@pointer, -1)
+
+      # Pop table off stack
+      stack_pop # <= Memory leak fixed here
+
+      len
+    end
+
+    # Override original to_h to fix bad implementation and convert nested
+    # collections.
+    # See http://www.lua.org/manual/5.1/manual.html#lua%5Fnext for details on
+    # this crazy complex iteration system in lua...
+    undef :to_h
+    def to_h
+      # Load table onto stack
+      load_onto_stack
+
+      # Get position of table on stack
+      table_pos = stack_top
+
+      # This is the starter for the lua_next iterator. Push a nil value as a
+      # sentinel onto the stack. This tells lua_next to fetch the first key-
+      # value pair from the table.
+      Lib.lua_pushnil(@pointer)
+
+      h = {}
+
+      # Fetch the next key-value pair. The last key must be on the top of the
+      # stack. The key on the stack will be overwritten with the next key
+      # fetched from the table. The value will be pushed onto the stack after
+      # the key.
+      #
+      # In order to iterate, the caller must pop the value off the stack.
+      # lua_next then uses the current key, which is once again at the top of
+      # the stack, to locate the next key.
+      #
+      # But! When there are no more key-value pairs, this call will remove the
+      # key from the stack, leaving the table at the top.
+      #
+      # WARNING: Lua keys can be anything (yes, that includes a table). We do
+      # not support tables as keys. Keys must be nice numbers or strings.
+      while Lib.lua_next(@pointer, table_pos) != 0 do
+        # Grab key and value from top of stack
+        value = stack_fetch(-1)
+        key = stack_fetch(-2)
+
+        # Pop the value off the stack leaving the key on top
+        stack_unstack
+
+        # Recurse into table if needed
+        if Table === value
+          if value[1]
+            h[key] = value.to_a
+          else
+            h[key] = value.to_h
+          end
+        else
+          h[key] = value
+        end
+      end
+
+      # Remove the table pushed onto the stack at the very beginning
+      Lib.lua_remove(@pointer, table_pos)
+
+      h
+    end
+  end
+end