rookout 0.1.0 → 0.1.56

data/lib/rookout/com_ws/output.rb

@@ -9,12 +9,17 @@ module Rookout
     require_relative "../processor/namespace_serializer"
 
     require_relative "token_bucket"
+    require_relative "envelope_wrapper"
+
+    require "concurrent"
 
     class Output
       def initialize
         @agent_id = nil
         @agent_com = nil
 
+        @skipped_aug_ids = Concurrent::Set.new
+
         @rule_status_update_bucket = TokenBucket.new Config.output_max_status_updates,
                                                      Config.output_bucket_refresh_rate do
           Logger.instance.error "Limit reached, dropping status updates"
@@ -41,7 +46,8 @@ module Rookout
         @closing = false
       end
 
-      attr_accessor :agent_id, :agent_com
+      attr_accessor :agent_id
+      attr_accessor :agent_com
 
       def close
         @closing = true
@@ -49,6 +55,21 @@ module Rookout
         Logger.instance.remove_output self
       end
 
+      def user_messages_queue_full?
+        @user_message_bucket.exhausted? || (!@agent_com.nil? && @agent_com.queue_full?)
+      end
+
+      def send_output_queue_full_warning aug_id
+        if @skipped_aug_ids.include? aug_id
+          return
+        end
+
+        @skipped_aug_ids.add aug_id
+        error = Processor::RookError.new Exceptions::RookOutputQueueFull.new
+        send_rule_status aug_id, :Warning, error
+        Logger.instance.warning "Skipping aug-\t#{aug_id} execution because the queue is full"
+      end
+
       def send_warning rule_id, error
         send_rule_status rule_id, :Warning, error
       end
@@ -57,20 +78,25 @@ module Rookout
         return if @closing || !@agent_com
 
         @rule_status_update_bucket.if_available do
-          protobuf_error = nil
-          if error
-            protobuf_error = Com::Rookout::Error.new message: error.messsage,
-                                                     type: error.type
-            protobuf_error.parameters.copy_from error.parameters
-            protobuf_error.exc.copy_from error.exc
-            protobuf_error.traceback.copy_from error.traceback
+          if active == "Deleted"
+            @skipped_aug_ids.delete? rule_id
           end
 
           status = Com::Rookout::RuleStatusMessage.new agent_id: @agent_id,
                                                        rule_id: rule_id,
-                                                       active: active,
-                                                       error: protobuf_error
-          @agent_com.add status
+                                                       active: active
+
+          if error
+            status.error = error.dumps
+          end
+
+          envelope_wrapper = EnvelopeWrapper.new status
+
+          begin
+            @agent_com.add envelope_wrapper
+          rescue Exceptions::RookOutputQueueFull
+            # Ignored
+          end
         end
       end
 
@@ -78,17 +104,31 @@ module Rookout
         return if @closing || !@agent_com
 
         @user_message_bucket.if_available do
-          if arguments.nil? || arguments.call_method("size", "") == 0
-            protobuf_arguments = nil
+          if Config.protobuf_version2
+            envelope_wrapper = Variant2EnvelopeWrapper.new @agent_id, aug_id, report_id, arguments
           else
-            protobuf_arguments = Processor::NamespaceSerializer.dump arguments
+            if arguments.nil? || arguments.call_method("size", "") == 0
+              protobuf_arguments = nil
+            else
+              protobuf_arguments = Processor::NamespaceSerializer.dump arguments, true
+            end
+
+            envelope_wrapper = EnvelopeWrapper.new(
+              Com::Rookout::AugReportMessage.new(
+                agent_id: @agent_id,
+                aug_id: aug_id,
+                report_id: report_id,
+                arguments: protobuf_arguments
+              )
+            )
           end
 
-          msg = Com::Rookout::AugReportMessage.new agent_id: @agent_id,
-                                                   aug_id: aug_id,
-                                                   report_id: report_id,
-                                                   arguments: protobuf_arguments
-          @agent_com.add msg
+          begin
+            @agent_com.add envelope_wrapper
+            @skipped_aug_ids.delete? aug_id
+          rescue Exceptions::RookOutputQueueFull
+            send_output_queue_full_warning aug_id
+          end
         end
       end
 
@@ -116,7 +156,8 @@ module Rookout
           protobuf_arguments = nil
         else
           protobuf_arguments = Processor::NamespaceSerializer.dump(
-            Processor::Namespaces::RubyObjectNamespace.new(arguments)
+            Processor::Namespaces::RubyObjectNamespace.new(arguments),
+            true
           )
         end
 
@@ -128,7 +169,14 @@ module Rookout
                                            text: text,
                                            formatted_message: formatted_message,
                                            legacy_arguments: protobuf_arguments
-        @agent_com.add msg
+
+        envelope_wrapper = EnvelopeWrapper.new msg
+
+        begin
+          @agent_com.add envelope_wrapper
+        rescue Exceptions::RookOutputQueueFull
+          # Ignored
+        end
       end
     end
   end

data/lib/rookout/com_ws/pinger.rb

@@ -0,0 +1,41 @@
+module Rookout
+  module ComWs
+    require_relative "../logger"
+
+    class Pinger
+      def initialize connection, interval, timeout
+        @interval = interval
+        @timeout = timeout
+        @connection = connection
+
+        @last_pong = Time.now
+        @last_ping = Time.now
+      end
+
+      def repeat
+        loop do
+          if Time.now - @last_ping > @interval
+            Logger.instance.debug "Sending Ping"
+            @last_ping = Time.now
+            begin
+              @connection.ping Time.now.to_s do
+                Logger.instance.debug "Got Ping reply"
+                @last_pong = Time.now
+              end
+            rescue RuntimeError, Errno::EPIPE
+              Logger.instance.debug "Failed to send ping"
+              break
+            end
+          end
+
+          if Time.now - @last_pong > @timeout
+            Logger.instance.debug "Ping timeout"
+            break
+          end
+
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/rookout/com_ws/websocket_client.rb

@@ -0,0 +1,173 @@
+module Rookout
+  module ComWs
+    require "openssl"
+    require "websocket/driver"
+    require "uri"
+
+    require_relative "../config"
+    require_relative "../exceptions"
+
+    class WebsocketClient
+      def initialize url, proxy, token
+        @token = token
+        @connection = WebsocketConnection.new url, proxy
+        @proxy = proxy
+        @driver = nil
+      end
+
+      def connect
+        connection_error = nil
+        @driver = WebSocket::Driver.client @connection
+
+        headers.each do |key, value|
+          @driver.set_header key, value
+        end
+
+        @driver.on :error do |error|
+          connection_error = error
+        end
+
+        # Connect to the remote server
+        # TODO: ADD CONNECT TIMEOUT
+        @connection.connect @driver
+        @driver.start
+
+        while @driver.state == :connecting
+          recv_data = @connection.read_char
+          @driver.parse recv_data
+        end
+
+        raise Exceptions::RookWebsocketException, connection_error if @driver.state != :open
+      end
+
+      def connection_pump message_handler
+        @driver.on :message do |e|
+          message_handler.call e.data
+        end
+
+        until @driver.state == :closed
+          recv_data = @connection.read_char
+          @driver.parse recv_data
+        end
+      end
+
+      def send_frame msg
+        @driver.binary msg
+      end
+
+      def ping message, &callback
+        @driver.ping message, &callback
+      end
+
+      def close
+        return if @driver.nil?
+
+        begin
+          @driver.close
+        rescue RuntimeError, Errno::EPIPE, OpenSSL::OpenSSLError
+          # Protocol close may fail if the connection is already closed
+          nil
+        end
+        @connection.close
+      end
+
+      private
+
+      def headers
+        result = {
+          "User-Agent" => "RookoutAgent/#{Config.rookout_version}+#{Config.rookout_commit}"
+        }
+
+        result["X-Rookout-Token"] = @token unless @token.nil?
+
+        result
+      end
+    end
+
+    class WebsocketConnection
+      def initialize url, proxy
+        @url = url
+        @proxy = proxy
+
+        @uri = URI.parse @url
+        @secure = %w[https wss].include? @uri.scheme
+
+        @socket = nil
+
+        @proxy_connected = false
+      end
+
+      def connect driver
+        @socket = tcp_connect
+
+        proxy_connect driver if @proxy
+
+        @socket = ssl_connect @socket if @secure
+      end
+
+      def proxy_connect driver
+        connection_error = nil
+        @proxy_connected = false
+        proxy = driver.proxy @proxy
+
+        proxy.on :connect do
+          @proxy_connected = true
+        end
+
+        proxy.on :error do |error|
+          connection_error = error
+        end
+
+        proxy.start
+
+        proxy.parse read_char until @proxy_connected == true || !connection_error.nil?
+
+        raise Exceptions::RookProxyException, connection_error unless connection_error.nil?
+      end
+
+      attr_reader :url
+
+      def read_char
+        @socket.getc
+      end
+
+      def write buffer
+        @socket << buffer
+      end
+
+      def close
+        return if @socket.nil?
+
+        @socket.close
+        @socket = nil
+      end
+
+      private
+
+      def tcp_connect
+        if @proxy
+          uri = URI(@proxy)
+          TCPSocket.new uri.host, uri.port
+        else
+          TCPSocket.new @uri.host,
+                        @uri.port || (@secure ? 443 : 80)
+        end
+      end
+
+      def ssl_connect tcp_socket
+        ctx = ::OpenSSL::SSL::SSLContext.new
+        ctx.min_version = ::OpenSSL::SSL::TLS1_2_VERSION
+        cert_store = ::OpenSSL::X509::Store.new
+        cert_store.set_default_paths
+        ctx.cert_store = cert_store
+
+        ssl_socket = ::OpenSSL::SSL::SSLSocket.new tcp_socket, ctx
+        ssl_socket.hostname = @uri.host
+        ssl_socket.sync_close = true
+        ssl_socket.connect
+
+        ssl_socket
+      end
+    end
+  end
+end

data/lib/rookout/commit.rb

@@ -0,0 +1,3 @@
+module Rookout
+  COMMIT = "90ea969881b03cf95cef779cbcd072ef05270308".freeze
+end

data/lib/rookout/config.rb

@@ -1,55 +1,131 @@
 module Rookout
   require_relative "version"
+  require_relative "commit"
 
   module Config
     # Magic to allow for module variables to be easily accessible
     class << self
       attr_accessor :debug
+      attr_accessor :quiet
+
       Rookout::Config.debug = false
+      Rookout::Config.quiet = false
+
+      attr_accessor :logger_filename
+      attr_accessor :logger_log_to_stderr
+      attr_accessor :logger_log_level
 
-      attr_accessor :logger_filename, :logger_log_to_stderr, :logger_log_level
       Rookout::Config.logger_filename = "rookout/ruby-rook.log".freeze
       Rookout::Config.logger_log_to_stderr = false
       Rookout::Config.logger_log_level = :info
 
-      attr_accessor :agent_com_configuration_command_thread_name,
-                    :agent_com_max_message_limit,
-                    :agent_com_timeout,
-                    :agent_com_ping_interval,
-                    :agent_com_ping_timeout,
-                    :agent_com_flush_timeout,
-                    :agent_com_max_queued_messages
+      attr_accessor :agent_com_configuration_command_thread_name
+      attr_accessor :agent_com_max_message_limit
+      attr_accessor :agent_com_max_queue_messages_length
+      attr_accessor :agent_com_timeout
+      attr_accessor :agent_com_ping_interval
+      attr_accessor :agent_com_ping_timeout
+      attr_accessor :agent_com_flush_timeout
+      attr_accessor :agent_com_max_queued_messages
+
       Rookout::Config.agent_com_max_message_limit = 1024 * 1024
+      Rookout::Config.agent_com_max_queue_messages_length = 15 * 1024 * 1024
       Rookout::Config.agent_com_timeout = 3
       Rookout::Config.agent_com_ping_interval = 10
       Rookout::Config.agent_com_ping_timeout = 30
       Rookout::Config.agent_com_flush_timeout = 3
-      Rookout::Config.agent_com_max_queued_messages = 500
+      Rookout::Config.agent_com_max_queued_messages = 1000
+
+      attr_accessor :backoff_factor
+      attr_accessor :backoff_reset_time
+      attr_accessor :backoff_max_time
 
-      attr_accessor :backoff_factor, :backoff_reset_time, :backoff_max_time
       Rookout::Config.backoff_factor = 0.2
       Rookout::Config.backoff_max_time = 60
       Rookout::Config.backoff_reset_time = 3 * 60.0
 
-      attr_accessor :output_max_status_updates,
-                    :output_max_aug_messages,
-                    :output_max_log_items,
-                    :output_bucket_refresh_rate
+      attr_accessor :output_max_status_updates
+      attr_accessor :output_max_aug_messages
+      attr_accessor :output_max_log_items
+      attr_accessor :output_bucket_refresh_rate
+
       Rookout::Config.output_max_status_updates = 200
-      Rookout::Config.output_max_aug_messages = 100
+      Rookout::Config.output_max_aug_messages = 1000
       Rookout::Config.output_max_log_items = 200
-      Rookout::Config.output_bucket_refresh_rate = 10
+      Rookout::Config.output_bucket_refresh_rate = 5
 
       attr_accessor :instrumentation_max_aug_time
+
       Rookout::Config.instrumentation_max_aug_time = 400
 
-      attr_accessor :user_git_commit, :user_git_origin
+      attr_accessor :user_git_commit
+      attr_accessor :user_git_origin
+      attr_accessor :user_git_sources
+
       Rookout::Config.user_git_commit = nil
       Rookout::Config.user_git_origin = nil
+      Rookout::Config.user_git_sources = nil
+
+      attr_accessor :rookout_version
+      attr_accessor :rookout_commit
 
-      attr_accessor :rookout_version, :rookout_commit
       Rookout::Config.rookout_version = Rookout::VERSION
       Rookout::Config.rookout_commit = Rookout::COMMIT
+
+      attr_accessor :protobuf_version2
+
+      Rookout::Config.protobuf_version2 = false
+
+      attr_accessor :true_values
+
+      Rookout::Config.true_values = ["y", "Y", "yes", "Yes", "YES", "true", "True", "TRUE", "1"]
+
+      attr_accessor :global_rate_limit_quota
+      attr_accessor :global_rate_limit_window_size
+      attr_accessor :global_rate_limit
+      attr_accessor :using_global_rate_limiter
+      attr_accessor :min_rate_limit_value_ns
+
+      Rookout::Config.global_rate_limit_quota = ""
+      Rookout::Config.global_rate_limit_window_size = ""
+      Rookout::Config.global_rate_limit = ""
+      Rookout::Config.using_global_rate_limiter = false
+      Rookout::Config.min_rate_limit_value_ns = 125 * 1000
+
+      def update_config configuration
+        update_var2_config configuration
+        update_global_rate_limit_config configuration
+      end
+
+      def update_var2_config configuration
+        is_config_proto2 = configuration["RUBY_PROTOBUF_VERSION_2"]
+        return if is_config_proto2.nil?
+        is_env_proto2 = ENV["ROOKOUT_Protobuf_Version2"]
+        if is_env_proto2.nil?
+          @protobuf_version2 = true if @true_values.include? is_config_proto2
+        elsif @true_values.include? is_env_proto2
+          Logger.instance.debug "ROOKOUT_Protobuf_Version2 env is preset: #{is_env_proto2}"
+          @protobuf_version2 = true
+        end
+        Logger.instance.info "Updating ROOKOUT_Protobuf_Version2 value to: #{@protobuf_version2}"
+      end
+
+      def update_global_rate_limit_config configuration
+        global_rate_limit = ENV["ROOKOUT_GLOBAL_RATE_LIMIT"]
+        if global_rate_limit.nil?
+          quota = configuration["RUBY_GLOBAL_RATE_LIMIT_QUOTA_MS"]
+          window_size = configuration["RUBY_GLOBAL_RATE_LIMIT_WINDOW_SIZE_MS"]
+
+          if quota != "" && !quota.nil? && window_size != "" && !window_size.nil?
+            global_rate_limit = "#{quota}/#{window_size}"
+          end
+        end
+
+        return if global_rate_limit.nil? || global_rate_limit == ""
+
+        @global_rate_limit = global_rate_limit
+        Logger.instance.info "Updating global rate limit to: #{global_rate_limit}"
+      end
     end
   end
 end