rookout 0.1.28 → 0.1.50

data/lib/rookout/com_ws/output.rb

@@ -9,12 +9,17 @@ module Rookout
     require_relative "../processor/namespace_serializer"
 
     require_relative "token_bucket"
+    require_relative "envelope_wrapper"
+
+    require "concurrent"
 
     class Output
       def initialize
         @agent_id = nil
         @agent_com = nil
 
+        @skipped_aug_ids = Concurrent::Set.new
+
         @rule_status_update_bucket = TokenBucket.new Config.output_max_status_updates,
                                                      Config.output_bucket_refresh_rate do
           Logger.instance.error "Limit reached, dropping status updates"
@@ -41,7 +46,8 @@ module Rookout
         @closing = false
       end
 
-      attr_accessor :agent_id, :agent_com
+      attr_accessor :agent_id
+      attr_accessor :agent_com
 
       def close
         @closing = true
@@ -53,6 +59,17 @@ module Rookout
         @user_message_bucket.exhausted? || (!@agent_com.nil? && @agent_com.queue_full?)
       end
 
+      def send_output_queue_full_warning aug_id
+        if @skipped_aug_ids.include? aug_id
+          return
+        end
+
+        @skipped_aug_ids.add aug_id
+        error = Processor::RookError.new Exceptions::RookOutputQueueFull.new
+        send_rule_status aug_id, :Warning, error
+        Logger.instance.warning "Skipping aug-\t#{aug_id} execution because the queue is full"
+      end
+
       def send_warning rule_id, error
         send_rule_status rule_id, :Warning, error
       end
@@ -61,14 +78,25 @@ module Rookout
         return if @closing || !@agent_com
 
         @rule_status_update_bucket.if_available do
+          if active == "Deleted"
+            @skipped_aug_ids.delete? rule_id
+          end
+
           status = Com::Rookout::RuleStatusMessage.new agent_id: @agent_id,
                                                        rule_id: rule_id,
                                                        active: active
+
           if error
             status.error = error.dumps
           end
 
-          @agent_com.add status
+          envelope_wrapper = EnvelopeWrapper.new status
+
+          begin
+            @agent_com.add envelope_wrapper
+          rescue Exceptions::RookOutputQueueFull
+            # Ignored
+          end
         end
       end
 
@@ -76,17 +104,31 @@ module Rookout
         return if @closing || !@agent_com
 
         @user_message_bucket.if_available do
-          if arguments.nil? || arguments.call_method("size", "") == 0
-            protobuf_arguments = nil
+          if Config.protobuf_version2
+            envelope_wrapper = Variant2EnvelopeWrapper.new @agent_id, aug_id, report_id, arguments
           else
-            protobuf_arguments = Processor::NamespaceSerializer.dump arguments
+            if arguments.nil? || arguments.call_method("size", "") == 0
+              protobuf_arguments = nil
+            else
+              protobuf_arguments = Processor::NamespaceSerializer.dump arguments, true
+            end
+
+            envelope_wrapper = EnvelopeWrapper.new(
+              Com::Rookout::AugReportMessage.new(
+                agent_id: @agent_id,
+                aug_id: aug_id,
+                report_id: report_id,
+                arguments: protobuf_arguments
+              )
+            )
           end
 
-          msg = Com::Rookout::AugReportMessage.new agent_id: @agent_id,
-                                                   aug_id: aug_id,
-                                                   report_id: report_id,
-                                                   arguments: protobuf_arguments
-          @agent_com.add msg
+          begin
+            @agent_com.add envelope_wrapper
+            @skipped_aug_ids.delete? aug_id
+          rescue Exceptions::RookOutputQueueFull
+            send_output_queue_full_warning aug_id
+          end
         end
       end
 
@@ -114,7 +156,8 @@ module Rookout
           protobuf_arguments = nil
         else
           protobuf_arguments = Processor::NamespaceSerializer.dump(
-            Processor::Namespaces::RubyObjectNamespace.new(arguments)
+            Processor::Namespaces::RubyObjectNamespace.new(arguments),
+            true
           )
         end
 
@@ -126,7 +169,14 @@ module Rookout
                                            text: text,
                                            formatted_message: formatted_message,
                                            legacy_arguments: protobuf_arguments
-        @agent_com.add msg
+
+        envelope_wrapper = EnvelopeWrapper.new msg
+
+        begin
+          @agent_com.add envelope_wrapper
+        rescue Exceptions::RookOutputQueueFull
+          # Ignored
+        end
       end
     end
   end

data/lib/rookout/commit.rb

@@ -1,3 +1,3 @@
 module Rookout
-  COMMIT = "ad844d5f1e4bb119998eaedc927902a531b95b1f".freeze
+  COMMIT = "9f3ecb29fbadc0c8f66233c9c44be100a912e577".freeze
 end

data/lib/rookout/config.rb

@@ -6,51 +6,122 @@ module Rookout
     # Magic to allow for module variables to be easily accessible
     class << self
       attr_accessor :debug
+      attr_accessor :quiet
+
       Rookout::Config.debug = false
+      Rookout::Config.quiet = false
+
+      attr_accessor :logger_filename
+      attr_accessor :logger_log_to_stderr
+      attr_accessor :logger_log_level
 
-      attr_accessor :logger_filename, :logger_log_to_stderr, :logger_log_level
       Rookout::Config.logger_filename = "rookout/ruby-rook.log".freeze
       Rookout::Config.logger_log_to_stderr = false
       Rookout::Config.logger_log_level = :info
 
-      attr_accessor :agent_com_configuration_command_thread_name,
-                    :agent_com_max_message_limit,
-                    :agent_com_timeout,
-                    :agent_com_ping_interval,
-                    :agent_com_ping_timeout,
-                    :agent_com_flush_timeout,
-                    :agent_com_max_queued_messages
+      attr_accessor :agent_com_configuration_command_thread_name
+      attr_accessor :agent_com_max_message_limit
+      attr_accessor :agent_com_max_queue_messages_length
+      attr_accessor :agent_com_timeout
+      attr_accessor :agent_com_ping_interval
+      attr_accessor :agent_com_ping_timeout
+      attr_accessor :agent_com_flush_timeout
+      attr_accessor :agent_com_max_queued_messages
+
       Rookout::Config.agent_com_max_message_limit = 1024 * 1024
+      Rookout::Config.agent_com_max_queue_messages_length = 15 * 1024 * 1024
       Rookout::Config.agent_com_timeout = 3
       Rookout::Config.agent_com_ping_interval = 10
       Rookout::Config.agent_com_ping_timeout = 30
       Rookout::Config.agent_com_flush_timeout = 3
       Rookout::Config.agent_com_max_queued_messages = 100
 
-      attr_accessor :backoff_factor, :backoff_reset_time, :backoff_max_time
+      attr_accessor :backoff_factor
+      attr_accessor :backoff_reset_time
+      attr_accessor :backoff_max_time
+
       Rookout::Config.backoff_factor = 0.2
       Rookout::Config.backoff_max_time = 60
       Rookout::Config.backoff_reset_time = 3 * 60.0
 
-      attr_accessor :output_max_status_updates,
-                    :output_max_aug_messages,
-                    :output_max_log_items,
-                    :output_bucket_refresh_rate
+      attr_accessor :output_max_status_updates
+      attr_accessor :output_max_aug_messages
+      attr_accessor :output_max_log_items
+      attr_accessor :output_bucket_refresh_rate
+
       Rookout::Config.output_max_status_updates = 200
       Rookout::Config.output_max_aug_messages = 100
       Rookout::Config.output_max_log_items = 200
       Rookout::Config.output_bucket_refresh_rate = 10
 
       attr_accessor :instrumentation_max_aug_time
+
       Rookout::Config.instrumentation_max_aug_time = 400
 
-      attr_accessor :user_git_commit, :user_git_origin
+      attr_accessor :user_git_commit
+      attr_accessor :user_git_origin
+
       Rookout::Config.user_git_commit = nil
       Rookout::Config.user_git_origin = nil
 
-      attr_accessor :rookout_version, :rookout_commit
+      attr_accessor :rookout_version
+      attr_accessor :rookout_commit
+
       Rookout::Config.rookout_version = Rookout::VERSION
       Rookout::Config.rookout_commit = Rookout::COMMIT
+
+      attr_accessor :protobuf_version2
+
+      Rookout::Config.protobuf_version2 = false
+
+      attr_accessor :true_values
+
+      Rookout::Config.true_values = ["y", "Y", "yes", "Yes", "YES", "true", "True", "TRUE", "1"]
+
+      attr_accessor :global_rate_limit_quota
+      attr_accessor :global_rate_limit_window_size
+      attr_accessor :global_rate_limit
+      attr_accessor :using_global_rate_limiter
+
+      Rookout::Config.global_rate_limit_quota = ""
+      Rookout::Config.global_rate_limit_window_size = ""
+      Rookout::Config.global_rate_limit = ""
+      Rookout::Config.using_global_rate_limiter = false
+
+      def update_config configuration
+        update_var2_config configuration
+        update_global_rate_limit_config configuration
+      end
+
+      def update_var2_config configuration
+        is_config_proto2 = configuration["RUBY_PROTOBUF_VERSION_2"]
+        return if is_config_proto2.nil?
+        is_env_proto2 = ENV["ROOKOUT_Protobuf_Version2"]
+        if is_env_proto2.nil?
+          @protobuf_version2 = true if @true_values.include? is_config_proto2
+        elsif @true_values.include? is_env_proto2
+          Logger.instance.debug "ROOKOUT_Protobuf_Version2 env is preset: #{is_env_proto2}"
+          @protobuf_version2 = true
+        end
+        Logger.instance.info "Updating ROOKOUT_Protobuf_Version2 value to: #{@protobuf_version2}"
+      end
+
+      def update_global_rate_limit_config configuration
+        global_rate_limit = ENV["ROOKOUT_GLOBAL_RATE_LIMIT"]
+        if global_rate_limit.nil?
+          quota = configuration["RUBY_GLOBAL_RATE_LIMIT_QUOTA_MS"]
+          window_size = configuration["RUBY_GLOBAL_RATE_LIMIT_WINDOW_SIZE_MS"]
+
+          if quota != "" && !quota.nil? && window_size != "" && !window_size.nil?
+            global_rate_limit = "#{quota}/#{window_size}"
+          end
+        end
+
+        return if global_rate_limit.nil? || global_rate_limit == ""
+
+        @global_rate_limit = global_rate_limit
+        Logger.instance.info "Updating global rate limit to: #{global_rate_limit}"
+      end
     end
   end
 end

data/lib/rookout/exceptions.rb

@@ -11,9 +11,6 @@ module Rookout
     end
 
     class RookInterfaceException < ToolException
-      def initialize message
-        super message
-      end
     end
 
     class RookVersionNotSupported < ToolException
@@ -92,16 +89,39 @@ module Rookout
     class RookMessageSizeExceeded < ToolException
       def initialize message_size, max_message_size
         super "Message size of #{message_size} exceeds max size limit of #{max_message_size}. " \
-                  "Change the depth of collection or change the default by setting ROOKOUT_MAX_MESSAGE_SIZE " \
-                  "as environment variable or system property",
+              "Change the depth of collection or change the default by setting ROOKOUT_MAX_MESSAGE_SIZE " \
+              "as environment variable or system property",
               { "message_size" => message_size, "max_message_size" => max_message_size }
       end
     end
 
-    class RookRuleRateLimited < ToolException
+    class RookOutputQueueFull < ToolException
+      def initialize
+        super "Breakpoint triggered but output queue is full. " \
+              "Data collection will be disabled until the queue has emptied."
+      end
+    end
+
+    class RookRuleAugRateLimited < ToolException
       def initialize
         super "Breakpoint was disabled due to rate-limiting. " \
-                   "For more information: https://docs.rookout.com/docs/breakpoints-tasks.html#rate-limiting"
+              "For more information: https://docs.rookout.com/docs/breakpoints-tasks.html#rate-limiting"
+      end
+    end
+
+    class RookRuleGlobalRateLimited < ToolException
+      def initialize
+        super "Breakpoint was disabled due to global rate-limiting. " \
+              "For more information: https://docs.rookout.com/docs/breakpoints-tasks.html#rate-limiting"
+      end
+    end
+
+    class RookRuleRateLimited
+      def self.create
+        if Rookout::Config.using_global_rate_limiter
+          return RookRuleGlobalRateLimited.new
+        end
+        RookRuleAugRateLimited.new
       end
     end
 
@@ -126,6 +146,13 @@ module Rookout
       end
     end
 
+    class RookInvalidPositionException < ToolException
+      def initialize filename, line
+        super "Invalid code position for TracePoint: #{filename}:#{line}",
+              { filename: filename, line: line }
+      end
+    end
+
     class RookObjectCannotBeSerialized < ToolException
       def initialize object, message
         super message, { "class" => object.class.to_s }
@@ -139,9 +166,6 @@ module Rookout
     end
 
     class RookInvalidOptions < ToolException
-      def initialize description
-        super description
-      end
     end
 
     class RookInvalidLabel < ToolException
@@ -219,5 +243,20 @@ module Rookout
               }
       end
     end
+
+    class RookInvalidRateLimitConfiguration < ToolException
+      def initialize config
+        super "Invalid rate limit configuration: #{config}",
+              {
+                "config" => config
+              }
+      end
+    end
+
+    class RookUnsupportedLiveLogger < ToolException
+      def initialize
+        super "Live Logger is not supported. Try using Rookout Live Debugger instead."
+      end
+    end
   end
 end

data/lib/rookout/interface.rb

@@ -4,6 +4,7 @@ module Rookout
     include Singleton
 
     require_relative "config"
+    require_relative "utils"
     require_relative "exceptions"
     include Exceptions
 
@@ -14,7 +15,7 @@ module Rookout
 
     def print_debug_messages
       puts "[Rookout] Running in debug mode"
-      puts "[Rookout] Rookout SDK for ruby: " + Config.rookout_version
+      puts "[Rookout] Rookout SDK for ruby: #{Config.rookout_version}"
     end
 
     def start options = {}
@@ -36,16 +37,16 @@ module Rookout
         rook.connect(**@start_options)
       rescue LoadError
         raise if throw_errors
-        STDERR.puts "[Rookout] Failed to load Rookout. Please make sure to force build native extensions by setting" \
-          "'bundle config force_ruby_platform true'"
+        $stderr.puts "[Rookout] Failed to load Rookout. Please make sure to force build native extensions by setting" \
+                     "'bundle config force_ruby_platform true'"
       rescue RookMissingToken, RookInvalidToken, RookInvalidOptions, RookVersionNotSupported, RookBadProtobuf => e
         raise if throw_errors
-        STDERR.puts "[Rookout] Failed to start Rookout: #{e.message}"
+        $stderr.puts "[Rookout] Failed to start Rookout: #{e.message}"
       rescue RookCommunicationException => e
         raise if throw_errors
-        STDERR.puts "[Rookout] " + e.message
+        Logger.instance.warning "[Rookout] #{e.message}"
       rescue Exception => e
-        STDERR.puts e.full_message if Config.debug
+        $stderr.puts e.full_message if Config.debug
         raise if throw_errors
       end
     end
@@ -82,12 +83,13 @@ module Rookout
       # Only test alpine
       return unless File.exist? "/etc/alpine-release"
 
+      # Make sure protobuf is built from source
       protobuf = Gem::Specification.find_by_path "google/protobuf"
-      STDERR.puts RookBadProtobuf.new.message if protobuf.nil?
+      $stderr.puts RookBadProtobuf.new.message if protobuf.nil?
       return unless protobuf.platform != "ruby"
 
       error = RookBadProtobufPlatform.new protobuf.platform
-      STDERR.puts error.message
+      $stderr.puts error.message
     end
 
     def configure_globals options

data/lib/rookout/logger.rb

@@ -66,7 +66,7 @@ module Rookout
 
         arguments.each do |argument|
           if argument.is_a? Exception
-            @formatted_message += "\n" + argument.message + "\n" + argument.backtrace.join("\n\t")
+            @formatted_message += "\n#{argument.message}\n#{argument.backtrace.join "\n\t"}"
           end
         end
 
@@ -74,7 +74,13 @@ module Rookout
         set_caller
       end
 
-      attr_reader :level, :time, :filename, :lineno, :message, :formatted_message, :arguments
+      attr_reader :level
+      attr_reader :time
+      attr_reader :filename
+      attr_reader :lineno
+      attr_reader :message
+      attr_reader :formatted_message
+      attr_reader :arguments
 
       def format
         "#{@time} #{Process.pid}:#{Thread.current.name}-" \
@@ -137,19 +143,19 @@ module Rookout
         file = nil
 
         if Config.debug
-          STDERR.puts "[Rookout] Failed to open log file: #{log_file_path}"
-          STDERR.puts e.backtrace
+          $stderr.puts "[Rookout] Failed to open log file: #{log_file_path}"
+          $stderr.puts e.backtrace
         end
       end
 
-      ->(record) { file.write record.format + "\n" if file }
+      ->(record) { file.write "#{record.format}\n" if file }
     end
 
     def calculate_log_file_path
       return Config.logger_filename if absolute_path? Config.logger_filename
 
       if RUBY_PLATFORM.include? "darwin"
-        File.join ENV["HOME"], Config.logger_filename
+        File.join Dir.home, Config.logger_filename
       elsif RUBY_PLATFORM.match?(/cygwin|mswin|mingw|bccwin|wince|emx/)
         File.join ENV["USERPROFILE"], Config.logger_filename
       else
@@ -158,7 +164,7 @@ module Rookout
     end
 
     def new_stderr_handler
-      ->(record) { STDERR.puts record.format }
+      ->(record) { $stderr.puts record.format }
     end
 
     def new_remote_handler

data/lib/rookout/processor/namespace_serializer.rb

@@ -7,7 +7,7 @@ module Rookout
 
       module_function
 
-      def dump namespace, log_errors = true
+      def dump namespace, log_errors
         namespace.dump log_errors
       rescue StandardError => e
         message = "Failed to serialize namespace"