rookout 0.1.28 → 0.1.50

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3056007f4e19ad4f973d551922cc733ceb394515a2cebad17f8e6d1186b3ec79
-  data.tar.gz: ee4920718d14a9ac843134a6fe37afa3b5292274924a6ded0c76247d9605f695
+  metadata.gz: 72f208b3a47ef61e59b2123afe3e666a2f3efa8295ddec10950e221e5547770a
+  data.tar.gz: 216a478be4998de21db6b2df156123d6cde587a54369914d93ba2f3ef8f9b5a7
 SHA512:
-  metadata.gz: a7a7ae5c046fa155ba5146ef30a2e381abb439ddd0936f56f3302258f72d7ae499bc18359cd8d5156434e809fc9704b93250e383c98ed3f09f89a80b56877c71
-  data.tar.gz: c71d7b41cb93686cae932edab6ce62c21d79e56b1368befe548db2046a5cf366c64d48112b07d6c09b49fdb0cb05949f3ffcfe15df90f763bb75e537aef886eb
+  metadata.gz: c9c2e1c91cba199a499fb360f519dcce9081f9c5693cf9919aaff369cd2389534900be7b580e03a3b9ef24e91394db0eec9d25e3968163de87a9390aec288392
+  data.tar.gz: 476c8d023ae8e04214693c3b929d0dae31bcf364cf23cc443bf3aab2c29bd05f08db4646a366cb54b4b5208d188df22024886bd35421849c73e331db720eb960

data/lib/rookout/augs/actions/action_run_processor.rb

@@ -5,6 +5,7 @@ module Rookout
     module Actions
       class ActionRunProcessor < Action
         def initialize arguments, processor_factory
+          super()
           @processor = processor_factory.create_processor arguments["operations"]
 
           post_operations = arguments["post_operations"]

data/lib/rookout/augs/aug.rb

@@ -10,14 +10,13 @@ module Rookout
     require_relative "../logger"
 
     class Aug
-      def initialize aug_id, action, condition, rate_limiter, _max_aug_time
-        # NOTE: max_aux_time is not implemented
-
+      def initialize aug_id, action, condition, limits_manager, _max_aug_time
         @id = aug_id
         @action = action
         @condition = condition
-        @rate_limiter = rate_limiter
+        @limits_manager = limits_manager
 
+        @executed = false
         @enabled = true
         @status = nil
         @log_cache = []
@@ -29,14 +28,16 @@ module Rookout
         return unless @enabled
 
         if output.user_messages_queue_full?
-          Logger.instance.warning "Skipping aug-\t#{id} execution because the queue is full"
+          output.send_output_queue_full_warning @id
           return
         end
 
         namespace = create_namespaces frame, extracted
         return if @condition && !@condition.evaluate(namespace)
 
-        @rate_limiter.with_limit do
+        should_skip_limiters = @condition.nil? && !@executed
+        @limits_manager.with_limit should_skip_limiters do
+          @executed = true
           report_id = Utils.uuid
           Logger.instance.info "Executing aug-\t#{id} (msg ID #{report_id})"
           @action.execute @id, report_id, namespace, output

data/lib/rookout/augs/aug_factory.rb

@@ -10,7 +10,9 @@ module Rookout
     require_relative "locations/location_exception_handler"
     require_relative "aug_rate_limiter"
     require_relative "aug"
+    require_relative "limits_manager"
     require_relative "../utils"
+    require_relative "../logger"
 
     class AugFactory
       def initialize output
@@ -29,14 +31,15 @@ module Rookout
         max_aug_time = configuration["maxAugTime"] || Config.instrumentation_max_aug_time
 
         condition_configuration = configuration["conditional"]
-        raise Exceptions::RookAugInvalidKey.new("conditional", configuration) unless
-            condition_configuration.nil? || condition_configuration.is_a?(String)
+        unless condition_configuration.nil? || condition_configuration.is_a?(String)
+          raise Exceptions::RookAugInvalidKey.new("conditional", configuration)
+        end
         condition = condition_configuration.nil? ? nil : Conditions::Condition.new(condition_configuration)
 
-        rate_limit = create_rate_limit configuration
+        limits_manager = create_limits_manager configuration
 
         max_aug_time_ns = Utils.milliseconds_to_nanoseconds max_aug_time
-        aug = Aug.new aug_id, action, condition, rate_limit, max_aug_time_ns
+        aug = Aug.new aug_id, action, condition, limits_manager, max_aug_time_ns
 
         location_configuration = configuration["location"]
         raise Exceptions::RookAugInvalidKey.new("location", configuration) unless location_configuration.is_a? Hash
@@ -47,37 +50,86 @@ module Rookout
 
       def create_location configuration, aug
         name = configuration["name"]
-        raise Exceptions::RookObjectNameMissing if name.nil?
+        raise Exceptions::RookObjectNameMissing, configuration if name.nil?
 
         case name
         when "file_line"
-          return Locations::LocationFileLine.new configuration, @output, aug
+          Locations::LocationFileLine.new configuration, @output, aug
         when "exception_handler"
-          return Locations::LocationExceptionHandler.new configuration, @output, aug
+          Locations::LocationExceptionHandler.new configuration, @output, aug
+        when "log_handler"
+          raise Exceptions::RookUnsupportedLiveLogger
         else
-          raise Exceptions::RookUnsupportedLocation if name != "file_line"
+          raise Exceptions::RookUnsupportedLocation, configuration if name != "file_line"
         end
       end
 
-      def create_rate_limit configuration
-        window_quota = 200
-        window_size = 5000
+      def create_limits_manager configuration
+        limiters = []
+        if global_rate_limiter.nil?
+          rate_limit = parse_rate_limit configuration["rateLimit"], configuration["rateLimitModifier"], 200, 5000
+          unless rate_limit.nil?
+            limiters.append AugRateLimiter.new(*rate_limit)
+          end
+        else
+          limiters.append global_rate_limiter
+        end
 
-        rate_limit = configuration["rateLimit"]
-        if rate_limit
-          rate_limit_splitted = rate_limit.split "/"
-          if rate_limit_splitted.length == 2
-            window_quota = rate_limit_splitted[0].to_i
-            window_size = rate_limit_splitted[1].to_i
+        LimitsManager.new limiters
+      end
+
+      def global_rate_limiter
+        if @global_rate_limiter.nil? && Rookout::Config.global_rate_limit != ""
+          begin
+            rate_limit = parse_rate_limit Rookout::Config.global_rate_limit,
+                                          "0", 0, 0
+            if rate_limit.nil?
+              raise Exceptions::RookInvalidRateLimitConfiguration, Rookout::Config.global_rate_limit
+            end
+          rescue Exceptions::RookInvalidRateLimitConfiguration => e
+            Logger.instance.warning "Failed to create global rate limiter: #{e.message}"
+            err = Processor::RookError.new e
+            UserWarnings.notify_error err
+            return nil
           end
+
+          @global_rate_limiter = AugRateLimiter.new(*rate_limit)
+          Logger.instance.debug "Using global rate limiter with configuration: #{rate_limit}"
+          Rookout::Config.using_global_rate_limiter = true
         end
 
-        rate_limit_modifier = configuration["rateLimitModifier"] || 5
+        @global_rate_limiter
+      end
+
+      def parse_rate_limit config, modifier_config, default_quota, default_window
+        window_quota = default_quota
+        window_size = default_window
+
+        unless config.nil? || config.empty?
+          rate_limit_split = config.split "/"
+          unless rate_limit_split.length == 2 && \
+                 Utils.is_number?(rate_limit_split[0]) && \
+                 Utils.is_number?(rate_limit_split[1])
+            raise Exceptions::RookInvalidRateLimitConfiguration, config
+          end
+
+          window_quota = rate_limit_split[0].to_i
+          window_size = rate_limit_split[1].to_i
+        end
 
         window_quota_ns = Utils.milliseconds_to_nanoseconds window_quota
         window_size_ns = Utils.milliseconds_to_nanoseconds window_size
+        rate_limit_modifier = modifier_config.to_i || 5
+
+        if window_quota_ns == 0
+          return nil
+        end
+
+        if window_quota_ns >= window_size_ns
+          raise Exceptions::RookInvalidRateLimitConfiguration, config
+        end
 
-        AugRateLimiter.new window_quota_ns, window_size_ns, rate_limit_modifier
+        [window_quota_ns, window_size_ns, rate_limit_modifier]
       end
     end
   end

data/lib/rookout/augs/aug_rate_limiter.rb

@@ -8,51 +8,50 @@ module Rookout
         @quota = quota
         @has_quota = !@quota.nil? && (@quota > 0)
         @window_size = window_size
-        @active_weight = quota / active_limit
+        if active_limit > 0
+          @active_weight = quota / active_limit
+        else
+          @active_weight = 0
+        end
 
         @mutex = Mutex.new
         @active_count = 0
         @windows = {}
       end
 
-      def with_limit start_time = nil
-        active = false
-        start_time ||= Time.now
-
-        # If quota, verify it
-        if @has_quota
-          # Get current time
-          now_ns = Utils.time_to_nanoseconds start_time
-
-          # Calculate window keys
-          current_window_key, prev_window_key = timestamp_to_window_keys now_ns
-
-          @mutex.synchronize do
-            # Clean old windows
-            cleanup now_ns
-            # Increase active count
-            @active_count += 1
-            active = true
-
-            # If exceeding quota
-            if current_usage(now_ns, current_window_key, prev_window_key) > @quota
-              warning = Processor::RookError.new Exceptions::RookRuleRateLimited.new
-              UserWarnings.notify_warning warning
-              return
-            end
+      def before_run start_time
+        # If no quota, we can safely exit
+        unless @has_quota
+          return true
+        end
+        now_ns = Utils.time_to_nanoseconds start_time
+        current_window_key, prev_window_key = timestamp_to_window_keys now_ns
+
+        @mutex.synchronize do
+          # Clean old windows
+          cleanup now_ns
+          usage = current_usage now_ns, current_window_key, prev_window_key
+          @active_count += 1
+
+          # If exceeding quota
+          if usage > @quota
+            warning = Processor::RookError.new Exceptions::RookRuleRateLimited.create
+            UserWarnings.notify_warning warning
+            return false
           end
+
+          return true
         end
+      end
 
-        begin
-          yield
-        ensure
-          if @has_quota
-            @mutex.synchronize { record_usage current_window_key, Utils.time_to_nanoseconds(Time.now) - now_ns }
-          end
+      def after_run start_time
+        if @has_quota
+          now_ns = Utils.time_to_nanoseconds start_time
+          current_window_key, = timestamp_to_window_keys now_ns
+          @mutex.synchronize { record_usage current_window_key, Utils.time_to_nanoseconds(Time.now) - now_ns }
         end
-      ensure
         # Reduce active count
-        @mutex.synchronize { @active_count -= 1 } if active
+        @mutex.synchronize { @active_count -= 1 }
       end
 
       private
@@ -75,7 +74,7 @@ module Rookout
         prev_window_usage = @windows[prev_window_key] || 0
 
         # Previous window weight
-        prev_weight = 1 - (now_ns - current_window_key) / @window_size.to_f
+        prev_weight = 1 - ((now_ns - current_window_key) / @window_size.to_f)
 
         # Final weighted usage
         (prev_window_usage * prev_weight) + (@active_count * @active_weight) + current_window_usage
@@ -91,7 +90,7 @@ module Rookout
       end
 
       def cleanup now_ns
-        @windows.reject! { |key, _| key < (now_ns - @window_size * 5) } if @windows.length > 10
+        @windows.reject! { |key, _| key < (now_ns - (@window_size * 5)) } if @windows.length > 10
       end
     end
   end

data/lib/rookout/augs/limits_manager.rb

@@ -0,0 +1,32 @@
+module Rookout
+  module Augs
+    class LimitsManager
+      def initialize limiters
+        @limiters = limiters
+      end
+
+      def with_limit skip_limiters, start_time = nil
+        start_time ||= Time.now
+        can_execute = true
+        after_execute = []
+
+        @limiters.each do |limiter|
+          limiter_passed = limiter.before_run start_time
+          if limiter_passed || skip_limiters
+            after_execute.append -> { limiter.after_run start_time }
+          else
+            can_execute = false
+          end
+        end
+
+        if can_execute
+          yield
+        end
+      ensure
+        unless after_execute.nil?
+          after_execute.each(&:call)
+        end
+      end
+    end
+  end
+end

data/lib/rookout/augs/locations/location.rb

@@ -53,7 +53,7 @@ module Rookout
         def notify_warning error
           return if silence_log? error
 
-          Logger.instance.warning error.message
+          Logger.instance.warning error.message, error.exception
 
           # For easier testing
           return if @output.nil?

data/lib/rookout/augs/locations/location_file_line.rb

@@ -21,7 +21,11 @@ module Rookout
           @line_unique = arguments["line_unique"] || false
         end
 
-        attr_reader :filename, :lineno, :file_hash, :line_crc, :line_unique
+        attr_reader :filename
+        attr_reader :lineno
+        attr_reader :file_hash
+        attr_reader :line_crc
+        attr_reader :line_unique
 
         def add_aug trigger_services
           trigger_services.get_service("position").add_aug self

data/lib/rookout/com_ws/agent_com_ws.rb

@@ -26,8 +26,12 @@ module Rookout
       attr_reader :pending_messages
 
       def initialize output, agent_host, agent_port, proxy, token, labels, print_on_connect
-        agent_host_with_protocl = agent_host.include?("://") ? agent_host : "ws://#{agent_host}"
-        @uri = "#{agent_host_with_protocl}:#{agent_port}/v1"
+        if agent_host.nil? || agent_host.empty?
+          @uri = ""
+        else
+          agent_host_with_protocl = agent_host.include?("://") ? agent_host : "ws://#{agent_host}"
+          @uri = "#{agent_host_with_protocl}:#{agent_port}/v1"
+        end
         if proxy.nil? || proxy.empty?
           @proxy = nil
         else
@@ -44,6 +48,7 @@ module Rookout
         @main_thread = nil
         @outgoing_thread = nil
         @pending_messages = Queue.new
+        @pending_messages_length = 0
 
         @running = false
         @ready_event = Concurrent::Event.new
@@ -52,18 +57,15 @@ module Rookout
         @print_on_initial_connection = print_on_connect
       end
 
-      def add message
-        buffer = wrap_in_envelope message
-        if buffer.length > Config.agent_com_max_message_limit
-          exc = Exceptions::RookMessageSizeExceeded.new buffer.length, Config.agent_com_max_message_limit
-          warning = Processor::RookError.new exc
-          UserWarnings.notify_warning warning
-
-          Logger.instance.warning "Dropping message, size was #{buffer.length} which is over the message size limit"
-          return
+      def add envelope_wrapper
+        msg_size = envelope_wrapper.calculate_size
+        if @pending_messages_length + msg_size > Config.agent_com_max_queue_messages_length ||
+           queue_full?
+          raise Exceptions::RookOutputQueueFull
         end
 
-        @pending_messages.push buffer if @pending_messages.length < Config.agent_com_max_queued_messages
+        @pending_messages.push envelope_wrapper
+        @pending_messages_length += msg_size
       end
 
       def queue_full?
@@ -103,7 +105,6 @@ module Rookout
 
       private
 
-      # rubocop:disable Style/StderrPuts
       def connection_thread
         backoff = Backoff.new
 
@@ -113,7 +114,8 @@ module Rookout
 
             if @print_on_initial_connection
               @print_on_initial_connection = false
-              STDERR.puts "[Rookout] Successfully connected to controller"
+              Utils.quiet_puts "[Rookout] Successfully connected to controller."
+              Logger.instance.debug "[Rookout] Agent ID is #{@agent_id}"
             end
             Logger.instance.debug "WebSocket connected successfully"
             Logger.instance.info "Finished initialization"
@@ -128,7 +130,7 @@ module Rookout
               @ready_event.set
             end
 
-            Logger.instance.info "Connection failed; reason = #{e.message}"
+            Logger.instance.warning "Connection failed; reason = #{e.message}"
           end
 
           backoff.after_disconnect
@@ -137,13 +139,11 @@ module Rookout
       rescue Exception => e
         Logger.instance.error "Unexpected error in connection_thread", e
       end
-      # rubocop:enable Style/StderrPuts
 
       def open_new_connection
         client = WebsocketClient.new @uri, @proxy, @token
         client.connect
 
-        Logger.instance.info "Registering agent with id #{@agent_id}"
         msg = Com::Rookout::NewAgentMessage.new agent_info: @info.pack
         client.send_frame wrap_in_envelope(msg)
 
@@ -191,15 +191,19 @@ module Rookout
       def outgoing client, on_exit
         Pinger.new(client, Config.agent_com_ping_interval, Config.agent_com_ping_timeout).repeat do
           begin
-            msg = @pending_messages.pop true
+            envelope_wrapper = @pending_messages.pop true
           rescue ThreadError
             sleep 0.25
             next
           end
 
-          if msg.is_a? ExitMessage
+          msg = envelope_wrapper.envelope
+          @pending_messages_length -= envelope_wrapper.calculate_size
+
+          case msg
+          when ExitMessage
             break if msg.thread == Thread.current
-          elsif msg.is_a? FlushMessage
+          when FlushMessage
             msg.event.set
           else
             begin

data/lib/rookout/com_ws/command_handler.rb

@@ -5,10 +5,10 @@ module Rookout
     class CommandHandler
       def initialize agent_com, augs_manager
         agent_com.on "Com::Rookout::InitialAugsCommand" do |initial_augs|
+          Config.update_config initial_augs.sdk_configuration
           augs = initial_augs.augs.map { |aug_json| JSON.parse aug_json }
           augs_manager.initialize_augs augs
         end
-
         agent_com.on "Com::Rookout::AddAugCommand" do |command|
           augs_manager.add_aug JSON.parse(command.aug_json)
         end

data/lib/rookout/com_ws/envelope_wrapper.rb

@@ -0,0 +1,68 @@
+require_relative "../protobuf/envelope_pb"
+require_relative "../processor/namespace_serializer2"
+
+
+class EnvelopeWrapperBase
+  def wrap_in_envelope message
+    any_message = Google::Protobuf::Any.pack message
+    timestamp = Google::Protobuf::Timestamp.new
+    timestamp.from_time Time.new
+    envelope = Com::Rookout::Envelope.new msg: any_message, timestamp: timestamp
+    Com::Rookout::Envelope.encode envelope
+  end
+
+  def envelope
+    raise NotImplementedError
+  end
+
+  def calculate_size
+    raise NotImplementedError
+  end
+end
+
+class Variant2EnvelopeWrapper < EnvelopeWrapperBase
+  def initialize agent_id, aug_id, report_id, arguments
+    super()
+
+    @aug_report_message = Com::Rookout::AugReportMessage.new agent_id: agent_id,
+                                                             aug_id: aug_id,
+                                                             report_id: report_id
+    @serializer = Rookout::Processor::NamespaceSerializer2.new
+    @aug_report_message.arguments2 = @serializer.dump arguments, true
+
+    @estimated_length = @serializer.estimated_pending_bytes
+    @envelope = nil
+  end
+
+  def envelope
+    if @envelope.nil?
+      @serializer.string_cache.each do |key, value|
+        @aug_report_message.strings_cache[key.encode "UTF-8", invalid: :replace, undef: :replace, replace: "?"] = value
+      end
+      @envelope = wrap_in_envelope @aug_report_message
+
+      @serializer = nil
+      @aug_report_message = nil
+    end
+    @envelope
+  end
+
+  def calculate_size
+    @estimated_length
+  end
+end
+
+class EnvelopeWrapper < EnvelopeWrapperBase
+  def initialize message
+    super()
+    @envelope = wrap_in_envelope message
+  end
+
+  def envelope
+    @envelope
+  end
+
+  def calculate_size
+    @envelope.length
+  end
+end

data/lib/rookout/com_ws/git.rb

@@ -42,7 +42,7 @@ module Rookout
         link_contents = File.read link_path
 
         if link_contents.start_with? "ref:"
-          next_link = (link_contents.split " ")[1].strip
+          next_link = link_contents.split[1].strip
           follow_sym_links root, next_link
         else
           link_contents.strip

data/lib/rookout/com_ws/information.rb

@@ -21,6 +21,8 @@ module Rookout
           @labels["k8s_namespace"] = k8_namespace
         end
 
+        collect_serverless_labels
+
         @ip_addr = local_ip
 
         @scm_info = create_scm_information
@@ -54,7 +56,8 @@ module Rookout
                                            scm: @scm_info
       end
 
-      attr_accessor :agent_id, :labels
+      attr_accessor :agent_id
+      attr_accessor :labels
 
       private
 
@@ -97,6 +100,47 @@ module Rookout
 
         Com::Rookout::SCMInformation.new origin: user_git_origin, commit: user_git_commit
       end
+
+      def aws_lambda?
+        !ENV["AWS_LAMBDA_FUNCTION_NAME"].nil?
+      end
+
+      def google_cloud_function?
+        !ENV["FUNCTION_TARGET"].nil? && !ENV["FUNCTION_SIGNATURE_TYPE"].nil?
+      end
+
+      def cloud_run_or_fire_base?
+        !ENV["K_SERVICE"].nil? &&
+          !ENV["K_REVISION"].nil? &&
+          !ENV["K_CONFIGURATION"].nil? &&
+          !ENV["PORT"].nil?
+      end
+
+      def azure_function?
+        !ENV["FUNCTIONS_WORKER_RUNTIME"].nil? && !ENV["WEBSITE_SITE_NAME"].nil?
+      end
+
+      def collect_serverless_labels
+        if aws_lambda?
+          @labels["function_name"] = ENV["AWS_LAMBDA_FUNCTION_NAME"]
+          @labels["aws_region"] = ENV["AWS_REGION"]
+
+        elsif google_cloud_function? || cloud_run_or_fire_base?
+          function_name = ENV["FUNCTION_NAME"]
+          if function_name.nil?
+            function_name = ENV["K_SERVICE"]
+          end
+          @labels["function_name"] = function_name
+
+        elsif azure_function?
+          @labels["function_name"] = ENV["WEBSITE_SITE_NAME"]
+          @labels["azure_region"] = ENV["REGION_NAME"]
+        end
+
+        return unless @labels.key? "function_name"
+
+        @labels["rookout_serverless"] = "true"
+      end
     end
   end
 end