rookout 0.1.0 → 0.1.56

data/lib/rookout/augs/locations/location.rb

@@ -1,10 +1,84 @@
 module Rookout
   module Augs
     module Locations
+      require_relative "../../processor/rook_error"
+
+      MAX_LOG_CACHE_SIZE = 10
+
       class Location
-        def add_aug _trigger_services, _aug
+        def initialize output, aug
+          @output = output
+          @aug = aug
+          @log_cache = []
+        end
+
+        def add_aug _trigger_services
           raise NotImplementedError
         end
+
+        def id
+          @aug.id
+        end
+
+        def execute frame_binding, stack_trace, extracted
+          UserWarnings.with self do
+            begin
+              @aug.execute frame_binding, stack_trace, extracted, @output
+            rescue SystemExit
+              raise
+            rescue Exception => e
+              message = "Exception while processing Aug"
+              error = Processor::RookError.new e, message
+              notify_warning error
+            end
+          end
+        end
+
+        def notify_active
+          send_rule_status :Active
+        end
+
+        def notify_pending
+          send_rule_status :Pending
+        end
+
+        def notify_removed
+          send_rule_status :Deleted
+        end
+
+        def notify_error error
+          send_rule_status :Error, error
+        end
+
+        def notify_warning error
+          return if silence_log? error
+
+          Logger.instance.warning error.message, error.exception
+
+          # For easier testing
+          return if @output.nil?
+          @output.send_rule_status id, :Warning, error
+        end
+
+        private
+
+        def silence_log? error
+          return true if @log_cache.length >= MAX_LOG_CACHE_SIZE || @log_cache.include?(error.message)
+
+          @log_cache << error.message
+          false
+        end
+
+        def send_rule_status status, error = nil
+          return if @status == status
+
+          Logger.instance.info "Updating rule status for #{@id} to #{status}"
+          @status = status
+
+          # For easier testing
+          return if @output.nil?
+          @output.send_rule_status id, status, error
+        end
       end
     end
   end

data/lib/rookout/augs/locations/location_exception_handler.rb

@@ -0,0 +1,22 @@
+module Rookout
+  module Augs
+    require_relative "../../logger"
+
+    require_relative "../../processor/rook_error"
+
+    module Locations
+      require_relative "location"
+
+      class LocationExceptionHandler < Location
+        NAME = "exception_handler".freeze
+
+        def initialize _arguments, output, aug
+          super output, aug
+        end
+
+        def add_aug _trigger_services
+        end
+      end
+    end
+  end
+end

data/lib/rookout/augs/locations/location_file_line.rb

@@ -1,24 +1,40 @@
 module Rookout
   module Augs
+    require_relative "../../logger"
+
+    require_relative "../../processor/rook_error"
+
     module Locations
       require_relative "location"
 
       class LocationFileLine < Location
         NAME = "file_line".freeze
 
-        def initialize arguments, _processor_factory
+        def initialize arguments, output, aug
+          super output, aug
           @filename = arguments["filename"]
           @lineno = arguments["lineno"]
 
+          # NOTE: Hashes are only used for suggestions, not for verification
           @file_hash = arguments["sha256"]
           @line_crc = arguments["line_crc32_2"]
-          @line_unique = arguments["line_unique"] != 0
+          @line_unique = arguments["line_unique"] || false
         end
 
-        attr_reader :filename, :lineno, :file_hash, :line_crc, :line_unique
+        attr_reader :filename
+        attr_reader :lineno
+        attr_reader :file_hash
+        attr_reader :line_crc
+        attr_reader :line_unique
 
-        def add_aug trigger_services, aug
-          trigger_services.get_service("position").add_aug self, aug
+        def add_aug trigger_services
+          trigger_services.get_service("position").add_aug self
+        rescue SystemExit
+          raise
+        rescue Exception => e
+          message = "Exception when adding aug"
+          Logger.instance.error message, e
+          notify_error Processor::RookError.new e, message
         end
       end
     end

data/lib/rookout/com_ws/agent_com_ws.rb

@@ -1,7 +1,5 @@
 module Rookout
   module ComWs
-    require "securerandom"
-    require "kontena-websocket-client"
     require "event_emitter"
     require "google/protobuf/well_known_types"
     require "concurrent"
@@ -10,21 +8,30 @@ module Rookout
     require_relative "../logger"
     require_relative "../user_warnings"
     require_relative "../exceptions"
+    require_relative "../utils"
 
     require_relative "../processor/rook_error"
 
     require_relative "../protobuf/messages_pb"
     require_relative "../protobuf/envelope_pb"
 
+    require_relative "websocket_client"
     require_relative "backoff"
     require_relative "information"
+    require_relative "pinger"
 
     class AgentComWs
       include EventEmitter
 
-      def initialize output, agent_host, agent_port, proxy, token, labels
-        agent_host_with_protocl = agent_host.include?("://") ? agent_host : "ws://#{agent_host}"
-        @uri = "#{agent_host_with_protocl}:#{agent_port}/v1"
+      attr_reader :pending_messages
+
+      def initialize output, agent_host, agent_port, proxy, token, labels, print_on_connect
+        if agent_host.nil? || agent_host.empty?
+          @uri = ""
+        else
+          agent_host_with_protocl = agent_host.include?("://") ? agent_host : "ws://#{agent_host}"
+          @uri = "#{agent_host_with_protocl}:#{agent_port}/v1"
+        end
         if proxy.nil? || proxy.empty?
           @proxy = nil
         else
@@ -38,37 +45,55 @@ module Rookout
         @info = Information.new labels
         reset_id
 
-        @thread = nil
+        @main_thread = nil
+        @outgoing_thread = nil
         @pending_messages = Queue.new
+        @pending_messages_length = 0
 
         @running = false
         @ready_event = Concurrent::Event.new
         once("Com::Rookout::InitialAugsCommand") { @ready_event.set }
-      end
 
-      def add message
-        buffer = wrap_in_envelope message
-        if buffer.length > Config.agent_com_max_message_limit
-          exc = Exceptions::RookMessageSizeExceeded.new buffer.length, Coonfig.agent_com_max_message_limit
-          warning = RookError.new exc, message
-          UserWarnings.notify_warning warning
+        @print_on_initial_connection = print_on_connect
+      end
 
-          Logger.instance.warn "Dropping message, size was #{buffer.length} which is over the message size limit"
-          return
+      def add envelope_wrapper
+        msg_size = envelope_wrapper.calculate_size
+        if @pending_messages_length + msg_size > Config.agent_com_max_queue_messages_length ||
+           queue_full?
+          raise Exceptions::RookOutputQueueFull
         end
 
-        @pending_messages.push buffer if @pending_messages.length < Config.agent_com_max_queued_messages
+        @pending_messages.push envelope_wrapper
+        @pending_messages_length += msg_size
+      end
+
+      def queue_full?
+        @pending_messages.length >= Config.agent_com_max_queued_messages
       end
 
       def connect
         @running = true
 
-        @thread = Thread.new { connection_thread }
-        @thread.name = "rookout-connection-thread"
+        @main_thread = Thread.new { connection_thread }
+        @main_thread.name = "rookout-connection-thread"
+      end
+
+      def stop
+        @running = false
+
+        # Ask outgoing thread to exit (if running)
+        @pending_messages << ExitMessage.new(@outgoing_thread)
+
+        @main_thread.join
       end
 
       def wait_for_ready
-        @ready_event.wait Config.agent_com_timeout
+        is_finished = @ready_event.wait Config.agent_com_timeout
+        # We didn't finish - will keep trying in the background
+        raise Exceptions::RookCommunicationException unless is_finished
+
+        # We finished - raise if we failed
         raise @connection_error if @connection_error
       end
 
@@ -85,50 +110,49 @@ module Rookout
 
         while @running
           begin
-            backoff.before_connection_attempt
-            Kontena::Websocket::Client.connect(@uri, **create_options) do |client|
-              register client
+            client = open_new_connection
 
-              Logger.instance.debug "WebSocket connected successfully"
-              @token_valid = true
-              backoff.after_connect
-
-              connection_pump client
+            if @print_on_initial_connection
+              @print_on_initial_connection = false
+              Utils.quiet_puts "[Rookout] Successfully connected to controller."
+              Logger.instance.debug "[Rookout] Agent ID is #{@agent_id}"
             end
+            Logger.instance.debug "WebSocket connected successfully"
+            Logger.instance.info "Finished initialization"
+
+            @token_valid = true
+            backoff.after_connect
+
+            connection_pump client
           rescue Exception => e
-            if !@token_valid && e.is_a?(Kontena::Websocket::ProtocolError) && e.message.include?("403")
+            if !@token_valid && e.message.include?("403")
               @connection_error = Exceptions::RookInvalidToken.new @token
               @ready_event.set
             end
 
-            Logger.instance.info "Connection failed; reason = #{e.message}"
+            if e.message.include? "400"
+              @connection_error = Exceptions::RookWebSocketError.new 400
+              @ready_event.set
+            end
+
+            Logger.instance.warning "Connection failed; reason = #{e.message}"
           end
 
           backoff.after_disconnect
           Logger.instance.debug "Reconnecting"
         end
+      rescue Exception => e
+        Logger.instance.error "Unexpected error in connection_thread", e
       end
 
-      def create_options
-        headers = {
-          "User-Agent" => "RookoutAgent/#{Config.rookout_version}+#{Config.rookout_commit}"
-        }
-
-        headers["X-Rookout-Token"] = @token unless @token.nil?
-
-        # NOTE: WE DONT HAVE PROXY SUPPORT (YET) - THIS WILL PROBABLY REQUIRE FORKING KONTENA
-        { headers: headers,
-          connect_timeout: Config.agent_com_timeout,
-          open_timeout: Config.agent_com_timeout,
-          ping_interval: Config.agent_com_ping_interval,
-          ping_timeout: Config.agent_com_ping_timeout }
-      end
-
-      def register client
-        Logger.instance.info "Registering agent with id #{@agent_id}"
+      def open_new_connection
+        client = WebsocketClient.new @uri, @proxy, @token
+        client.connect
 
         msg = Com::Rookout::NewAgentMessage.new agent_info: @info.pack
-        client.send wrap_in_envelope(msg)
+        client.send_frame wrap_in_envelope(msg)
+
+        client
       end
 
       ACCEPTED_MESSAGE_TYPES = [
@@ -140,11 +164,18 @@ module Rookout
       ].freeze
 
       def connection_pump client
-        on_outgoing_exit = proc { client.disconnect }
-        send_thread = Thread.new { outgoing client, on_outgoing_exit }
-        send_thread.name = "rookout-outgoing-thread"
+        on_outgoing_exit = proc {
+          begin
+            client.close
+          rescue Exception => e
+            Logger.instance.error "Unexpected error exiting outgoing thread", e
+          end
+        }
+
+        @outgoing_thread = Thread.new { outgoing client, on_outgoing_exit }
+        @outgoing_thread.name = "rookout-outgoing-thread"
 
-        client.read do |raw_message|
+        message_handler = proc do |raw_message|
           envelope = Com::Rookout::Envelope.decode raw_message.pack("c*")
 
           ACCEPTED_MESSAGE_TYPES.each do |klass|
@@ -153,28 +184,35 @@ module Rookout
           end
         end
 
+        client.connection_pump message_handler
+
         Logger.instance.debug "Incoming loop - socket disconnected"
 
         @pending_messages.push ExitMessage.new(send_thread)
-        send_thread.join
+        @outgoing_thread.join
+        @outgoing_thread = nil
       end
 
       def outgoing client, on_exit
-        loop do
+        Pinger.new(client, Config.agent_com_ping_interval, Config.agent_com_ping_timeout).repeat do
           begin
-            msg = @pending_messages.pop true
+            envelope_wrapper = @pending_messages.pop true
           rescue ThreadError
             sleep 0.25
             next
           end
 
-          if msg.is_a? ExitMessage
+          msg = envelope_wrapper.envelope
+          @pending_messages_length -= envelope_wrapper.calculate_size
+
+          case msg
+          when ExitMessage
             break if msg.thread == Thread.current
-          elsif msg.is_a? FlushMessage
+          when FlushMessage
             msg.event.set
           else
             begin
-              client.send msg
+              client.send_frame msg
             rescue RuntimeError
               @queue << msg
               break
@@ -184,6 +222,7 @@ module Rookout
       rescue Exception => e
         Logger.instance.exception "Outgoing thread failed", e
       ensure
+        Logger.instance.debug "Outgoing thread exiting"
         on_exit.call
       end
 
@@ -196,7 +235,7 @@ module Rookout
       end
 
       def reset_id
-        @agent_id = SecureRandom.uuid
+        @agent_id = Utils.uuid
         @output.agent_id = @agent_id
         @info.agent_id = @agent_id
       end

data/lib/rookout/com_ws/backoff.rb

@@ -4,25 +4,20 @@ module Rookout
 
     class Backoff
       def initialize
-        @connected = false
-        @last_successful_connection = Time.mktime 1970
-        reset_backoff
-      end
-
-      def before_connection_attempt
-        return unless Time.new > @last_successful_connection + Config.backoff_reset_time
+        @connect_time = nil
         reset_backoff
       end
 
       def after_disconnect
-        @connected = false
+        reset_backoff if @connect_time && Time.new > @connect_time + Config.backoff_reset_time
+        @connect_time = nil
+
         sleep @next_backoff
         @next_backoff = [@next_backoff * 2, Config.backoff_max_time].min
       end
 
       def after_connect
-        @connected = true
-        @last_successful_connection = Time.now
+        @connect_time = Time.now
       end
 
       private

data/lib/rookout/com_ws/command_handler.rb

@@ -5,10 +5,10 @@ module Rookout
     class CommandHandler
       def initialize agent_com, augs_manager
         agent_com.on "Com::Rookout::InitialAugsCommand" do |initial_augs|
+          Config.update_config initial_augs.sdk_configuration
           augs = initial_augs.augs.map { |aug_json| JSON.parse aug_json }
           augs_manager.initialize_augs augs
         end
-
         agent_com.on "Com::Rookout::AddAugCommand" do |command|
           augs_manager.add_aug JSON.parse(command.aug_json)
         end

data/lib/rookout/com_ws/envelope_wrapper.rb

@@ -0,0 +1,68 @@
+require_relative "../protobuf/envelope_pb"
+require_relative "../processor/namespace_serializer2"
+
+
+class EnvelopeWrapperBase
+  def wrap_in_envelope message
+    any_message = Google::Protobuf::Any.pack message
+    timestamp = Google::Protobuf::Timestamp.new
+    timestamp.from_time Time.new
+    envelope = Com::Rookout::Envelope.new msg: any_message, timestamp: timestamp
+    Com::Rookout::Envelope.encode envelope
+  end
+
+  def envelope
+    raise NotImplementedError
+  end
+
+  def calculate_size
+    raise NotImplementedError
+  end
+end
+
+class Variant2EnvelopeWrapper < EnvelopeWrapperBase
+  def initialize agent_id, aug_id, report_id, arguments
+    super()
+
+    @aug_report_message = Com::Rookout::AugReportMessage.new agent_id: agent_id,
+                                                             aug_id: aug_id,
+                                                             report_id: report_id
+    @serializer = Rookout::Processor::NamespaceSerializer2.new
+    @aug_report_message.arguments2 = @serializer.dump arguments, true
+
+    @estimated_length = @serializer.estimated_pending_bytes
+    @envelope = nil
+  end
+
+  def envelope
+    if @envelope.nil?
+      @serializer.string_cache.each do |key, value|
+        @aug_report_message.strings_cache[key.encode "UTF-8", invalid: :replace, undef: :replace, replace: "?"] = value
+      end
+      @envelope = wrap_in_envelope @aug_report_message
+
+      @serializer = nil
+      @aug_report_message = nil
+    end
+    @envelope
+  end
+
+  def calculate_size
+    @estimated_length
+  end
+end
+
+class EnvelopeWrapper < EnvelopeWrapperBase
+  def initialize message
+    super()
+    @envelope = wrap_in_envelope message
+  end
+
+  def envelope
+    @envelope
+  end
+
+  def calculate_size
+    @envelope.length
+  end
+end

data/lib/rookout/com_ws/git.rb

@@ -42,7 +42,7 @@ module Rookout
         link_contents = File.read link_path
 
         if link_contents.start_with? "ref:"
-          next_link = (link_contents.split " ")[1].strip
+          next_link = link_contents.split[1].strip
           follow_sym_links root, next_link
         else
           link_contents.strip

data/lib/rookout/com_ws/information.rb

@@ -11,10 +11,17 @@ module Rookout
       require_relative "git"
       include Git
 
-      def initialize labels
+      def initialize labels, k8s_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
         @agent_id = nil
         @labels = labels.clone
-        @labels["rookout_debug"] = "on"
+        @labels["rookout_debug"] = "on" if Config.debug
+
+        k8_namespace = create_cluster_namespace k8s_file_path
+        unless k8_namespace == "" || @labels.key?("k8s_namespace")
+          @labels["k8s_namespace"] = k8_namespace
+        end
+
+        collect_serverless_labels
 
         @ip_addr = local_ip
 
@@ -50,6 +57,7 @@ module Rookout
       end
 
       attr_accessor :agent_id
+      attr_accessor :labels
 
       private
 
@@ -65,10 +73,52 @@ module Rookout
       end
       # rubocop:enable Style/ParallelAssignment
 
+      def create_cluster_namespace k8s_file_path
+        return_value = ""
+
+        # No Kubernetes is valid
+        if File.file? k8s_file_path
+          # Read the file contents and return it as result
+          return_value = File.read k8s_file_path
+        end
+
+        return_value
+      end
+
+      def parse_raw_git_sources
+        git_sources_raw = ENV["ROOKOUT_SOURCES"]
+        if git_sources_raw.nil?
+          return nil
+        end
+
+        git_sources = git_sources_raw.split ";"
+        user_git_sources = {}
+        git_sources.each do |url|
+          url_parts = url.split "#"
+          if url_parts.length != 2
+            next
+          end
+
+          remote_url = url_parts[0]
+          commit = url_parts[1]
+          user_git_sources[remote_url] = commit
+        end
+
+        user_git_sources
+      end
+
       def create_scm_information
-        user_git_origin = Config.user_git_origin || ENV["ROOKOUT_ORIGIN"]
+        user_git_origin = Config.user_git_origin || ENV["ROOKOUT_REMOTE_ORIGIN"]
         user_git_commit = Config.user_git_commit || ENV["ROOKOUT_COMMIT"]
 
+        user_git_sources = Config.user_git_sources || parse_raw_git_sources
+
+        if !user_git_sources.nil? && !user_git_sources.empty?
+          user_git_sources = user_git_sources.map do |remote_url, commit|
+            Com::Rookout::SCMInformation::SourceInfo.new remoteOriginUrl: remote_url, commit: commit
+          end
+        end
+
         if user_git_origin.nil? && user_git_commit.nil?
           search_path = File.dirname File.absolute_path($PROGRAM_NAME)
           git_root = find_root search_path
@@ -78,7 +128,48 @@ module Rookout
           end
         end
 
-        Com::Rookout::SCMInformation.new origin: user_git_origin, commit: user_git_commit
+        Com::Rookout::SCMInformation.new origin: user_git_origin, commit: user_git_commit, sources: user_git_sources
+      end
+
+      def aws_lambda?
+        !ENV["AWS_LAMBDA_FUNCTION_NAME"].nil?
+      end
+
+      def google_cloud_function?
+        !ENV["FUNCTION_TARGET"].nil? && !ENV["FUNCTION_SIGNATURE_TYPE"].nil?
+      end
+
+      def cloud_run_or_fire_base?
+        !ENV["K_SERVICE"].nil? &&
+          !ENV["K_REVISION"].nil? &&
+          !ENV["K_CONFIGURATION"].nil? &&
+          !ENV["PORT"].nil?
+      end
+
+      def azure_function?
+        !ENV["FUNCTIONS_WORKER_RUNTIME"].nil? && !ENV["WEBSITE_SITE_NAME"].nil?
+      end
+
+      def collect_serverless_labels
+        if aws_lambda?
+          @labels["function_name"] = ENV["AWS_LAMBDA_FUNCTION_NAME"]
+          @labels["aws_region"] = ENV["AWS_REGION"]
+
+        elsif google_cloud_function? || cloud_run_or_fire_base?
+          function_name = ENV["FUNCTION_NAME"]
+          if function_name.nil?
+            function_name = ENV["K_SERVICE"]
+          end
+          @labels["function_name"] = function_name
+
+        elsif azure_function?
+          @labels["function_name"] = ENV["WEBSITE_SITE_NAME"]
+          @labels["azure_region"] = ENV["REGION_NAME"]
+        end
+
+        return unless @labels.key? "function_name"
+
+        @labels["rookout_serverless"] = "true"
       end
     end
   end