hoss-agent 1.0.1

data/lib/hoss/transport/worker.rb

@@ -0,0 +1,319 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+require 'hoss/report'
+require 'zlib'
+require 'cgi'
+
+module Hoss
+  module Transport
+    # @api private
+    class Worker
+      include Logging
+
+      class << self
+        def adapter
+          @adapter ||= Connection
+        end
+
+        attr_writer :adapter
+      end
+
+      # @api private
+      class StopMessage; end
+
+      # @api private
+      class FlushMessage; end
+
+      def initialize(
+        config,
+        queue,
+        serializers:,
+        filters:
+      )
+        @config = config
+        @queue = queue
+
+        @serializers = serializers
+        @filters = filters
+
+        @connection = self.class.adapter.new(config)
+      end
+
+      attr_reader :config, :queue, :filters, :name, :connection, :serializers
+      def work_forever
+        
+        done = false
+        while (!done && msg = queue.pop)
+          debug 'working message', msg
+
+          # wait if we don't have a config
+          while config.agentConfig.nil?
+            sleep 0.1
+          end
+          
+          case msg
+          when StopMessage
+            debug 'Stopping worker -- %s', self
+            done = true
+            break
+          else
+            batch = []
+            current_batch_size = 0
+
+            # Use this as the first message in the batch
+            event = msg.filtered ? msg : filter_resource(msg)
+            unless event.nil?
+              event_size = resource_size(event)
+              if current_batch_size + event_size <= @config.batch_size
+                unless host_blacklisted(event)
+                  current_batch_size += event_size
+                  if event.retries < @config.max_event_retries
+                    batch.push(event) 
+                  else
+                    debug "max retries hit for event"
+                  end
+                end
+              else
+                debug "Event is too large, body needs to be truncated"
+              end
+            end
+
+            # Do inner loop reading queue to build report
+            requeue_messages = []
+            while current_batch_size < @config.batch_size && !queue.empty?
+              next_msg = queue.pop
+              case next_msg
+              when StopMessage
+                debug 'Stopping worker -- %s', self
+                done = true
+                break
+              else
+                event = next_msg.filtered ? next_msg : filter_resource(next_msg) unless 
+                unless event.nil?
+                  event_size = resource_size(event)
+                  if current_batch_size + event_size <= @config.batch_size
+                    unless host_blacklisted(event)
+                      current_batch_size += event_size
+                      if event.retries < @config.max_event_retries
+                        batch.push(event) 
+                      else
+                        debug "max retries hit for event"
+                      end
+                    end
+                  else
+                    debug "Event too large for this batch, requeue"
+                    requeue_messages.push(event)
+                  end
+                end
+              end
+            end
+          
+            if batch.length == 0
+              debug "batch is empty, breaking"
+              break
+            end
+
+            debug "Requeue #{requeue_messages.length} messages" if requeue_messages.length > 0
+            requeue_messages.each {|msg| queue.push(msg, false) }
+
+            report = Report.new
+            report.events = batch.map {|event| serializers.serialize(event) }
+
+            debug "Finished building report"
+            data = serializers.serialize(report)
+            json = JSON.fast_generate(data)
+            begin
+              debug json
+            rescue Exception => e
+              debug 'unable to print body'
+              puts json
+            end
+            begin
+              connection.write(json)
+            rescue Exception => e
+              error format('Failed send report: %s %s', e.inspect, e.backtrace)
+              batch.each do |m|
+                m.retries += 1
+                queue.push(m, false)
+              end
+              sleep 1
+            end
+          end
+        end
+      rescue Exception => e
+        warn 'Worker died with exception: %s', e.inspect
+        debug e.backtrace.join("\n")
+      end
+
+      private
+
+      def resource_size(resource)
+        size = 20
+        size += JSON.fast_generate(resource.request.headers).length if resource.request && resource.request.headers
+        size += JSON.fast_generate(resource.response.headers).length if resource.response && resource.response.headers
+        size += resource.request.body.length if resource.request && resource.request.body
+        size += resource.response.body.length if resource.response && resource.response.body
+        size
+      end
+
+      def decompress_body(body, encoding)
+        return body unless body
+        case (encoding || '').downcase
+        when "gzip"
+          Zlib::GzipReader.new(StringIO.new(body)).read.to_s
+        when "deflate"
+          Zlib::Inflate.new.inflate(body)
+        else
+          body
+        end
+      end
+
+      def get_header(headers, name)
+        headers[headers.keys.select{|h| h.casecmp(name) == 0}[0]]
+      end
+
+      def filter_resource(resource)
+        # Make sure the body is not compressed
+        resource.request.body = decompress_body(resource.request.body, get_header(resource.request.headers, "content-encoding")) if resource.request
+        resource.response.body = decompress_body(resource.response.body, get_header(resource.response.headers,"content-encoding")) if resource.response
+
+        # Start applying filters
+        config = get_configuration_for_host(resource.request.headers['host'])
+        debug "Using config for event #{config.to_s}"
+
+        # Filter the headers
+        filter_headers(resource.request.headers, config['sanitizedHeaders']) if resource.request
+        filter_headers(resource.response.headers, config['sanitizedHeaders']) if resource.response
+
+        # Filter the body
+        if config['bodyCapture'] == 'Off' || (config['bodyCapture'] == 'OnError' && !contains_error(resource))
+          resource.request.body = nil if resource.request
+          resource.response.body = nil if resource.response
+        else
+          resource.request.body = filter_body(resource.request.body, config['sanitizedBodyFields']) if resource.request
+          resource.response.body = filter_body(resource.response.body, config['sanitizedBodyFields']) if resource.response
+        end
+
+        # Filter the query params
+        resource.request.url = filter_url(resource.request.url, config['sanitizedQueryParams'])
+
+        # Record this event as filtered in case of retry
+        resource.filtered = true
+
+        resource
+      end
+
+      def contains_error(resource)
+        if !resource.response.nil? && resource.response.status_code >= 400
+          return true
+        end
+        false
+      end
+
+      def filter_url(url, config)
+        uri = URI(url)
+        if uri.query
+          query = CGI::parse(uri.query)
+          query.keys.each{|k| query[k] = 'xxx' if contains_string(config, k)}
+          uri.query = URI.encode_www_form(query)
+          return uri.to_s
+        end
+        url
+      rescue Exception => e
+        error "Error filtering url", e
+        url
+      end
+
+      def filter_headers(headers, names)
+          headers.each { |h| headers[h[0]] = 'xxx' if contains_string(names, h[0]) }
+      end
+
+      def filter_body(body, fields)
+        return nil if body.nil?
+        json = JSON.parse(body)
+        JSON.fast_generate(mask_object_fields(json, fields))
+      rescue Exception => e
+        body
+      end
+
+      def mask_object_fields(obj, fields)
+        field_names = fields.map {|f| f['value'] }
+        case obj
+        when Hash
+          obj.keys.each do |k|
+            if contains_string(field_names, k)
+              obj[k] = 'xxx'
+            else
+              obj[k] = mask_object_fields(obj[k], fields)
+            end
+          end
+        when Array
+          obj.each_index do |i|
+            obj[i] = mask_object_fields(obj[i], fields) if (obj[i].is_a?(Hash) || obj[i].is_a?(Array))
+          end
+        end
+        obj
+      end
+
+      # Get the configuration for a host, either use a defined api or the default account config
+      def get_configuration_for_host(host)
+        api_config = api_configurations.select{ |api| contains_string(api['hosts'], host)}[0]
+        return api_config['configuration'] if api_config
+        account_api_configuration
+      end
+
+      # Parse out the host blacklist
+      def get_account_host_blacklist
+        account_api_configuration['hostBlacklist']
+      end
+
+      # Do a case insensitive search for a string in array
+      def contains_string(arr, a)
+        !arr.select { |b| b.casecmp(a) == 0 }.empty?
+      end
+
+      # Return 'accountApiConfiguration' or empty object
+      def account_api_configuration
+        return {} if config.agentConfig.nil? or config.agentConfig['accountApiConfiguration'].nil?
+        config.agentConfig['accountApiConfiguration']
+      end
+
+      # Return the 'apis' or empty object
+      def api_configurations
+        return {} if config.agentConfig.nil? or config.agentConfig['apis'].nil?
+        config.agentConfig['apis']
+      end
+
+      def host_blacklisted(event)
+        blacklist = get_account_host_blacklist
+        return false if blacklist.nil?
+        is_blacklisted = false
+        unless event.request.nil? || event.request.url.nil?
+          url = URI(event.request.url)
+          blacklist.each do |host|
+            is_blacklisted = true if url.host == host
+            is_blacklisted = true if url.host.end_with? '.'+host
+          end
+        end
+        is_blacklisted
+      end
+    end
+  end
+end

data/lib/hoss/util/inflector.rb

@@ -0,0 +1,110 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+module Hoss
+  # rubocop:disable all
+  module Util
+    # From https://github.com/rails/rails/blob/v5.2.0/activesupport/lib/active_support/inflector/methods.rb#L254-L332
+    module Inflector
+      extend self
+
+      #
+      # Tries to find a constant with the name specified in the argument string.
+      #
+      #   constantize('Module')   # => Module
+      #   constantize('Foo::Bar') # => Foo::Bar
+      #
+      # The name is assumed to be the one of a top-level constant, no matter
+      # whether it starts with "::" or not. No lexical context is taken into
+      # account:
+      #
+      #   C = 'outside'
+      #   module M
+      #     C = 'inside'
+      #     C                # => 'inside'
+      #     constantize('C') # => 'outside', same as ::C
+      #   end
+      #
+      # NameError is raised when the name is not in CamelCase or the constant is
+      # unknown.
+      def constantize(camel_cased_word)
+        names = camel_cased_word.split("::".freeze)
+
+        # Trigger a built-in NameError exception including the ill-formed constant in the message.
+        Object.const_get(camel_cased_word) if names.empty?
+
+        # Remove the first blank element in case of '::ClassName' notation.
+        names.shift if names.size > 1 && names.first.empty?
+
+        names.inject(Object) do |constant, name|
+          if constant == Object
+            constant.const_get(name)
+          else
+            candidate = constant.const_get(name)
+            next candidate if constant.const_defined?(name, false)
+            next candidate unless Object.const_defined?(name)
+
+            # Go down the ancestors to check if it is owned directly. The check
+            # stops when we reach Object or the end of ancestors tree.
+            constant = constant.ancestors.inject(constant) do |const, ancestor|
+              break const    if ancestor == Object
+              break ancestor if ancestor.const_defined?(name, false)
+              const
+            end
+
+            # owner is in Object, so raise
+            constant.const_get(name, false)
+          end
+        end
+      end
+
+      # Tries to find a constant with the name specified in the argument string.
+      #
+      #   safe_constantize('Module')   # => Module
+      #   safe_constantize('Foo::Bar') # => Foo::Bar
+      #
+      # The name is assumed to be the one of a top-level constant, no matter
+      # whether it starts with "::" or not. No lexical context is taken into
+      # account:
+      #
+      #   C = 'outside'
+      #   module M
+      #     C = 'inside'
+      #     C                     # => 'inside'
+      #     safe_constantize('C') # => 'outside', same as ::C
+      #   end
+      #
+      # +nil+ is returned when the name is not in CamelCase or the constant (or
+      # part of it) is unknown.
+      #
+      #   safe_constantize('blargle')                  # => nil
+      #   safe_constantize('UnknownModule')            # => nil
+      #   safe_constantize('UnknownModule::Foo::Bar')  # => nil
+      def safe_constantize(camel_cased_word)
+        constantize(camel_cased_word)
+      rescue NameError => e
+        raise if e.name && !(camel_cased_word.to_s.split("::").include?(e.name.to_s) ||
+                             e.name.to_s == camel_cased_word.to_s)
+      rescue ArgumentError => e
+        raise unless /not missing constant #{const_regexp(camel_cased_word)}!$/.match(e.message)
+      end
+    end
+  end
+  # rubocop:enable all
+end

data/lib/hoss/util/lru_cache.rb

@@ -0,0 +1,65 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+module Hoss
+  module Util
+    # @api private
+    class LruCache
+      def initialize(max_size = 512, &block)
+        @max_size = max_size
+        @data = Hash.new(&block)
+        @mutex = Mutex.new
+      end
+
+      def [](key)
+        @mutex.synchronize do
+          val = @data[key]
+          return unless val
+          add(key, val)
+          val
+        end
+      end
+
+      def []=(key, val)
+        @mutex.synchronize do
+          add(key, val)
+        end
+      end
+
+      def length
+        @data.length
+      end
+
+      def to_a
+        @data.to_a
+      end
+
+      private
+
+      def add(key, val)
+        @data.delete(key)
+        @data[key] = val
+
+        return unless @data.length > @max_size
+
+        @data.delete(@data.first[0])
+      end
+    end
+  end
+end

data/lib/hoss/util/throttle.rb

@@ -0,0 +1,52 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+module Hoss
+  module Util
+    # @api private
+
+    # Usage example:
+    #   Throttle.new(5) { thing to only do once per 5 secs }
+    class Throttle
+      def initialize(buffer_secs, &block)
+        @buffer_secs = buffer_secs
+        @block = block
+      end
+
+      def call
+        if @last_call && seconds_since_last_call < @buffer_secs
+          return @last_result
+        end
+
+        @last_call = now
+        @last_result = @block.call
+      end
+
+      private
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def seconds_since_last_call
+        now - @last_call
+      end
+    end
+  end
+end

data/lib/hoss/util.rb

@@ -0,0 +1,54 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+module Hoss
+  # @api private
+  module Util
+    def self.micros(target = Time.now)
+      utc = target.utc
+      utc.to_i * 1_000_000 + utc.usec
+    end
+
+    def self.monotonic_micros
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
+    end
+
+    def self.git_sha
+      sha = `git rev-parse --verify HEAD 2>&1`.chomp
+      $?&.success? ? sha : nil
+    end
+
+    def self.hex_to_bits(str)
+      str.hex.to_s(2).rjust(str.size * 4, '0')
+    end
+
+    def self.reverse_merge!(first, *others)
+      others.reduce(first) do |curr, other|
+        curr.merge!(other) { |_, _, new| new }
+      end
+    end
+
+    def self.truncate(value, max_length: 1024)
+      return unless value
+      return value if value.length <= max_length
+
+      value[0...(max_length - 1)] + '…'
+    end
+  end
+end

data/lib/hoss/version.rb

@@ -0,0 +1,22 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# frozen_string_literal: true
+
+module Hoss
+  VERSION = '1.0.1'
+end