supergood 0.1.5 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a20e9c78ef66594eaed117db23b1581b3531497aaf3a087ad92466f76a8e07a8
-  data.tar.gz: 22e62c82c3cea5bfba10a8d5199ebf1a385bab78861c6f20efcd55921e65eee0
+  metadata.gz: 5ee319f07b495599aa2deafcf5b6c995402e55b4d7a58ddb82d55379aa6ba99a
+  data.tar.gz: 15ab686e3bec0c37771e8fa2892f7cdb69312595cc8726c0d690855681eeb26d
 SHA512:
-  metadata.gz: da801337729a2975cd42231da0de2e773aafdcc0a391a69111b4acb55dfe1d8126e1017e72481688dcf410d23a072c51204e9885e8938d9903ed1f947de94bfc
-  data.tar.gz: b9848d4363bcb7b04bb19b46dc61f8adcb8bbf9b911664a3ba367721f171e2ad950c3fdcb3d236192fe5c160e4d33aa926615583647c7b3e44b0038fc4ed6c27
+  metadata.gz: 11eed3208aeec8618d71cb7a832988ef0e88cfb1d55fed315b203418cb93ab4bfba251b22ef978afd0ec672b4b849d2af28ad03435d20003db1020e154faa74c
+  data.tar.gz: 583dacf60c012289be925e8c49471a14d401be64a00ef0cf299e758797bd318dadb0bb3d3a64cb53ef925ba0620fe253213d6406e1ffc35e094c0c5b5e69a239

data/.vscode/settings.json

@@ -1,6 +1,6 @@
 {
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": "explicit"
   },
   "editor.tabSize": 2,
 }

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    supergood (0.1.3)
+    supergood (1.0.0)
       rudash (~> 4.0, >= 4.0.2)
 
 GEM

data/lib/supergood/api.rb

@@ -8,7 +8,9 @@ module Supergood
       @base_url = base_url
       @header_options = {
         'Content-Type' => 'application/json',
-        'Authorization' => 'Basic ' + Base64.encode64(client_id + ':' + client_secret).gsub(/\n/, '')
+        'Authorization' => "Basic #{Base64.encode64("#{client_id}:#{client_secret}").gsub(/\n/, '')}",
+        'supergood-api' => 'supergood-rb',
+        'supergood-api-version' => VERSION
       }
       @local_only = client_id == LOCAL_CLIENT_ID && client_secret == LOCAL_CLIENT_SECRET
     end
@@ -29,11 +31,12 @@ module Supergood
       if @local_only
         @log.debug(payload)
       else
-        uri = URI(@base_url + '/events')
+        uri = URI("#{@base_url}/events")
         response = Net::HTTP.post(uri, payload.to_json, @header_options)
-        if response.code == '200'
-          return JSON.parse(response.body, symbolize_names: true)
-        elsif response.code == '401'
+
+        return JSON.parse(response.body) if response.code == '200'
+
+        if response.code == '401'
           raise SupergoodException.new ERRORS[:UNAUTHORIZED]
         elsif response.code != '200' && response.code != '201'
           raise SupergoodException.new ERRORS[:POSTING_EVENTS]
@@ -45,14 +48,20 @@ module Supergood
       if @local_only
         @log.debug(payload)
       else
-        uri = URI(@base_url + '/errors')
+        uri = URI("#{@base_url}/errors")
         response = Net::HTTP.post(uri, payload.to_json, @header_options)
-        if response.code == '200'
-          return JSON.parse(response.body, symbolize_names: true)
-        else
-          @log.warn(ERRORS[:POSTING_ERRORS])
-        end
+        return JSON.parse(response.body, symbolize_names: true) if response.code == '200'
+
+        @log.warn(ERRORS[:POSTING_ERRORS])
       end
     end
+
+    def get_remote_config
+      uri = URI(@base_url + '/config')
+      response = Net::HTTP.get_response(uri, @header_options)
+      return JSON.parse(response.body) if response.code == '200'
+
+      raise SupergoodException.new ERRORS[:CONFIG_FETCH_ERROR]
+    end
   end
 end

data/lib/supergood/client.rb

@@ -31,7 +31,6 @@ module Supergood
 
       @allowed_domains = @config[:allowedDomains]
       @ignored_domains = @config[:ignoredDomains]
-      @keys_to_hash = @config[:keysToHash]
       @logger = Supergood::Logger.new(@api, @config, @api.header_options)
 
       @api.set_logger(@logger)
@@ -40,13 +39,15 @@ module Supergood
       @response_cache = {}
 
       @interval_thread = set_interval(@config[:flushInterval]) { flush_cache }
+      @remote_config_thread = set_interval(@config[:remoteConfigFetchInterval]) { fetch_and_process_remote_config }
 
       @http_clients = [
         Supergood::Vendor::NetHTTP,
         Supergood::Vendor::HTTPrb
       ]
 
-      patch_all()
+      fetch_and_process_remote_config
+      patch_all
       self
     end
 
@@ -58,6 +59,15 @@ module Supergood
       @api
     end
 
+    def fetch_and_process_remote_config
+      begin
+        remote_config = @api.get_remote_config
+        @config = @config.merge({ :remote_config => Supergood::Utils.process_remote_config(remote_config) })
+      rescue => e
+        log.error({}, e, ERRORS[:CONFIG_FETCH_ERROR])
+      end
+    end
+
     def flush_cache(force = false)
       # If there's notthing in the response cache, and we're not forcing a flush, then return
 
@@ -67,11 +77,8 @@ module Supergood
         return
       end
 
-      data = @response_cache.values
-
-      if force
-        data += @request_cache.values
-      end
+      data = Supergood::Utils.prepare_data(@response_cache.values, @config[:remote_config], @config[:forceRedactAll])
+      data += Supergood::Utils.prepare_data(@request_cache.values, @config[:remote_config], @config[:forceRedactAll]) if force
 
       begin
         api.post_events(data)
@@ -86,6 +93,7 @@ module Supergood
 
     def cleanup()
       @interval_thread.kill
+      @remote_config_thread.kill
       unpatch_all()
     end
 
@@ -117,16 +125,25 @@ module Supergood
     end
 
     def intercept(request)
+      remote_config = @config[:remote_config]
+
+      if remote_config.nil?
+        response = yield
+        return response[:original_response]
+      end
+
       request_id = SecureRandom.uuid
       requested_at = Time.now
-      if !ignored?(request[:domain])
-        puts "Caching Request"
+
+      endpoint_config = Supergood::Utils.get_endpoint_config(request.transform_keys(&:to_s), remote_config)
+      ignore_endpoint = endpoint_config ? endpoint_config['ignored'] : false
+
+      if !ignore_endpoint && !ignored?(request[:domain])
         cache_request(request_id, requested_at, request)
       end
 
       response = yield
-      if !ignored?(request[:domain]) && defined?(response)
-        puts "Caching Response"
+      if !ignore_endpoint && !ignored?(request[:domain]) && defined?(response)
         cache_response(request_id, requested_at, response)
       end
 
@@ -134,41 +151,47 @@ module Supergood
     end
 
     def cache_request(request_id, requested_at, request)
+      if !@config[:remote_config]
+        return
+      end
+
       begin
         request_payload = {
-          id: request_id,
-          headers: request[:headers],
-          method: request[:method],
-          url: request[:url],
-          path: request[:path],
-          search: request[:search] || '',
-          body: Supergood::Utils.safe_parse_json(request[:body]),
-          requestedAt: requested_at
+          'id' => request_id,
+          'headers' => Supergood::Utils.safe_parse_json(request[:headers]),
+          'method' => request[:method],
+          'url' => request[:url],
+          'path' => request[:path],
+          'search' => request[:search] || '',
+          'body' => Supergood::Utils.safe_parse_json(request[:body]),
+          'requestedAt' => requested_at
         }
         @request_cache[request_id] = {
-          request: request_payload
+          'request' => request_payload
         }
       rescue => e
-        log.error({ request: request }, e, ERRORS[:CACHING_REQUEST])
+        log.error({ 'request' => request }, e, ERRORS[:CACHING_REQUEST])
       end
     end
 
     def cache_response(request_id, requested_at, response)
+      if !@config[:remote_config]
+        return
+      end
+
       begin
         responded_at = Time.now
         duration = (responded_at - requested_at) * 1000
         request_payload = @request_cache[request_id]
         response_payload = {
-          headers: response[:headers],
-          status: response[:status],
-          statusText: response[:statusText],
-          body: Supergood::Utils.safe_parse_json(response[:body]),
-          respondedAt: responded_at,
-          duration: duration.round,
+          'headers' => Supergood::Utils.safe_parse_json(response[:headers]),
+          'status' => response[:status],
+          'statusText' => response[:statusText],
+          'body' => Supergood::Utils.safe_parse_json(response[:body]),
+          'respondedAt' => responded_at,
+          'duration' => duration.round
         }
-        @response_cache[request_id] = Supergood::Utils.hash_values_from_keys(request_payload.merge({
-          response: response_payload
-        }), @keys_to_hash)
+        @response_cache[request_id] = request_payload.merge({ 'response' => response_payload })
         @request_cache.delete(request_id)
       rescue => e
         log.error(
@@ -179,7 +202,7 @@ module Supergood
     end
 
     def ignored?(domain)
-      base_domain = URI.parse(@base_url).hostname
+      base_domain = Supergood::Utils.get_host_without_www(@base_url)
       if domain == base_domain
         return true
       elsif @allowed_domains.any?

data/lib/supergood/constants.rb

@@ -5,7 +5,8 @@ ERRORS = {
   POSTING_EVENTS: 'Error Posting Events',
   POSTING_ERRORS: 'Error Posting Errors',
   WRITING_TO_DISK: 'Error writing to disk',
-  TEST_ERROR: 'Test Error for Testing Purpos es',
+  TEST_ERROR: 'Test Error for Testing Purposes',
+  CONFIG_FETCH_ERROR: 'Error Fetching Remote Config',
   UNAUTHORIZED: 'Unauthorized: Invalid Client ID or Secret. Exiting.',
   NO_CLIENT_ID:
     'No Client ID Provided, set SUPERGOOD_CLIENT_ID or pass it as an argument',
@@ -16,13 +17,14 @@ ERRORS = {
 LOCAL_CLIENT_ID = 'local-client-id';
 LOCAL_CLIENT_SECRET = 'local-client-secret';
 
-DEFAULT_SUPERGOOD_BYTE_LIMIT = 500000
+DEFAULT_SUPERGOOD_BYTE_LIMIT = 500_000
 
 DEFAULT_CONFIG = {
-  keysToHash: [],
   flushInterval: 1000,
+  remoteConfigFetchInterval: 10_000,
   ignoredDomains: [],
   allowedDomains: [],
+  forceRedactAll: true
 }
 
 # GZIP_START_BYTES = b'\x1f\x8b'

data/lib/supergood/logger.rb

@@ -13,7 +13,9 @@ module Supergood
     end
 
     def error(data, error, msg)
-      super(error)
+      if(ENV['SUPERGOOD_LOG_LEVEL'] == 'debug')
+        super(error)
+      end
       @api.post_errors(
         {
           error: error.backtrace.join('\n'),

data/lib/supergood/utils.rb

@@ -1,62 +1,28 @@
 require 'rudash'
 require 'digest'
+require 'uri'
+require 'json'
 
 module Supergood
   module Utils
-    def self.hash_value(input)
-      hash = Digest::SHA1.new
-      if input == nil
-        return ''
-      elsif input.class == Array
-        return [Base64.strict_encode64(hash.update(input.to_json).to_s)]
-      elsif input.class == Hash
-        return {'hashed': Base64.strict_encode64(hash.update(input.to_json).to_s)}
-      elsif input.class == String
-        return Base64.strict_encode64(hash.update(input).to_s)
-      end
-    end
-
-    # Hash values from specified keys, or hash if the bodies exceed a byte limit
-    def self.hash_values_from_keys(obj, keys_to_hash, byte_limit=DEFAULT_SUPERGOOD_BYTE_LIMIT)
-      _obj = obj
-
-      if !keys_to_hash.include?('response.body')
-        payload = R_.get(_obj, 'response.body')
-        payload_size = payload.to_s.length()
-        if(payload_size >= byte_limit)
-          R_.set(_obj, 'response.body', Supergood::Utils.hash_value(payload))
-        end
-      end
-
-      if !keys_to_hash.include?('request.body')
-        payload = R_.get(_obj, 'request.body')
-        payload_size = payload.to_s.length()
-        if(payload_size >= byte_limit)
-          R_.set(_obj, 'request.body', Supergood::Utils.hash_value(payload))
-        end
-      end
-
-      keys_to_hash.each { |key|
-        value = R_.get(_obj, key)
-        if !!value
-          R_.set(_obj, key, Supergood::Utils.hash_value(value))
-        end
-      }
 
-      return _obj
+    def self.get_host_without_www(url)
+      uri = URI.parse(url)
+      uri = URI.parse("http://#{url}") if uri.scheme.nil?
+      host = uri.host.downcase
+      host.start_with?('www.') ? host[4..] : host
     end
 
     def self.safe_parse_json(input)
-      if !input || input == ''
-        return ''
-      end
+      return '' if !input || input == ''
 
       begin
-        return JSON.parse(input)
+        JSON.parse(input)
       rescue => e
         input
       end
     end
+
     def self.get_header(request_or_response)
       header = {}
       request_or_response.each_header do |k,v|
@@ -66,11 +32,250 @@ module Supergood
     end
 
     def self.request_url(http, request)
-      URI::DEFAULT_PARSER.unescape("http#{"s" if http.use_ssl?}://#{http.address}#{request.path}")
+      URI::DEFAULT_PARSER.unescape("http#{'s' if http.use_ssl?}://#{http.address}#{request.path}")
     end
 
     def self.make_config(config)
-      return DEFAULT_CONFIG.merge(config)
+      DEFAULT_CONFIG.merge(config)
+    end
+
+    def self.process_remote_config(remote_config_payload)
+      remote_config_payload ||= []
+      remote_config_payload.reduce({}) do |remote_config, domain_config|
+        domain = domain_config['domain']
+        endpoints = domain_config['endpoints']
+        endpoint_config = endpoints.reduce({}) do |config, endpoint|
+          matching_regex = endpoint['matchingRegex']
+          regex = matching_regex['regex']
+          location = matching_regex['location']
+
+          endpoint_configuration = endpoint['endpointConfiguration']
+          action = endpoint_configuration['action']
+          sensitive_keys = endpoint_configuration['sensitiveKeys'] || []
+          sensitive_keys = sensitive_keys.map { |key| key['keyPath'] }
+
+          config[regex] = {
+            'location' => location,
+            'regex' => regex,
+            'ignored' => action == 'Ignore',
+            'sensitive_keys' => sensitive_keys
+          }
+
+          config
+        end
+
+        remote_config[domain] = endpoint_config
+        remote_config
+      end
+    end
+
+    def self.get_str_representation_from_path(request, location)
+      url = URI(request['url'])
+
+      case location
+      when 'domain'
+        get_host_without_www(url)
+      when 'url'
+        url.to_s
+      when 'path'
+        url.path
+      when 'requestHeaders'
+        request['headers'].to_s
+      when 'requestBody'
+        request['body'].to_s
+      else
+        request[location.to_sym].to_s if request.key?(location.to_sym)
+      end
+    end
+
+    def self.get_endpoint_config(request, remote_config)
+      domain = remote_config.keys.find { |d| get_host_without_www(request['url']).include?(d) }
+      return nil unless domain
+
+      endpoint_configs = remote_config[domain]
+      endpoint_configs.each_value do |endpoint_config|
+        regex = endpoint_config['regex']
+        location = endpoint_config['location']
+        regex_obj = Regexp.new(regex)
+        str_representation = get_str_representation_from_path(request, location)
+        next unless str_representation
+        return endpoint_config if regex_obj.match?(str_representation)
+      end
+      nil
+    end
+
+    def self.expand(parts, obj, key_path)
+      path = key_path
+      return [path] if parts.empty?
+
+      part = parts.first
+      is_property = !part.start_with?('[')
+      separator = !path.empty? && is_property ? '.' : ''
+
+      # Check for array notations
+      if part.match?(/\[\*?\]/)
+        return [] unless obj.is_a?(Array)
+
+        # Expand for each element in the array
+        obj.flat_map.with_index do |_, index|
+          expand(parts[1..], obj[index], "#{path}#{separator}[#{index}]")
+        end
+      elsif part.start_with?('[') && part.end_with?(']')
+        # Specific index in the array
+        index = part[1...-1].to_i
+        if index.is_a?(Numeric) && index < obj.length
+          expand(parts[1..], obj[index], "#{path}#{separator}#{part}")
+        else
+          []
+        end
+      else
+        if obj && obj.is_a?(Hash) && obj.key?(part)
+          expand(parts[1..], obj[part], "#{path}#{separator}#{part}")
+        else
+          []
+        end
+      end
+    end
+
+    def self.expand_key(key, obj)
+      parts = key.scan(/[^.\[\]]+|\[\d*\]|\[\*\]/) || []
+      expand(parts, obj, '')
+    end
+
+    def self.expand_sensitive_key_set_for_arrays(obj, sensitive_keys)
+      sensitive_keys.flat_map { |key| expand_key(key, obj) }
+    end
+
+    def self.marshal_key_path(keypath)
+      keypath.gsub(/^requestHeaders/, 'request.headers')
+             .gsub(/^requestBody/, 'request.body')
+             .gsub(/^responseHeaders/, 'response.headers')
+             .gsub(/^responseBody/, 'response.body')
+    end
+
+    def self.unmarshal_key_path(keypath)
+      keypath.gsub(/^request\.headers/, 'requestHeaders')
+             .gsub(/^request\.body/, 'requestBody')
+             .gsub(/^response\.headers/, 'responseHeaders')
+             .gsub(/^response\.body/, 'responseBody')
+    end
+
+    def self.set_value_to_nil(hash, key_path)
+      keys = key_path.split('.')
+      current_key = keys.first
+      index = current_key.match(/\[(\d+)\]/)
+
+      index = index[1].to_i if index
+
+      # Convert current_key to symbol if necessary
+      current_key = current_key.gsub(/\[\d+\]/, '') if index
+
+      return hash unless hash.keys.include?(current_key)
+
+      if keys.length == 1
+        index ? hash[current_key][index] = nil : hash[current_key] = nil
+      elsif hash[current_key].is_a?(Hash)
+        set_value_to_nil(hash[current_key], keys[1..].join('.'))
+      elsif hash[current_key].is_a?(Array)
+        set_value_to_nil(hash[current_key][index], keys[1..].join('.'))
+      end
+
+      hash
+    end
+
+    def self.find_leaf_key_paths(structure, current_path = [])
+      key_paths = []
+
+      if structure.is_a?(Hash)
+        # Iterate through each key-value pair in the hash
+        structure.each do |key, value|
+          # Recursively find key paths in the value
+          key_paths += find_leaf_key_paths(value, current_path + [key.to_s])
+        end
+      elsif structure.is_a?(Array)
+        # Iterate through each element in the array
+        structure.each_with_index do |element, index|
+          # Modify how indices are appended to the path
+          # Check if the last element in the current_path is a hash key or an array index
+          if current_path.last && current_path.last.include?('[')
+            new_path = current_path[0...-1] + ["#{current_path.last}[#{index}]"]
+          else
+            new_path = current_path + ["[#{index}]"]
+          end
+
+          # Recursively find key paths in the element
+          key_paths += find_leaf_key_paths(element, new_path)
+        end
+      else
+        # Leaf node: construct the key path and add it to the list
+        key_path = current_path.join('.').gsub('.[', '[')
+        key_paths << key_path unless key_path.empty?
+      end
+
+      key_paths
+    end
+
+    def self.redact_values_from_keys(event, remote_config, force_redact_all)
+      sensitive_key_metadata = []
+      endpoint_config = get_endpoint_config(event['request'], remote_config)
+
+      unless (endpoint_config && endpoint_config['sensitive_keys'].any?) || force_redact_all
+        return { 'event' => event, 'sensitive_key_metadata' => sensitive_key_metadata }
+      end
+
+      if force_redact_all
+        # Need response.body in path
+        sensitive_keys = find_leaf_key_paths(event['response']['body'], ['response', 'body'])
+        sensitive_keys += find_leaf_key_paths(event['request']['body'], ['request', 'body'])
+        sensitive_keys += find_leaf_key_paths(event['request']['headers'], ['request', 'headers'])
+        sensitive_keys += find_leaf_key_paths(event['response']['headers'], ['response', 'headers'])
+      else
+        sensitive_keys = endpoint_config['sensitive_keys']
+      end
+
+      sensitive_keys = expand_sensitive_key_set_for_arrays(
+        event, sensitive_keys.map { |key| marshal_key_path(key) }
+      )
+      sensitive_keys.each do |key_path|
+        value = R_.get(event, key_path)
+        event = set_value_to_nil(event, key_path)
+        # Add sensitive key for array expansion
+        sensitive_key_metadata << { 'keyPath' => unmarshal_key_path(key_path) }.merge(redact_value(value))
+      end
+
+      { 'event' => event, 'sensitive_key_metadata' => sensitive_key_metadata }
+    end
+
+    def self.redact_value(input)
+      data_length = 0
+      data_type = 'null'
+      case input
+      when Array
+        data_length = input.size
+        data_type = 'array'
+      when Hash
+        data_length = input.to_json.bytesize
+        data_type = 'object'
+      when String
+        data_length = input.size
+        data_type = 'string'
+      when Numeric
+        data_length = input.to_s.size
+        data_type = input.integer? ? 'integer' : 'float'
+      when TrueClass, FalseClass # This is a better way to check for booleans
+        data_length = 1
+        data_type = 'boolean'
+      end
+      { 'length' => data_length, 'type' => data_type }
+    end
+
+    def self.prepare_data(events, remote_config, force_redact_all)
+      events.map do |event|
+        redacted_event_with_metadata = redact_values_from_keys(event, remote_config, force_redact_all)
+        redacted_event_with_metadata['event'].merge(
+          'metadata': { 'sensitiveKeys': redacted_event_with_metadata['sensitive_key_metadata'] }
+        )
+      end
     end
   end
 end

data/lib/supergood/vendors/http.rb

@@ -14,7 +14,7 @@ module Supergood
                 url: original_request_payload.uri.to_s,
                 path: original_request_payload.uri.path,
                 search: original_request_payload.uri.query,
-                domain: original_request_payload.uri.host
+                domain: Supergood::Utils.get_host_without_www(original_request_payload.uri.host)
               }
               Supergood.intercept(request) do
                 original_response = original_perform(original_request_payload, original_options)

data/lib/supergood/vendors/net-http.rb

@@ -10,7 +10,7 @@ module Supergood
           block = lambda do |x|
             alias original_request_method request
             def request(original_request_payload, body = nil, &block)
-              http = self;
+              http = self
               url = Supergood::Utils.request_url(http, original_request_payload)
               uri = URI.parse(url)
               request = {
@@ -20,7 +20,7 @@ module Supergood
                 url: url,
                 path: original_request_payload.path,
                 search: uri.query,
-                domain: uri.host,
+                domain: Supergood::Utils.get_host_without_www(uri.host)
               }
               Supergood.intercept(request) do
                 original_response = original_request_method(original_request_payload, body, &block)

data/lib/supergood/version.rb

@@ -1,3 +1,3 @@
 module Supergood
-  VERSION = '0.1.5'.freeze
+  VERSION = '1.0.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: supergood
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 1.0.0
 platform: ruby
 authors:
 - Alex Klarfeld