resurfaceio-logger 1.10.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '081b638ef0c63585d851de54a9b30ef9b54de58cf68f880785c2e1a8d214cb54'
-  data.tar.gz: a71c49ec92a27637f7d77b52177df39d0b197757a056154e5247f5e40aa350d1
+  metadata.gz: 78d67f05915dcaee82ef7e3809856549bf09d01ac1f26d32f4dcb02003d317f3
+  data.tar.gz: f3d3598661aed83877882554183991ef228ac92d3f875f087eafddd3221cfa59
 SHA512:
-  metadata.gz: f2f2c6be12c8977756ab185cc34c64cc29b0956940139315a1d9c023f4df0756cbea1c3fdb8eaab9207070ddda63ee1dac34688d8272d1782d5664b531de94a3
-  data.tar.gz: 8eed5ac226184687025edf8d3f213585b73d1577c8c6a5420f71b4bc5205add88470672e350746edef7e2a48bf3441276517fcb7ec176bdb16f88b21f9059c7a
+  metadata.gz: 5aa5de88c997daddb5f345e4e41d5a173e59adb449dec141ee015747a48ef2f82a7ec623a6b1630d913f5a018f72c4aa1907e310fa1a7808f5c5c579db0281d9
+  data.tar.gz: e998c80806e0488c0790e551285e3ae90b7897aa7e027019ad18fd9c9794f0bb37add6d5aaeddb8eb275e66da18b3b9e21b042af636009966d9b6773f200ae88

data/lib/resurfaceio/all.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 require 'resurfaceio/base_logger'
 require 'resurfaceio/http_logger'

data/lib/resurfaceio/base_logger.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 require 'uri'
 require 'net/http'
@@ -51,7 +51,12 @@ class BaseLogger
       end
     end
 
+    # finalize internal properties
     @enableable = !@queue.nil? || !@url.nil?
+    @submit_failures = 0
+    @submit_failures_lock = Mutex.new
+    @submit_successes = 0
+    @submit_successes_lock = Mutex.new
   end
 
   def agent
@@ -102,10 +107,10 @@ class BaseLogger
 
   def submit(msg)
     if msg.nil? || @skip_submission || !enabled?
-      true
+      # do nothing
     elsif @queue
       @queue << msg
-      true
+      @submit_successes_lock.synchronize { @submit_successes += 1 }
     else
       begin
         @url_parsed ||= URI.parse(@url)
@@ -119,14 +124,26 @@ class BaseLogger
           request.body = Zlib::Deflate.deflate(msg)
         end
         response = @url_connection.request(request)
-        response.code.to_i == 204
+        if response.code.to_i == 204
+          @submit_successes_lock.synchronize { @submit_successes += 1 }
+        else
+          @submit_failures_lock.synchronize { @submit_failures += 1 }
+        end
       rescue SocketError
+        @submit_failures_lock.synchronize { @submit_failures += 1 }
         @url_connection = nil
-        false
       end
     end
   end
 
+  def submit_failures
+    @submit_failures
+  end
+
+  def submit_successes
+    @submit_successes
+  end
+
   def url
     @url
   end

data/lib/resurfaceio/http_logger.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 require 'json'
 require 'resurfaceio/base_logger'
@@ -11,53 +11,20 @@ class HttpLogger < BaseLogger
 
   AGENT = 'http_logger.rb'.freeze
 
-  @@default_rules = HttpRules.strict_rules
-
-  def self.default_rules
-    @@default_rules
-  end
-
-  def self.default_rules=(val)
-    @@default_rules = val.gsub(/^\s*include default\s*$/, '')
-  end
-
-  def self.string_content_type?(s)
-    !s.nil? && !(s =~ /^(text\/(html|plain|xml))|(application\/(json|soap|xml|x-www-form-urlencoded))/i).nil?
-  end
-
   def initialize(options = {})
     super(AGENT, options)
 
-    # read rules from param or defaults
+    # parse specified rules
     if options.respond_to?(:has_key?) && options.has_key?(:rules)
-      @rules = options[:rules].gsub(/^\s*include default\s*$/, @@default_rules)
-      @rules = @@default_rules unless @rules.strip.length > 0
+      @rules = HttpRules.new(options[:rules])
     else
-      @rules = @@default_rules
+      @rules = HttpRules.new(nil)
     end
 
-    # parse and break rules out by verb
-    prs = HttpRules.parse(@rules)
-    @rules_allow_http_url = prs.select {|r| 'allow_http_url' == r.verb}.length > 0
-    @rules_copy_session_field = prs.select {|r| 'copy_session_field' == r.verb}
-    @rules_remove = prs.select {|r| 'remove' == r.verb}
-    @rules_remove_if = prs.select {|r| 'remove_if' == r.verb}
-    @rules_remove_if_found = prs.select {|r| 'remove_if_found' == r.verb}
-    @rules_remove_unless = prs.select {|r| 'remove_unless' == r.verb}
-    @rules_remove_unless_found = prs.select {|r| 'remove_unless_found' == r.verb}
-    @rules_replace = prs.select {|r| 'replace' == r.verb}
-    @rules_sample = prs.select {|r| 'sample' == r.verb}
-    @rules_stop = prs.select {|r| 'stop' == r.verb}
-    @rules_stop_if = prs.select {|r| 'stop_if' == r.verb}
-    @rules_stop_if_found = prs.select {|r| 'stop_if_found' == r.verb}
-    @rules_stop_unless = prs.select {|r| 'stop_unless' == r.verb}
-    @rules_stop_unless_found = prs.select {|r| 'stop_unless_found' == r.verb}
-    @skip_compression = prs.select {|r| 'skip_compression' == r.verb}.length > 0
-    @skip_submission = prs.select {|r| 'skip_submission' == r.verb}.length > 0
-
-    # finish validating rules
-    raise RuntimeError.new('Multiple sample rules') if @rules_sample.length > 1
-    unless @url.nil? || @url.start_with?('https') || @rules_allow_http_url
+    # apply configuration rules
+    @skip_compression = @rules.skip_compression
+    @skip_submission = @rules.skip_submission
+    unless @url.nil? || @url.start_with?('https') || @rules.allow_http_url
       @enableable = false
       @enabled = false
     end
@@ -67,58 +34,21 @@ class HttpLogger < BaseLogger
     @rules
   end
 
-  def log(request, response, response_body = nil, request_body = nil)
-    !enabled? || submit(format(request, response, response_body, request_body))
-  end
-
-  def format(request, response, response_body = nil, request_body = nil, now = nil)
-    details = HttpMessage.build(request, response, response_body, request_body)
+  def submit_if_passing(details)
+    # apply active rules
+    details = @rules.apply(details)
+    return nil if details.nil?
 
-    # copy data from session if configured
-    unless @rules_copy_session_field.empty?
-      ssn = request.session
-      if !ssn.nil? && ssn.respond_to?(:keys)
-        @rules_copy_session_field.each do |r|
-          ssn.keys.each {|d| (details << ["session_field:#{d}", ssn[d].to_s]) if r.param1.match(d)}
-        end
-      end
-    end
-
-    # quit early based on stop rules if configured
-    @rules_stop.each {|r| details.each {|d| return nil if r.scope.match(d[0])}}
-    @rules_stop_if_found.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
-    @rules_stop_if.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
-    passed = 0
-    @rules_stop_unless_found.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
-    return nil if passed != @rules_stop_unless_found.length
-    passed = 0
-    @rules_stop_unless.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
-    return nil if passed != @rules_stop_unless.length
-
-    # do sampling if configured
-    return nil if !@rules_sample[0].nil? && (rand * 100 >= @rules_sample[0].param1)
-
-    # winnow sensitive details based on remove rules if configured
-    @rules_remove.each {|r| details.delete_if {|d| r.scope.match(d[0])}}
-    @rules_remove_unless_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
-    @rules_remove_if_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
-    @rules_remove_unless.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
-    @rules_remove_if.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
-    return nil if details.empty?
-
-    # mask sensitive details based on replace rules if configured
-    @rules_replace.each {|r| details.each {|d| d[1] = d[1].gsub(r.param1, r.param2) if r.scope.match(d[0])}}
-
-    # remove any details with empty values
-    details.delete_if {|d| '' == d[1]}
-    return nil if details.empty?
-
-    # finish message
-    details << ['now', now.nil? ? (Time.now.to_f * 1000).floor.to_s : now]
+    # finalize message
     details << ['agent', @agent]
     details << ['host', @host]
     details << ['version', @version]
-    JSON.generate details
+
+    # let's do this thing
+    submit(JSON.generate(details))
   end
 
+  def self.string_content_type?(s)
+    !s.nil? && !(s =~ /^(text\/(html|plain|xml))|(application\/(json|soap|xml|x-www-form-urlencoded))/i).nil?
+  end
 end

data/lib/resurfaceio/http_logger_for_rack.rb

@@ -1,8 +1,9 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 require 'rack'
 require 'resurfaceio/http_logger'
+require 'resurfaceio/http_message'
 
 class HttpLoggerForRack # http://rack.rubyforge.org/doc/SPEC.html
 
@@ -21,7 +22,7 @@ class HttpLoggerForRack # http://rack.rubyforge.org/doc/SPEC.html
       response = Rack::Response.new(body, status, headers)
       if HttpLogger::string_content_type?(response.content_type)
         request = Rack::Request.new(env)
-        @logger.submit(@logger.format(request, response))
+        HttpMessage.send(logger, request, response) # todo add timing details
       end
     end
     [status, headers, body]

data/lib/resurfaceio/http_logger_for_rails.rb

@@ -1,7 +1,8 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 require 'resurfaceio/http_logger'
+require 'resurfaceio/http_message'
 
 class HttpLoggerForRails
 
@@ -20,7 +21,7 @@ class HttpLoggerForRails
       response = controller.response
       status = response.status
       if (status < 300 || status == 302) && HttpLogger::string_content_type?(response.content_type)
-        @logger.submit(@logger.format(request, response))
+        HttpMessage.send(logger, request, response) # todo add timing details
       end
     end
   end

data/lib/resurfaceio/http_message.rb

@@ -1,10 +1,33 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 require 'json'
 
 class HttpMessage
 
+  def self.send(logger, request, response, response_body = nil, request_body = nil, now = nil, interval = nil)
+    return unless logger.enabled?
+
+    # copy details from request & response
+    message = build(request, response, response_body, request_body)
+
+    # copy details from active session
+    unless logger.rules.copy_session_field.empty?
+      ssn = request.session
+      if !ssn.nil? && ssn.respond_to?(:keys)
+        logger.rules.copy_session_field.each do |r|
+          ssn.keys.each {|d| (message << ["session_field:#{d}", ssn[d].to_s]) if r.param1.match(d)}
+        end
+      end
+    end
+
+    # add timing details
+    message << ['now', now.nil? ? (Time.now.to_f * 1000).floor.to_s : now]
+    message << ['interval', interval] unless interval.nil?
+
+    logger.submit_if_passing(message)
+  end
+
   def self.build(request, response, response_body = nil, request_body = nil)
     message = []
     append_value message, 'request_method', request.request_method unless request.request_method.nil?
@@ -19,8 +42,6 @@ class HttpMessage
     return message
   end
 
-  protected
-
   def self.append_request_headers(message, request)
     respond_to_env = request.respond_to?(:env)
     if respond_to_env || request.respond_to?(:headers)
@@ -74,12 +95,12 @@ class HttpMessage
     unless key.nil?
       unless value.nil?
         case value
-          when Array
-            message << [key, value.join]
-          when String
-            message << [key, value]
-          else
-            message << [key, value.to_s]
+        when Array
+          message << [key, value.join]
+        when String
+          message << [key, value]
+        else
+          message << [key, value.to_s]
         end
       end
     end

data/lib/resurfaceio/http_request_impl.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 class HttpRequestImpl
 

data/lib/resurfaceio/http_response_impl.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 class HttpResponseImpl
 

data/lib/resurfaceio/http_rule.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 class HttpRule
 

data/lib/resurfaceio/http_rules.rb

@@ -1,37 +1,39 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 class HttpRules
 
+  DEBUG_RULES = "allow_http_url\ncopy_session_field /.*/\n".freeze
+
+  STANDARD_RULES = %q(/request_header:cookie|response_header:set-cookie/ remove
+/(request|response)_body|request_param/ replace /[a-zA-Z0-9.!#$%&’*+\/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)/, /x@y.com/
+/request_body|request_param|response_body/ replace /[0-9\.\-\/]{9,}/, /xyxy/
+).freeze
+
+  STRICT_RULES = %q(/request_url/ replace /([^\?;]+).*/, !\\\\1!
+/request_body|response_body|request_param:.*|request_header:(?!user-agent).*|response_header:(?!(content-length)|(content-type)).*/ remove
+).freeze
+
+  @@default_rules = STRICT_RULES
+
+  def self.default_rules
+    @@default_rules
+  end
+
+  def self.default_rules=(val)
+    @@default_rules = val.gsub(/^\s*include default\s*$/, '')
+  end
+
   def self.debug_rules
-    "allow_http_url\ncopy_session_field /.*/\n"
+    DEBUG_RULES
   end
 
   def self.standard_rules
-    %q(/request_header:cookie|response_header:set-cookie/ remove
-/(request|response)_body|request_param/ replace /[a-zA-Z0-9.!#$%&’*+\/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)/, /x@y.com/
-/request_body|request_param|response_body/ replace /[0-9\.\-\/]{9,}/, /xyxy/
-)
+    STANDARD_RULES
   end
 
   def self.strict_rules
-    %q(/request_url/ replace /([^\?;]+).*/, !\\\\1!
-/request_body|response_body|request_param:.*|request_header:(?!user-agent).*|response_header:(?!(content-length)|(content-type)).*/ remove
-)
-  end
-
-  def self.parse(rules)
-    result = []
-    unless rules.nil?
-      rules = rules.gsub(/^\s*include debug\s*$/, debug_rules)
-      rules = rules.gsub(/^\s*include standard\s*$/, standard_rules)
-      rules = rules.gsub(/^\s*include strict\s*$/, strict_rules)
-      rules.each_line do |rule|
-        parsed = parse_rule(rule)
-        result << parsed unless parsed.nil?
-      end
-    end
-    result
+    STRICT_RULES
   end
 
   def self.parse_rule(r)
@@ -76,8 +78,6 @@ class HttpRules
     end
   end
 
-  protected
-
   def self.parse_regex(r, regex)
     s = parse_string(r, regex)
     raise RuntimeError.new("Invalid regex (#{regex}) in rule: #{r}") if '*' == s || '+' == s || '?' == s
@@ -109,6 +109,154 @@ class HttpRules
     raise RuntimeError.new("Invalid expression (#{expr}) in rule: #{r}")
   end
 
+  def initialize(rules)
+    rules = HttpRules.default_rules if rules.nil?
+
+    # todo load rules from external files
+
+    # force default rules if necessary
+    rules = rules.gsub(/^\s*include default\s*$/, HttpRules.default_rules)
+    rules = HttpRules.default_rules unless rules.strip.length > 0
+
+    # expand rule inclues
+    rules = rules.gsub(/^\s*include debug\s*$/, DEBUG_RULES)
+    rules = rules.gsub(/^\s*include standard\s*$/, STANDARD_RULES)
+    rules = rules.gsub(/^\s*include strict\s*$/, STRICT_RULES)
+    @text = rules
+
+    # parse all rules
+    prs = []
+    rules.each_line do |rule|
+      parsed = HttpRules.parse_rule(rule)
+      prs << parsed unless parsed.nil?
+    end
+    @length = prs.length
+
+    # break out rules by verb
+    @allow_http_url = prs.select {|r| 'allow_http_url' == r.verb}.length > 0
+    @copy_session_field = prs.select {|r| 'copy_session_field' == r.verb}
+    @remove = prs.select {|r| 'remove' == r.verb}
+    @remove_if = prs.select {|r| 'remove_if' == r.verb}
+    @remove_if_found = prs.select {|r| 'remove_if_found' == r.verb}
+    @remove_unless = prs.select {|r| 'remove_unless' == r.verb}
+    @remove_unless_found = prs.select {|r| 'remove_unless_found' == r.verb}
+    @replace = prs.select {|r| 'replace' == r.verb}
+    @sample = prs.select {|r| 'sample' == r.verb}
+    @skip_compression = prs.select {|r| 'skip_compression' == r.verb}.length > 0
+    @skip_submission = prs.select {|r| 'skip_submission' == r.verb}.length > 0
+    @stop = prs.select {|r| 'stop' == r.verb}
+    @stop_if = prs.select {|r| 'stop_if' == r.verb}
+    @stop_if_found = prs.select {|r| 'stop_if_found' == r.verb}
+    @stop_unless = prs.select {|r| 'stop_unless' == r.verb}
+    @stop_unless_found = prs.select {|r| 'stop_unless_found' == r.verb}
+
+    # validate rules
+    raise RuntimeError.new('Multiple sample rules') if @sample.length > 1
+  end
+
+  def allow_http_url
+    @allow_http_url
+  end
+
+  def copy_session_field
+    @copy_session_field
+  end
+
+  def length
+    @length
+  end
+
+  def remove
+    @remove
+  end
+
+  def remove_if
+    @remove_if
+  end
+
+  def remove_if_found
+    @remove_if_found
+  end
+
+  def remove_unless
+    @remove_unless
+  end
+
+  def remove_unless_found
+    @remove_unless_found
+  end
+
+  def replace
+    @replace
+  end
+
+  def sample
+    @sample
+  end
+
+  def skip_compression
+    @skip_compression
+  end
+
+  def skip_submission
+    @skip_submission
+  end
+
+  def stop
+    @stop
+  end
+
+  def stop_if
+    @stop_if
+  end
+
+  def stop_if_found
+    @stop_if_found
+  end
+
+  def stop_unless
+    @stop_unless
+  end
+
+  def stop_unless_found
+    @stop_unless_found
+  end
+
+  def text
+    @text
+  end
+
+  def apply(details)
+    # stop rules come first
+    @stop.each {|r| details.each {|d| return nil if r.scope.match(d[0])}}
+    @stop_if_found.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
+    @stop_if.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
+    passed = 0
+    @stop_unless_found.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
+    return nil if passed != @stop_unless_found.length
+    passed = 0
+    @stop_unless.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
+    return nil if passed != @stop_unless.length
+
+    # do sampling if configured
+    return nil if !@sample[0].nil? && (rand * 100 >= @sample[0].param1)
+
+    # winnow sensitive details based on remove rules if configured
+    @remove.each {|r| details.delete_if {|d| r.scope.match(d[0])}}
+    @remove_unless_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
+    @remove_if_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
+    @remove_unless.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
+    @remove_if.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
+    return nil if details.empty?
+
+    # mask sensitive details based on replace rules if configured
+    @replace.each {|r| details.each {|d| d[1] = d[1].gsub(r.param1, r.param2) if r.scope.match(d[0])}}
+
+    # remove any details with empty values
+    details.delete_if {|d| '' == d[1]}
+    details.empty? ? nil : details
+  end
+
   REGEX_ALLOW_HTTP_URL = /^\s*allow_http_url\s*(#.*)?$/.freeze
   REGEX_BLANK_OR_COMMENT = /^\s*([#].*)*$/.freeze
   REGEX_COPY_SESSION_FIELD = /^\s*copy_session_field\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze

data/lib/resurfaceio/usage_loggers.rb

@@ -1,5 +1,5 @@
 # coding: utf-8
-# © 2016-2019 Resurface Labs Inc.
+# © 2016-2020 Resurface Labs Inc.
 
 class UsageLoggers
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: resurfaceio-logger
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 2.0.0
 platform: ruby
 authors:
 - RobDickinson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-06 00:00:00.000000000 Z
+date: 2020-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler