resurfaceio-logger 1.8.4 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a583801501358c56e7f049b70bb41f3695612f7a
-  data.tar.gz: a3db5f5fad430ecadbcef92b75011ec62e478d54
+  metadata.gz: 95113134bbf15fba0ef898a514d24a24dd98da6e
+  data.tar.gz: 998377326921781e89c99e9905ebef0c6f932fc8
 SHA512:
-  metadata.gz: a65d4f79c84c6c8b3768184c205e527e67704d963e154db0f4d1b1ff3d7ab9a8c3f8f09623a19d55f5eaae194830a3e1cfb0ceffd481cbe581f76050c82ce3ab
-  data.tar.gz: 47a2f99ed81a16fbddd0248f743a63a124a67da57f8f2ba02363cf954f563bc3039b34f06f926685ad5fb3db527fa79eb901c5af13adbe1c405e5306c1ef73f3
+  metadata.gz: d64d21c53d46de1336ac33fc81f71e3ed1354709c8ec91fc239ed964c1e7d73fa7fc484b42223077836b6a2be5bdf9ad851d406c2c1cf72b82d8b2b00c544569
+  data.tar.gz: d7bac6e321c7d292e2190f15500e4df3e6d18dd0458bb221c565573fea7a19073501261d093c1dbef45f684bc2882d74720b981b84b68f6817e6653a0beb1c62

data/lib/resurfaceio/all.rb

@@ -5,9 +5,11 @@ require 'resurfaceio/base_logger'
 require 'resurfaceio/http_logger'
 require 'resurfaceio/http_logger_for_rack'
 require 'resurfaceio/http_logger_for_rails'
-require 'resurfaceio/http_message_impl'
+require 'resurfaceio/http_message'
 require 'resurfaceio/http_request_impl'
 require 'resurfaceio/http_response_impl'
+require 'resurfaceio/http_rule'
+require 'resurfaceio/http_rules'
 require 'resurfaceio/usage_loggers'
 
 module Resurfaceio
@@ -24,7 +26,7 @@ module Resurfaceio
   class HttpLoggerForRails < HttpLoggerForRails
   end
 
-  class HttpMessageImpl < HttpMessageImpl
+  class HttpMessage < HttpMessage
   end
 
   class HttpRequestImpl < HttpRequestImpl
@@ -33,6 +35,12 @@ module Resurfaceio
   class HttpResponseImpl < HttpResponseImpl
   end
 
+  class HttpRule < HttpRule
+  end
+
+  class HttpRules < HttpRules
+  end
+
   class UsageLoggers < UsageLoggers
   end
 

data/lib/resurfaceio/base_logger.rb

@@ -91,7 +91,7 @@ class BaseLogger
   end
 
   def submit(json)
-    if @skip_submission || !enabled?
+    if json.nil? || @skip_submission || !enabled?
       true
     elsif @queue
       @queue << json

data/lib/resurfaceio/http_logger.rb

@@ -3,30 +3,121 @@
 
 require 'json'
 require 'resurfaceio/base_logger'
-require 'resurfaceio/http_message_impl'
+require 'resurfaceio/http_message'
+require 'resurfaceio/http_rule'
+require 'resurfaceio/http_rules'
 
 class HttpLogger < BaseLogger
 
   AGENT = 'http_logger.rb'.freeze
 
+  @@default_rules = HttpRules.strict_rules
+
+  def self.default_rules
+    @@default_rules
+  end
+
+  def self.default_rules=(val)
+    @@default_rules = val.gsub(/^\s*include default\s*$/, '')
+  end
+
   def self.string_content_type?(s)
     !s.nil? && !(s =~ /^(text\/(html|plain|xml))|(application\/(json|soap|xml|x-www-form-urlencoded))/i).nil?
   end
 
   def initialize(options = {})
     super(AGENT, options)
+
+    # read rules from param or defaults
+    if options.respond_to?(:has_key?) && options.has_key?(:rules)
+      @rules = options[:rules].gsub(/^\s*include default\s*$/, @@default_rules)
+      @rules = @@default_rules unless @rules.strip.length > 0
+    else
+      @rules = @@default_rules
+    end
+
+    # parse and break rules out by verb
+    prs = HttpRules.parse(@rules)
+    @rules_allow_http_url = prs.select {|r| 'allow_http_url' == r.verb}.length > 0
+    @rules_copy_session_field = prs.select {|r| 'copy_session_field' == r.verb}
+    @rules_remove = prs.select {|r| 'remove' == r.verb}
+    @rules_remove_if = prs.select {|r| 'remove_if' == r.verb}
+    @rules_remove_if_found = prs.select {|r| 'remove_if_found' == r.verb}
+    @rules_remove_unless = prs.select {|r| 'remove_unless' == r.verb}
+    @rules_remove_unless_found = prs.select {|r| 'remove_unless_found' == r.verb}
+    @rules_replace = prs.select {|r| 'replace' == r.verb}
+    @rules_sample = prs.select {|r| 'sample' == r.verb}
+    @rules_stop = prs.select {|r| 'stop' == r.verb}
+    @rules_stop_if = prs.select {|r| 'stop_if' == r.verb}
+    @rules_stop_if_found = prs.select {|r| 'stop_if_found' == r.verb}
+    @rules_stop_unless = prs.select {|r| 'stop_unless' == r.verb}
+    @rules_stop_unless_found = prs.select {|r| 'stop_unless_found' == r.verb}
+    @skip_compression = prs.select {|r| 'skip_compression' == r.verb}.length > 0
+    @skip_submission = prs.select {|r| 'skip_submission' == r.verb}.length > 0
+
+    # finish validating rules
+    raise RuntimeError.new('Multiple sample rules') if @rules_sample.length > 1
+    unless @url.nil? || @url.start_with?('https') || @rules_allow_http_url
+      @enableable = false
+      @enabled = false
+    end
   end
 
-  def format(request, response, response_body = nil, request_body = nil, now = nil)
-    message = HttpMessageImpl.build(request, response, response_body, request_body)
-    message << ['agent', @agent]
-    message << ['version', @version]
-    message << ['now', now.nil? ? (Time.now.to_f * 1000).floor.to_s : now]
-    JSON.generate message
+  def rules
+    @rules
   end
 
   def log(request, response, response_body = nil, request_body = nil)
     !enabled? || submit(format(request, response, response_body, request_body))
   end
 
+  def format(request, response, response_body = nil, request_body = nil, now = nil)
+    details = HttpMessage.build(request, response, response_body, request_body)
+
+    # copy data from session if configured
+    unless @rules_copy_session_field.empty?
+      ssn = request.session
+      if !ssn.nil? && ssn.respond_to?(:keys)
+        @rules_copy_session_field.each do |r|
+          ssn.keys.each {|d| (details << ["session_field:#{d}", ssn[d].to_s]) if r.param1.match(d)}
+        end
+      end
+    end
+
+    # quit early based on stop rules if configured
+    @rules_stop.each {|r| details.each {|d| return nil if r.scope.match(d[0])}}
+    @rules_stop_if_found.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
+    @rules_stop_if.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
+    passed = 0
+    @rules_stop_unless_found.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
+    return nil if passed != @rules_stop_unless_found.length
+    passed = 0
+    @rules_stop_unless.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
+    return nil if passed != @rules_stop_unless.length
+
+    # do sampling if configured
+    return nil if !@rules_sample[0].nil? && (rand * 100 >= @rules_sample[0].param1)
+
+    # winnow sensitive details based on remove rules if configured
+    @rules_remove.each {|r| details.delete_if {|d| r.scope.match(d[0])}}
+    @rules_remove_unless_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
+    @rules_remove_if_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
+    @rules_remove_unless.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
+    @rules_remove_if.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
+    return nil if details.empty?
+
+    # mask sensitive details based on replace rules if configured
+    @rules_replace.each {|r| details.each {|d| d[1] = d[1].gsub(r.param1, r.param2) if r.scope.match(d[0])}}
+
+    # remove any details with empty values
+    details.delete_if {|d| '' == d[1]}
+    return nil if details.empty?
+
+    # finish message
+    details << ['now', now.nil? ? (Time.now.to_f * 1000).floor.to_s : now]
+    details << ['agent', @agent]
+    details << ['version', @version]
+    JSON.generate details
+  end
+
 end

data/lib/resurfaceio/http_logger_for_rack.rb

@@ -21,8 +21,7 @@ class HttpLoggerForRack # http://rack.rubyforge.org/doc/SPEC.html
       response = Rack::Response.new(body, status, headers)
       if HttpLogger::string_content_type?(response.content_type)
         request = Rack::Request.new(env)
-        message = @logger.format(request, response)
-        @logger.submit(message)
+        @logger.submit(@logger.format(request, response))
       end
     end
     [status, headers, body]

data/lib/resurfaceio/http_logger_for_rails.rb

@@ -20,8 +20,7 @@ class HttpLoggerForRails
       response = controller.response
       status = response.status
       if (status < 300 || status == 302) && HttpLogger::string_content_type?(response.content_type)
-        message = @logger.format(request, response)
-        @logger.submit(message)
+        @logger.submit(@logger.format(request, response))
       end
     end
   end

data/lib/resurfaceio/{http_message_impl.rb → http_message.rb}

@@ -3,7 +3,7 @@
 
 require 'json'
 
-class HttpMessageImpl
+class HttpMessage
 
   def self.build(request, response, response_body = nil, request_body = nil)
     message = []

data/lib/resurfaceio/http_request_impl.rb

@@ -7,6 +7,7 @@ class HttpRequestImpl
     @form_hash = Hash.new
     @headers = Hash.new
     @query_hash = Hash.new
+    @session = Hash.new
   end
 
   def add_header(key, value)
@@ -40,6 +41,10 @@ class HttpRequestImpl
     @query_hash
   end
 
+  def session
+    @session
+  end
+
   attr_accessor :request_method
   attr_accessor :url
 

data/lib/resurfaceio/http_rule.rb

@@ -0,0 +1,15 @@
+# coding: utf-8
+# © 2016-2018 Resurface Labs LLC
+
+class HttpRule
+
+  def initialize(verb, scope = nil, param1 = nil, param2 = nil)
+    @verb = verb
+    @scope = scope
+    @param1 = param1
+    @param2 = param2
+  end
+
+  attr_reader :verb, :scope, :param1, :param2
+
+end

data/lib/resurfaceio/http_rules.rb

@@ -0,0 +1,130 @@
+# coding: utf-8
+# © 2016-2018 Resurface Labs LLC
+
+class HttpRules
+
+  def self.debug_rules
+    "allow_http_url\ncopy_session_field /.*/\n"
+  end
+
+  def self.standard_rules
+    %q(/request_header:cookie|response_header:set-cookie/ remove
+/(request|response)_body|request_param/ replace /[a-zA-Z0-9.!#$%&’*+\/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)/, /x@y.com/
+/request_body|request_param|response_body/ replace /[0-9\.\-\/]{9,}/, /xyxy/
+)
+  end
+
+  def self.strict_rules
+    %q(/request_url/ replace /([^\?;]+).*/, !\\\\1!
+/request_body|response_body|request_param:.*|request_header:(?!user-agent).*|response_header:(?!(content-length)|(content-type)).*/ remove
+)
+  end
+
+  def self.parse(rules)
+    result = []
+    unless rules.nil?
+      rules = rules.gsub(/^\s*include debug\s*$/, debug_rules)
+      rules = rules.gsub(/^\s*include standard\s*$/, standard_rules)
+      rules = rules.gsub(/^\s*include strict\s*$/, strict_rules)
+      rules.each_line do |rule|
+        parsed = parse_rule(rule)
+        result << parsed unless parsed.nil?
+      end
+    end
+    result
+  end
+
+  def self.parse_rule(r)
+    if r.nil? || r.match(REGEX_BLANK_OR_COMMENT)
+      nil
+    elsif r.match(REGEX_ALLOW_HTTP_URL)
+      HttpRule.new('allow_http_url')
+    elsif (m = r.match(REGEX_COPY_SESSION_FIELD))
+      HttpRule.new('copy_session_field', nil, parse_regex(r, m[1]))
+    elsif (m = r.match(REGEX_REMOVE))
+      HttpRule.new('remove', parse_regex(r, m[1]))
+    elsif (m = r.match(REGEX_REMOVE_IF))
+      HttpRule.new('remove_if', parse_regex(r, m[1]), parse_regex(r, m[2]))
+    elsif (m = r.match(REGEX_REMOVE_IF_FOUND))
+      HttpRule.new('remove_if_found', parse_regex(r, m[1]), parse_regex_find(r, m[2]))
+    elsif (m = r.match(REGEX_REMOVE_UNLESS))
+      HttpRule.new('remove_unless', parse_regex(r, m[1]), parse_regex(r, m[2]))
+    elsif (m = r.match(REGEX_REMOVE_UNLESS_FOUND))
+      HttpRule.new('remove_unless_found', parse_regex(r, m[1]), parse_regex_find(r, m[2]))
+    elsif (m = r.match(REGEX_REPLACE))
+      HttpRule.new('replace', parse_regex(r, m[1]), parse_regex_find(r, m[2]), parse_string(r, m[3]))
+    elsif (m = r.match(REGEX_SAMPLE))
+      m1 = m[1].to_i
+      raise RuntimeError.new("Invalid sample percent: #{m1}") if m1 < 1 || m1 > 99
+      HttpRule.new('sample', nil, m1)
+    elsif r.match(REGEX_SKIP_COMPRESSION)
+      HttpRule.new('skip_compression')
+    elsif r.match(REGEX_SKIP_SUBMISSION)
+      HttpRule.new('skip_submission')
+    elsif (m = r.match(REGEX_STOP))
+      HttpRule.new('stop', parse_regex(r, m[1]))
+    elsif (m = r.match(REGEX_STOP_IF))
+      HttpRule.new('stop_if', parse_regex(r, m[1]), parse_regex(r, m[2]))
+    elsif (m = r.match(REGEX_STOP_IF_FOUND))
+      HttpRule.new('stop_if_found', parse_regex(r, m[1]), parse_regex_find(r, m[2]))
+    elsif (m = r.match(REGEX_STOP_UNLESS))
+      HttpRule.new('stop_unless', parse_regex(r, m[1]), parse_regex(r, m[2]))
+    elsif (m = r.match(REGEX_STOP_UNLESS_FOUND))
+      HttpRule.new('stop_unless_found', parse_regex(r, m[1]), parse_regex_find(r, m[2]))
+    else
+      raise RuntimeError.new("Invalid rule: #{r}")
+    end
+  end
+
+  protected
+
+  def self.parse_regex(r, regex)
+    str = parse_string(r, regex)
+    raise RuntimeError.new("Invalid regex (#{regex}) in rule: #{r}") if '*' == str || '+' == str || '?' == str
+    str = "^#{str}" unless str.start_with?('^')
+    str = "#{str}$" unless str.end_with?('$')
+    begin
+      return Regexp.compile(str)
+    rescue RegexpError
+      raise RuntimeError.new("Invalid regex (#{regex}) in rule: #{r}")
+    end
+  end
+
+  def self.parse_regex_find(r, regex)
+    begin
+      return Regexp.compile(parse_string(r, regex))
+    rescue RegexpError
+      raise RuntimeError.new("Invalid regex (#{regex}) in rule: #{r}")
+    end
+  end
+
+  def self.parse_string(r, str)
+    %w(~ ! % | /).each do |sep|
+      if (m = str.match(/^[#{sep}](.*)[#{sep}]$/))
+        m1 = m[1]
+        raise RuntimeError.new("Unescaped separator (#{sep}) in rule: #{r}") if m1.match(/^[#{sep}].*|.*[^\\][#{sep}].*/)
+        return m1.gsub("\\#{sep}", sep)
+      end
+    end
+    raise RuntimeError.new("Invalid expression (#{str}) in rule: #{r}")
+  end
+
+  REGEX_ALLOW_HTTP_URL = /^\s*allow_http_url\s*(#.*)?$/.freeze
+  REGEX_BLANK_OR_COMMENT = /^\s*([#].*)*$/.freeze
+  REGEX_COPY_SESSION_FIELD = /^\s*copy_session_field\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_REMOVE = /^\s*([~!%|\/].+[~!%|\/])\s*remove\s*(#.*)?$/.freeze
+  REGEX_REMOVE_IF = /^\s*([~!%|\/].+[~!%|\/])\s*remove_if\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_REMOVE_IF_FOUND = /^\s*([~!%|\/].+[~!%|\/])\s*remove_if_found\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_REMOVE_UNLESS = /^\s*([~!%|\/].+[~!%|\/])\s*remove_unless\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_REMOVE_UNLESS_FOUND = /^\s*([~!%|\/].+[~!%|\/])\s*remove_unless_found\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_REPLACE = /^\s*([~!%|\/].+[~!%|\/])\s*replace[\s]+([~!%|\/].+[~!%|\/]),[\s]+([~!%|\/].*[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_SAMPLE = /^\s*sample\s+(\d+)\s*(#.*)?$/.freeze
+  REGEX_SKIP_COMPRESSION = /^\s*skip_compression\s*(#.*)?$/.freeze
+  REGEX_SKIP_SUBMISSION = /^\s*skip_submission\s*(#.*)?$/.freeze
+  REGEX_STOP = /^\s*([~!%|\/].+[~!%|\/])\s*stop\s*(#.*)?$/.freeze
+  REGEX_STOP_IF = /^\s*([~!%|\/].+[~!%|\/])\s*stop_if\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_STOP_IF_FOUND = /^\s*([~!%|\/].+[~!%|\/])\s*stop_if_found\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_STOP_UNLESS = /^\s*([~!%|\/].+[~!%|\/])\s*stop_unless\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+  REGEX_STOP_UNLESS_FOUND = /^\s*([~!%|\/].+[~!%|\/])\s*stop_unless_found\s+([~!%|\/].+[~!%|\/])\s*(#.*)?$/.freeze
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: resurfaceio-logger
 version: !ruby/object:Gem::Version
-  version: 1.8.4
+  version: 1.9.0
 platform: ruby
 authors:
 - RobDickinson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-30 00:00:00.000000000 Z
+date: 2018-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -77,9 +77,11 @@ files:
 - lib/resurfaceio/http_logger.rb
 - lib/resurfaceio/http_logger_for_rack.rb
 - lib/resurfaceio/http_logger_for_rails.rb
-- lib/resurfaceio/http_message_impl.rb
+- lib/resurfaceio/http_message.rb
 - lib/resurfaceio/http_request_impl.rb
 - lib/resurfaceio/http_response_impl.rb
+- lib/resurfaceio/http_rule.rb
+- lib/resurfaceio/http_rules.rb
 - lib/resurfaceio/usage_loggers.rb
 homepage: https://github.com/resurfaceio/logger-ruby
 licenses: