logasm 0.2.6 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c37a9355e4d9426215d500f6fdb50a22954c2e4
-  data.tar.gz: 58289a6f19d649b8c405638428a43938ca586132
+  metadata.gz: 440665fe491c115f43c411822de433ead259e193
+  data.tar.gz: 06568509dacd084a9607ac545b3cfd03447947db
 SHA512:
-  metadata.gz: 6712eee9ed98800f0be30370509848b0c8a70b587e278fd63c7e742d5949aa0c3703b70cb17bc2c8a5d9d3cb02ab7e548c1d9a06c65a5f31711d3628edb55f7e
-  data.tar.gz: e60591d1432b92075757b0fb76acbd7613f3f46fb6ec60af58bb2bd0e54331a90f73e301b3338f8f3667f4feabc547821a8bbf99c613aac288f6193e4309cbff
+  metadata.gz: 380b1118c988bbf141b9c03a63ca3c68d33f1f8873709bd92d5e52188dfb3adf980edf7d040eca62c2bdbc66fb4a142e0906e2272aa278cceb4109d6c88e52a2
+  data.tar.gz: b93a280171c1123d5559ac630a5db52710cd429a7f5bbaddbb73885f7aa75d287d21b117ca3a8fe75b6a2399667c05bcb1631ce889795f2b493036f664742992

data/README.md

@@ -61,3 +61,74 @@ logasm = Logasm.build('myApp', { stdout: nil, logstash: { host: "localhost", por
 ```
 
 When no loggers are specified, it creates a stdout logger by default.
+
+## Preprocessors
+
+Preprocessors allow modification of log messages, prior to sending of the message to the configured logger(s).
+
+### Blacklist
+
+Excludes or masks defined fields of the passed hash object.
+You can specify the name of the field and which action to take on it.
+Nested hashes of any level are preprocessed as well.
+
+Available actions:
+    
+* exclude(`default`) - fully excludes the field and its value from the hash.
+* mask - replaces every character from the original value with `*`. In case of `array`, `hash` or `boolean` value is replaced with one `*`.
+
+#### Configuration
+
+```yaml
+preprocessors:
+  blacklist:
+    fields:
+      - key: password
+      - key: phone
+        action: mask
+```
+
+#### Usage
+
+```ruby
+logger = Logasm.build(application_name, logger_config, preprocessors)
+
+input = {password: 'password', info: {phone: '+12055555555'}}
+
+logger.debug("Received request", input)
+```
+
+Logger output:
+
+```
+Received request {"info":{"phone":"************"}}
+```
+
+### Whitelist
+
+Masks all the fields except those whitelisted in the configuration using [JSON Pointer](https://tools.ietf.org/html/rfc6901).
+Only simple values(`string`, `number`, `boolean`) can be whitelisted.
+
+#### Configuration
+
+```yaml
+preprocessors:
+  whitelist:
+    pointers: ['/info/phone']
+```
+
+#### Usage
+
+```ruby
+logger = Logasm.build(application_name, logger_config, preprocessors)
+
+input = {password: 'password', info: {phone: '+12055555555'}}
+
+logger.debug("Received request", input)
+```
+
+Logger output:
+
+```
+Received request {password: "********", "info":{"phone":"+12055555555"}}
+```

data/lib/logasm/preprocessors/blacklist.rb

@@ -0,0 +1,67 @@
+class Logasm
+  module Preprocessors
+    class Blacklist
+
+      DEFAULT_ACTION = 'exclude'
+      MASK_SYMBOL = '*'
+
+      class UnsupportedActionException < Exception
+      end
+
+      def initialize(config = {})
+        @fields_to_process = config[:fields].inject({}) do |mem, field|
+          key = field.delete(:key)
+          options = {action: DEFAULT_ACTION}.merge(field)
+          validate_action_supported(options[:action])
+          mem.merge(key => options)
+        end
+      end
+
+      def process(data)
+        if data.is_a? Hash
+          data.inject({}) do |mem, (key, val)|
+            if (field = @fields_to_process[key.to_s])
+              self.send(action_method(field[:action]), mem, key, val)
+            else
+              mem.merge(key => process(val))
+            end
+          end
+        elsif data.is_a? Array
+          data.inject([]) do |mem, val|
+            mem + [process(val)]
+          end
+        else
+          data
+        end
+      end
+
+      private
+
+      def action_method(action)
+        "#{action}_field"
+      end
+
+      def validate_action_supported(action)
+        unless self.respond_to?(action_method(action).to_sym, true)
+          raise UnsupportedActionException.new("Action: #{action} is not supported")
+        end
+      end
+
+      def mask_field(data, key, val)
+        if val.is_a?(Hash) || val.is_a?(Array) || is_boolean?(val)
+          data.merge(key => MASK_SYMBOL)
+        else
+          data.merge(key => MASK_SYMBOL * val.to_s.length)
+        end
+      end
+
+      def exclude_field(data, *)
+        data
+      end
+
+      def is_boolean?(val)
+        val.is_a?(TrueClass) || val.is_a?(FalseClass)
+      end
+    end
+  end
+end

data/lib/logasm/preprocessors/whitelist.rb

@@ -0,0 +1,88 @@
+class Logasm
+  module Preprocessors
+    class Whitelist
+
+      DEFAULT_WHITELIST = %w(/id /message /correlation_id /queue)
+      MASK_SYMBOL = '*'
+
+      class InvalidPointerFormatException < Exception
+      end
+
+      def initialize(config = {})
+        pointers = (config[:pointers] || []) + DEFAULT_WHITELIST
+        @fields_to_include = pointers.inject({}) do |mem, pointer|
+          validate_pointer(pointer)
+          mem.merge(decode(pointer) => true)
+        end
+      end
+
+      def process(data)
+        process_data('', data)
+      end
+
+      private
+
+      def validate_pointer(pointer)
+        if pointer.slice(-1) == '/'
+          raise InvalidPointerFormatException.new('Pointer should not contain trailing slash')
+        end
+      end
+
+      def decode(pointer)
+        pointer
+          .gsub('~0', '~')
+          .gsub('~1', '/')
+      end
+
+      def process_data(parent_pointer, data)
+        self.send("process_#{get_type(data)}", parent_pointer, data)
+      end
+
+      def get_type(data)
+        if data.is_a? Hash
+          'hash'
+        elsif data.is_a? Array
+          'array'
+        else
+          'value'
+        end
+      end
+
+      def process_hash(parent_pointer, hash)
+        hash.inject({}) do |mem, (key, value)|
+          pointer = "#{parent_pointer}/#{key}"
+          processed_value = process_data(pointer, value)
+          mem.merge(key => processed_value)
+        end
+      end
+
+      def process_array(parent_pointer, array)
+        array.each_with_index.inject([]) do |mem, (value, index)|
+          pointer = "#{parent_pointer}/#{index}"
+          processed_value = process_data(pointer, value)
+          mem + [processed_value]
+        end
+      end
+
+      def process_value(parent_pointer, value)
+        if @fields_to_include[parent_pointer]
+          value
+        else
+          mask value
+        end
+      end
+
+      def mask(value)
+        if value && value.respond_to?(:to_s) && !is_boolean?(value)
+          MASK_SYMBOL * value.to_s.length
+        else
+          MASK_SYMBOL
+        end
+      end
+
+      def is_boolean?(value)
+        value.is_a?(TrueClass) || value.is_a?(FalseClass)
+      end
+    end
+  end
+end

data/lib/logasm/preprocessors.rb

@@ -0,0 +1,9 @@
+class Logasm
+  module Preprocessors
+    def self.get(type, arguments)
+      require_relative "preprocessors/#{type.to_s}"
+      preprocessor = const_get(Inflecto.camelize(type.to_s))
+      preprocessor.new(arguments)
+    end
+  end
+end

data/lib/logasm.rb

@@ -4,20 +4,25 @@ require 'json'
 require_relative 'logasm/adapters'
 require_relative 'logasm/utils'
 require_relative 'logasm/null_logger'
+require_relative 'logasm/preprocessors'
 
 LOG_LEVEL_QUERY_METHODS = [:debug?, :info?, :warn?, :error?, :fatal?]
 
 class Logasm
-  def self.build(service_name, loggers_config)
+  def self.build(service_name, loggers_config, preprocessors_config = {})
     loggers_config ||= {stdout: nil}
+    preprocessors = preprocessors_config.map do |type, arguments|
+      Preprocessors.get(type.to_s, arguments || {})
+    end
     adapters = loggers_config.map do |type, arguments|
       Adapters.get(type.to_s, service_name, arguments || {})
     end
-    new(adapters)
+    new(adapters, preprocessors)
   end
 
-  def initialize(adapters)
+  def initialize(adapters, preprocessors)
     @adapters = adapters
+    @preprocessors = preprocessors
   end
 
   def debug(*args)
@@ -50,9 +55,16 @@ class Logasm
 
   def log(level, *args)
     data = parse_log_data(*args)
+    processed_data = preprocess(data)
 
     @adapters.each do |adapter|
-      adapter.log(level, data)
+      adapter.log(level, processed_data)
+    end
+  end
+
+  def preprocess(data)
+    @preprocessors.inject(data) do |data_to_process, preprocessor|
+      preprocessor.process(data_to_process)
     end
   end
 

data/logasm.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "logasm"
-  gem.version       = '0.2.6'
+  gem.version       = '0.3.0'
   gem.authors       = ["Salemove"]
   gem.email         = ["support@salemove.com"]
   gem.description   = %q{It's logasmic}

data/spec/logasm_spec.rb

@@ -38,11 +38,36 @@ describe Logasm do
 
       described_class.build('test_service', nil)
     end
+
+    it 'creates preprocessor when preprocessor defined' do
+      expect(described_class).to receive(:new) do |adapters, preprocessors|
+        expect(preprocessors.count).to be(1)
+        expect(preprocessors.first).to be_a(described_class::Preprocessors::Blacklist)
+      end
+
+      preprocessors = {blacklist: {fields: []}}
+      described_class.build('test_service', nil, preprocessors)
+    end
+  end
+
+  context 'when preprocessor defined' do
+    let(:logasm) { described_class.new([adapter], [preprocessor]) }
+    let(:adapter) { double }
+    let(:preprocessor) { double }
+    let(:data) { {data: 'data'} }
+
+    it 'preprocesses data before logging' do
+      expect(preprocessor).to receive(:process).with(data).and_return(data.merge(processed: true)).ordered
+      expect(adapter).to receive(:log).with(:info, data.merge(processed: true)).ordered
+
+      logasm.info(data)
+    end
   end
 
   context 'when parsing log data' do
-    let(:logasm) { described_class.new([adapter]) }
+    let(:logasm) { described_class.new([adapter], preprocessors) }
     let(:adapter) { double }
+    let(:preprocessors) { [] }
 
     it 'parses empty string with nil metadata' do
       expect(adapter).to receive(:log).with(:info, message: '')
@@ -114,4 +139,10 @@ describe Logasm do
       end
     end
   end
+
+  it 'has the same interface as Ruby logger' do
+    skip "https://salemove.atlassian.net/browse/INF-464"
+    logger = described_class.build('test_service', stdout: {level: 'debug'})
+    expect(logger).to implement_interface(Logger)
+  end
 end

data/spec/preprocessors/blacklist_spec.rb

@@ -0,0 +1,131 @@
+require 'spec_helper'
+require_relative '../../lib/logasm/preprocessors/blacklist'
+
+describe Logasm::Preprocessors::Blacklist do
+  subject(:processed_data) { described_class.new(config).process(data) }
+
+  let(:config) {{
+    fields: [{ key: 'field', action: action }]
+  }}
+
+  let(:action) {}
+  let(:data) {{
+    field: value,
+    data: {
+      field: 'secret'
+    },
+    array: [{field: 'secret'}]
+  }}
+
+  let(:value) { 'secret' }
+
+  context 'when action is unsupported' do
+    let(:action) { 'reverse' }
+
+    it 'throws exception' do
+      expect { processed_data }.to raise_exception(Logasm::Preprocessors::Blacklist::UnsupportedActionException)
+    end
+  end
+
+  context 'when action is "exclude"' do
+    let(:action) { 'exclude' }
+
+    it 'removes the field' do
+      expect(processed_data).not_to include(:field)
+    end
+
+    it 'removes nested field' do
+      expect(processed_data).not_to include_at_depth({field: 'secret'}, 1)
+    end
+
+    it 'removes nested in array field' do
+      expect(processed_data[:array]).not_to include({field: 'secret'})
+    end
+
+    context 'when field is deeply nested' do
+      let(:depth) { 10 }
+      let(:data) { data_with_nested_field({field: 'secret'}, depth) }
+
+      it 'removes deeply nested field' do
+        expect(processed_data).not_to include_at_depth({field: 'secret'}, depth)
+      end
+    end
+  end
+
+  context 'when action is "mask"' do
+    let(:action) { 'mask' }
+
+    it 'masks nested field' do
+      expect(processed_data).to include_at_depth({field: '******'}, 1)
+    end
+
+    it 'masks nested in array field' do
+      expect(processed_data[:array]).to include({field: '******'})
+    end
+
+    context 'when field is string' do
+      let(:value) { 'secret' }
+
+      it 'masks value with asterisks' do
+        expect(processed_data).to include(field: '******')
+      end
+    end
+
+    context 'when field is number' do
+      let(:value) { 42 }
+
+      it 'masks number value' do
+        expect(processed_data).to include(field: '**')
+      end
+    end
+
+    context 'when field is boolean' do
+      let(:value) { true }
+
+      it 'masks value with one asterisk' do
+        expect(processed_data).to include(field: '*')
+      end
+    end
+
+    context 'when field is array' do
+      let(:value) { [1,2,3,4] }
+
+      it 'masks value with one asterisk' do
+        expect(processed_data).to include(field: '*')
+      end
+    end
+
+    context 'when field is hash' do
+      let(:value) { {data: {}} }
+
+      it 'masks value with one asterisk' do
+        expect(processed_data).to include(field: '*')
+      end
+    end
+
+    context 'when field is deeply nested' do
+      let(:depth) { 10 }
+      let(:data) { data_with_nested_field({field: 'secret'}, depth) }
+
+      it 'masks deeply nested field' do
+        expect(processed_data).to include_at_depth({field: '******'}, depth)
+      end
+    end
+  end
+
+  def data_with_nested_field(field, depth)
+    depth.times.inject(field) do |mem|
+      {}.merge(data: mem)
+    end
+  end
+
+  RSpec::Matchers.define :include_at_depth do |expected_hash, depth|
+    match do |actual|
+      nested_data = depth.times.inject(actual) do |mem|
+        mem[:data]
+      end
+
+      expect(nested_data).to include(expected_hash)
+    end
+  end
+end

data/spec/preprocessors/whitelist_spec.rb

@@ -0,0 +1,152 @@
+require 'spec_helper'
+require_relative '../../lib/logasm/preprocessors/whitelist'
+
+describe Logasm::Preprocessors::Whitelist do
+  subject(:processed_data) { described_class.new(config).process(data) }
+
+  let(:config) { {pointers: pointers} }
+  let(:pointers) { [] }
+  let(:data) {{
+    field: 'secret',
+    data: {
+      field: 'secret'
+    },
+    array: [{field: 'secret'}]
+  }}
+
+  it 'masks all non-whitelisted fields' do
+    expect(processed_data).to eq({
+      field: '******',
+      data: {
+        field: '******'
+      },
+      array: [{field: '******'}]
+    })
+  end
+
+  context 'when pointer has trailing slash' do
+    let(:pointers) { ['/field/'] }
+
+    it 'throws exception' do
+      expect { processed_data }.to raise_exception(Logasm::Preprocessors::Whitelist::InvalidPointerFormatException)
+    end
+  end
+
+  context 'with whitelisted field' do
+    let(:pointers) { ['/field'] }
+
+    it 'includes the field' do
+      expect(processed_data).to eq({
+        field: 'secret',
+        data: {
+          field: '******'
+        },
+        array: [{field: '******'}]
+      })
+    end
+  end
+
+  context 'with whitelisted nested field' do
+    let(:pointers) { ['/data/field'] }
+
+    it 'includes nested field' do
+      expect(processed_data).to eq({
+        field: '******',
+        data: {
+          field: 'secret'
+        },
+        array: [{field: '******'}]
+      })
+    end
+  end
+
+  context 'with whitelisted array element field' do
+    let(:pointers) { ['/array/0/field'] }
+
+    it 'includes array element' do
+      expect(processed_data).to eq({
+        field: '******',
+        data: {
+          field: '******'
+        },
+        array: [{field: 'secret'}]
+      })
+    end
+  end
+
+  context 'with whitelisted array element' do
+    let(:pointers) { ['/array/0'] }
+
+    it 'masks array element' do
+      expect(processed_data).to include(array: [{field: '******'}])
+    end
+  end
+
+  context 'with whitelisted array' do
+    let(:pointers) { ['/array'] }
+
+    it 'masks array' do
+      expect(processed_data).to include(array: [{field: '******'}])
+    end
+  end
+
+  context 'with whitelisted hash' do
+    let(:pointers) { ['/data'] }
+
+    it 'masks hash' do
+      expect(processed_data).to include(data: {field: '******'})
+    end
+  end
+
+  context 'when boolean present' do
+    let(:data) { {bool: true} }
+
+    it 'masks it with single asteriks' do
+      expect(processed_data).to eq(bool: '*')
+    end
+  end
+
+  context 'when field has slash in the name' do
+    let(:data) {{
+      'field_with_/' => 'secret'
+    }}
+    let(:pointers) { ['/field_with_~1'] }
+
+    it 'includes field' do
+      expect(processed_data).to include('field_with_/'=> 'secret')
+    end
+  end
+
+  context 'when field has tilde in the name' do
+    let(:data) {{
+      'field_with_~' => 'secret'
+    }}
+    let(:pointers) { ['/field_with_~0'] }
+
+    it 'includes field' do
+      expect(processed_data).to include('field_with_~'=> 'secret')
+    end
+  end
+
+  context 'when field has slash in the name' do
+    let(:data) {{
+      'field_with_/' => 'secret'
+    }}
+    let(:pointers) { ['/field_with_~1'] }
+
+    it 'includes field' do
+      expect(processed_data).to include('field_with_/'=> 'secret')
+    end
+  end
+
+  context 'when field has tilde in the name' do
+    let(:data) {{
+      'field_with_~' => 'secret'
+    }}
+    let(:pointers) { ['/field_with_~0'] }
+
+    it 'includes field' do
+      expect(processed_data).to include('field_with_~'=> 'secret')
+    end
+  end
+end

data/spec/support/implement_interface.rb

@@ -0,0 +1,11 @@
+RSpec::Matchers.define :implement_interface do |expected|
+  required_methods = expected.instance_methods(false)
+  match do |actual|
+    required_methods - actual.methods == []
+  end
+
+  failure_message do |actual|
+    missing_methods = required_methods - actual.methods
+    "Expected instance methods #{missing_methods.inspect} to be implemented"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logasm
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors:
 - Salemove
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-30 00:00:00.000000000 Z
+date: 2016-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inflecto
@@ -88,6 +88,9 @@ files:
 - lib/logasm/adapters/rabbitmq_adapter.rb
 - lib/logasm/adapters/stdout_adapter.rb
 - lib/logasm/null_logger.rb
+- lib/logasm/preprocessors.rb
+- lib/logasm/preprocessors/blacklist.rb
+- lib/logasm/preprocessors/whitelist.rb
 - lib/logasm/utils.rb
 - logasm.gemspec
 - spec/adapters/logstash_adapter/formatter_spec.rb
@@ -95,8 +98,11 @@ files:
 - spec/adapters/rabbitmq_adapter_spec.rb
 - spec/adapters/stdout_adapter_spec.rb
 - spec/logasm_spec.rb
+- spec/preprocessors/blacklist_spec.rb
+- spec/preprocessors/whitelist_spec.rb
 - spec/spec_helper.rb
 - spec/support/freddy_mock.rb
+- spec/support/implement_interface.rb
 - spec/utils_spec.rb
 homepage: 
 licenses:
@@ -118,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: What description said
@@ -128,6 +134,9 @@ test_files:
 - spec/adapters/rabbitmq_adapter_spec.rb
 - spec/adapters/stdout_adapter_spec.rb
 - spec/logasm_spec.rb
+- spec/preprocessors/blacklist_spec.rb
+- spec/preprocessors/whitelist_spec.rb
 - spec/spec_helper.rb
 - spec/support/freddy_mock.rb
+- spec/support/implement_interface.rb
 - spec/utils_spec.rb