logasm 0.8.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f563cce04590bb7f18f91f0546b67c9cfb6317b
-  data.tar.gz: d0351070693d8d0e2ea59d1e05de36d9db34e016
+  metadata.gz: 3c09d965be2c9ada94a7ee35d4b7518f411b9a81
+  data.tar.gz: d3442723ac39e8661ac4d789b159aca7f2152895
 SHA512:
-  metadata.gz: b2904cdfca1b3020475ef4695910c646d768ba548a24333046ade905d8ddf9a2e4e867189509ca47ede5a4f0bcf99180cf4182b74509855db5c57f325ae5b674
-  data.tar.gz: 800f52b72bf15df87d88537589a8e25a4affb05c69a45933e9978b5d1c14ae4ebf67f87b71b5a92d5fefcf0629e011298c9e0440aa62f5ebfc4c5e1014ec2363
+  metadata.gz: 5a896dc77965ca1b6022ec65918327b8a94435d85ae592e6a516bed903f2588f4b1257e692c7e5915f5431af2564ec14fe9e73b6627d689b0f8f2b4a035df48d
+  data.tar.gz: 32deab6ec961e0ffdc123ec6065fa09eabf4a604740c98a9782aeaaf7927f6c5e0160bc99574a94018dcff667aec84eeab62f2de8349af46d84f7a6add776354

data/.ruby-version

@@ -1 +1 @@
-ruby-2.1.2
+ruby-2.4.1

data/Gemfile

@@ -1,8 +1,8 @@
 source "https://rubygems.org"
 
 group :test, :development do
-  gem 'rspec'
   gem 'pry'
+  gem 'rspec'
 end
 
 gemspec

data/benchmark/whitelisting.rb

@@ -0,0 +1,52 @@
+require 'bundler/setup'
+require 'logasm/preprocessors/whitelist'
+require 'benchmark/ips'
+
+pointers = %w[
+  /scalar
+  /flat_hash/~
+  /nested_hash/~/deep_hash/~
+  /flat_array/~
+  /nested_array/~/deep_array/~
+]
+
+preprocessor = Logasm::Preprocessors::Whitelist.new(pointers: pointers)
+
+
+Benchmark.ips do |x|
+  x.config(time: 5, warmup: 2)
+
+  x.report('Scalar value whitelisting') do
+    preprocessor.process(scalar: 'value', bad_scalar: 'value', hash: {})
+  end
+
+  x.report('Flat hash whitelisting') do
+    preprocessor.process(flat_hash: { scalar: 'value', array: [1, 2], hash: {} })
+  end
+
+  x.report('Nested hash whitelisting') do
+    preprocessor.process(
+      nested_hash: {
+        next_level_hash: {
+          deep_hash: { scalar: 'value', array: [1, 2] }
+        },
+        next_level_hash2: {
+          deep_hash: { scalar: 'value', array: [1, 2] }
+        },
+        next_level_hash3: {
+          deep_hash: { scalar: 'value', array: [1, 2] }
+        }
+      }
+    )
+  end
+
+  x.report('Flat array whitelisting') do
+    preprocessor.process(
+      nested_array: [
+        { deep_array: [1, 2, 3] },
+        { deep_array: [1, 2, 3] },
+        { deep_array: [1, 2, 3] }
+      ]
+    )
+  end
+end

data/lib/logasm/preprocessors/json_pointer_trie.rb

@@ -0,0 +1,40 @@
+require 'lru_redux'
+
+class Logasm
+  module Preprocessors
+    class JSONPointerTrie
+      SEPARATOR = '/'.freeze
+      WILDCARD = '~'.freeze
+      DEFAULT_CACHE_SIZE = 100
+
+      def initialize(cache_size: DEFAULT_CACHE_SIZE, **)
+        @root_node = {}
+        @cache = LruRedux::Cache.new(cache_size)
+      end
+
+      def insert(pointer)
+        split_path(pointer).reduce(@root_node) do |tree, key|
+          tree[key] ||= {}
+        end
+
+        self
+      end
+
+      def include?(path)
+        @cache.getset(path) { traverse_path(path) }
+      end
+
+      private
+
+      def traverse_path(path)
+        split_path(path).reduce(@root_node) do |node, key|
+          node[key] || node[WILDCARD] || (break false)
+        end
+      end
+
+      def split_path(path)
+        path.split(SEPARATOR).reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/logasm/preprocessors/whitelist.rb

@@ -1,28 +1,23 @@
+require 'logasm/preprocessors/json_pointer_trie'
+
 class Logasm
   module Preprocessors
     class Whitelist
-
-      DEFAULT_WHITELIST = %w(/id /message /correlation_id /queue)
-      MASK_SYMBOL = '*'
+      DEFAULT_WHITELIST = %w[/id /message /correlation_id /queue].freeze
+      MASK_SYMBOL = '*'.freeze
       MASKED_VALUE = MASK_SYMBOL * 5
-      WILDCARD = '~'
 
       class InvalidPointerFormatException < Exception
       end
 
       def initialize(config = {})
         pointers = (config[:pointers] || []) + DEFAULT_WHITELIST
-        decoded_pointers = pointers
-          .each(&method(:validate_pointer))
-          .map(&method(:decode))
-        @fields_to_include = decoded_pointers.inject({}) do |mem, pointer|
-          mem.merge(pointer => true)
+
+        @trie = pointers.reduce(JSONPointerTrie.new(config)) do |trie, pointer|
+          validate_pointer(pointer)
+
+          trie.insert(decode(pointer))
         end
-        @wildcards = decoded_pointers
-          .select(&method(:has_wildcard?))
-          .inject({}) do |mem, pointer|
-            mem.merge(get_wildcard_roots_of(pointer))
-          end
       end
 
       def process(data)
@@ -31,38 +26,9 @@ class Logasm
 
       private
 
-
-      def has_wildcard?(pointer)
-        pointer.include?("/#{WILDCARD}/") || pointer.end_with?("/#{WILDCARD}")
-      end
-
-      # From a pointer with wildcards builds a hash with roots that contains a wildcard. Hash is used to easily match
-      # find if hash element matches the pointer while processing the log.
-      #
-      # Example:
-      #
-      # Input:
-      #   "/array/~/nested_array/~/fields"
-      #
-      # Output:
-      # {
-      #   "/array/~/nested_array/~" => true,
-      #   "/array/~" => true
-      # }
-      #
-      def get_wildcard_roots_of(pointer)
-        if (index = pointer.rindex("/#{WILDCARD}/"))
-          wildcard_root = pointer.slice(0, index + 2)
-          wildcard_path = pointer.end_with?(WILDCARD) ? pointer : wildcard_root
-          get_wildcard_roots_of(wildcard_root).merge(wildcard_path => true)
-        else
-          {pointer => true}
-        end
-      end
-
       def validate_pointer(pointer)
         if pointer.slice(-1) == '/'
-          raise InvalidPointerFormatException.new('Pointer should not contain trailing slash')
+          raise InvalidPointerFormatException, 'Pointer should not contain trailing slash'
         end
       end
 
@@ -73,52 +39,30 @@ class Logasm
       end
 
       def process_data(parent_pointer, data)
-        self.send("process_#{get_type(data)}", parent_pointer, data)
-      end
+        return MASKED_VALUE unless @trie.include?(parent_pointer)
+
+        case data
+        when Hash
+          process_hash(parent_pointer, data)
+
+        when Array
+          process_array(parent_pointer, data)
 
-      def get_type(data)
-        if data.is_a? Hash
-          'hash'
-        elsif data.is_a? Array
-          'array'
         else
-          'value'
+          data
         end
       end
 
       def process_hash(parent_pointer, hash)
-        create_child_pointer =
-          if @wildcards["#{parent_pointer}/~"]
-            lambda { |_| "#{parent_pointer}/~" }
-          else
-            lambda { |key| "#{parent_pointer}/#{key}" }
-          end
-        hash.inject({}) do |mem, (key, value)|
-          pointer = create_child_pointer.call(key)
-          processed_value = process_data(pointer, value)
-          mem.merge(key => processed_value)
+        hash.each_with_object({}) do |(key, value), result|
+          processed = process_data("#{parent_pointer}/#{key}", value)
+          result[key] = processed
         end
       end
 
       def process_array(parent_pointer, array)
-        create_child_pointer =
-          if @wildcards["#{parent_pointer}/~"]
-            lambda { |_| "#{parent_pointer}/~" }
-          else
-            lambda { |index| "#{parent_pointer}/#{index}" }
-          end
-        array.each_with_index.inject([]) do |mem, (value, index)|
-          pointer = create_child_pointer.call(index)
-          processed_value = process_data(pointer, value)
-          mem + [processed_value]
-        end
-      end
-
-      def process_value(parent_pointer, value)
-        if @fields_to_include[parent_pointer]
-          value
-        else
-          MASKED_VALUE
+        array.each_with_index.map do |value, index|
+          process_data("#{parent_pointer}/#{index}", value)
         end
       end
     end

data/logasm.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |gem|
   gem.name          = "logasm"
-  gem.version       = '0.8.1'
+  gem.version       = '0.9.0'
   gem.authors       = ["Salemove"]
   gem.email         = ["support@salemove.com"]
   gem.description   = %q{It's logasmic}
@@ -17,8 +17,10 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
 
   gem.add_dependency 'inflecto'
+  gem.add_dependency 'lru_redux'
 
   gem.add_development_dependency "bundler", "~> 1.3"
   gem.add_development_dependency "rake"
   gem.add_development_dependency "bunny"
+  gem.add_development_dependency "benchmark-ips"
 end

data/spec/preprocessors/json_pointer_trie.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+require 'logasm/preprocessors/json_pointer_trie'
+
+RSpec.describe Logasm::Preprocessors::JSONPointerTrie do
+  let(:trie) { described_class.new }
+
+  describe '#includes?' do
+    it 'returns true for empty prefix' do
+      expect(trie).to include('')
+    end
+
+    it 'returns true if trie contains requested prefix or value itself' do
+      trie.insert('/data/nested/key')
+
+      expect(trie).to include('/data')
+      expect(trie).to include('/data/nested')
+      expect(trie).to include('/data/nested/key')
+    end
+
+    it 'returns false if trie does not contain requested prefix or value' do
+      trie.insert('/data/nested/key')
+
+      expect(trie).to_not include('/bad_data')
+      expect(trie).to_not include('/data/bad_nested')
+      expect(trie).to_not include('/data/nested/bad_key')
+    end
+
+    it 'returns true if trie contains requested prefix under wildcard' do
+      trie.insert('/data/~/key')
+
+      expect(trie).to include('/data/arbitrary_key/key')
+      expect(trie).to include('/data/another_key/key')
+      expect(trie).to_not include('/data/arbitrary_key/bad_nested_key')
+    end
+  end
+end

data/spec/preprocessors/whitelist_spec.rb

@@ -4,24 +4,24 @@ require_relative '../../lib/logasm/preprocessors/whitelist'
 describe Logasm::Preprocessors::Whitelist do
   subject(:processed_data) { described_class.new(config).process(data) }
 
-  let(:config) { {pointers: pointers} }
+  let(:config) { { pointers: pointers } }
   let(:pointers) { [] }
-  let(:data) {{
-    field: 'secret',
-    data: {
-      field: 'secret'
-    },
-    array: [{field: 'secret'}]
-  }}
+  let(:data) do
+    {
+      field: 'secret',
+      data: {
+        field: 'secret'
+      },
+      array: [{ field: 'secret' }]
+    }
+  end
 
   it 'masks all non-whitelisted fields' do
-    expect(processed_data).to eq({
+    expect(processed_data).to eq(
       field: '*****',
-      data: {
-        field: '*****'
-      },
-      array: [{field: '*****'}]
-    })
+      data: '*****',
+      array: '*****'
+    )
   end
 
   context 'when pointer has trailing slash' do
@@ -36,13 +36,11 @@ describe Logasm::Preprocessors::Whitelist do
     let(:pointers) { ['/field'] }
 
     it 'includes the field' do
-      expect(processed_data).to eq({
+      expect(processed_data).to eq(
         field: 'secret',
-        data: {
-          field: '*****'
-        },
-        array: [{field: '*****'}]
-      })
+        data: '*****',
+        array: '*****'
+      )
     end
   end
 
@@ -50,13 +48,13 @@ describe Logasm::Preprocessors::Whitelist do
     let(:pointers) { ['/data/field'] }
 
     it 'includes nested field' do
-      expect(processed_data).to eq({
+      expect(processed_data).to eq(
         field: '*****',
         data: {
           field: 'secret'
         },
-        array: [{field: '*****'}]
-      })
+        array: '*****'
+      )
     end
   end
 
@@ -64,62 +62,70 @@ describe Logasm::Preprocessors::Whitelist do
     let(:pointers) { ['/array/0/field'] }
 
     it 'includes array element' do
-      expect(processed_data).to eq({
+      expect(processed_data).to eq(
         field: '*****',
-        data: {
-          field: '*****'
-        },
-        array: [{field: 'secret'}]
-      })
+        data: '*****',
+        array: [{ field: 'secret' }]
+      )
     end
   end
 
   context 'with whitelisted hash' do
     it 'includes all whitelisted hash elements' do
-      source = {foo: {bar: 'baz'}}
-      target = {foo: {bar: 'baz'}}
+      source = { foo: { bar: 'baz' } }
+      target = { foo: { bar: 'baz' } }
       expect(process(['/foo/~'], source)).to eq(target)
     end
 
     it 'does not include nested elements' do
-      source = {foo: {bar: {baz: 'asd'}}}
-      target = {foo: {bar: {baz: '*****'}}}
+      source = { foo: { bar: { baz: 'asd' } } }
+      target = { foo: { bar: { baz: '*****' } } }
       expect(process(['/foo/~'], source)).to eq(target)
     end
   end
 
   context 'with whitelisted array elements field with wildcard' do
-    let(:data) {{
-      array: [{field: 'data1', secret: 'secret1'}, {field: 'data2', secret: 'secret2'}]
-    }}
+    let(:data) do
+      {
+        array: [
+          { field: 'data1', secret: 'secret1' },
+          { field: 'data2', secret: 'secret2' }
+        ]
+      }
+    end
     let(:pointers) { ['/array/~/field'] }
 
     it 'includes array elements field' do
       expect(processed_data).to include(
-        array: [{field: 'data1', secret: '*****'}, {field: 'data2', secret: '*****'}]
+        array: [
+          { field: 'data1', secret: '*****' },
+          { field: 'data2', secret: '*****' }
+        ]
       )
     end
   end
 
   context 'with whitelisted string array elements with wildcard' do
-    let(:data) {{
-      array: ['secret', 'secret']
-    }}
+    let(:data) do
+      { array: %w[secret secret] }
+    end
     let(:pointers) { ['/array/~'] }
 
     it 'includes array elements' do
-      expect(processed_data).to include(array: ['secret', 'secret'])
+      expect(processed_data).to include(array: %w[secret secret])
     end
   end
 
   context 'with whitelisted string array elements in an array with wildcard' do
-    let(:data) {{
-      nested: [{array: ['secret', 'secret']}]
-    }}
+    let(:data) do
+      {
+        nested: [{ array: %w[secret secret] }]
+      }
+    end
     let(:pointers) { ['/nested/~/array/~'] }
 
     it 'includes array elements' do
-      expect(processed_data).to include(nested: [{array: ['secret', 'secret']}])
+      expect(processed_data).to include(nested: [{ array: %w[secret secret] }])
     end
   end
 
@@ -128,7 +134,7 @@ describe Logasm::Preprocessors::Whitelist do
     let(:pointers) { ['/array/0'] }
 
     it 'masks array element' do
-      expect(processed_data).to include(array: [{field: '*****'}])
+      expect(processed_data).to include(array: [{ field: '*****' }])
     end
   end
 
@@ -136,7 +142,7 @@ describe Logasm::Preprocessors::Whitelist do
     let(:pointers) { ['/array'] }
 
     it 'masks array' do
-      expect(processed_data).to include(array: [{field: '*****'}])
+      expect(processed_data).to include(array: ['*****'])
     end
   end
 
@@ -144,12 +150,12 @@ describe Logasm::Preprocessors::Whitelist do
     let(:pointers) { ['/data'] }
 
     it 'masks hash' do
-      expect(processed_data).to include(data: {field: '*****'})
+      expect(processed_data).to include(data: { field: '*****' })
     end
   end
 
   context 'when boolean present' do
-    let(:data) { {bool: true} }
+    let(:data) { { bool: true } }
 
     it 'masks it with asteriks' do
       expect(processed_data).to eq(bool: '*****')
@@ -157,31 +163,9 @@ describe Logasm::Preprocessors::Whitelist do
   end
 
   context 'when field has slash in the name' do
-    let(:data) {{
-      'field_with_/' => 'secret'
-    }}
-    let(:pointers) { ['/field_with_~1'] }
-
-    it 'includes field' do
-      expect(processed_data).to include('field_with_/'=> 'secret')
-    end
-  end
-
-  context 'when field has tilde in the name' do
-    let(:data) {{
-      'field_with_~' => 'secret'
-    }}
-    let(:pointers) { ['/field_with_~0'] }
-
-    it 'includes field' do
-      expect(processed_data).to include('field_with_~'=> 'secret')
+    let(:data) do
+      { 'field_with_/' => 'secret' }
     end
-  end
-
-  context 'when field has slash in the name' do
-    let(:data) {{
-      'field_with_/' => 'secret'
-    }}
     let(:pointers) { ['/field_with_~1'] }
 
     it 'includes field' do
@@ -190,9 +174,9 @@ describe Logasm::Preprocessors::Whitelist do
   end
 
   context 'when field has tilde in the name' do
-    let(:data) {{
-      'field_with_~' => 'secret'
-    }}
+    let(:data) do
+      { 'field_with_~' => 'secret' }
+    end
     let(:pointers) { ['/field_with_~0'] }
 
     it 'includes field' do
@@ -201,9 +185,9 @@ describe Logasm::Preprocessors::Whitelist do
   end
 
   context 'when field has tilde and 1' do
-    let(:data) {{
-      'field_with_~1' => 'secret'
-    }}
+    let(:data) do
+      { 'field_with_~1' => 'secret' }
+    end
     let(:pointers) { ['/field_with_~01'] }
 
     it 'includes field' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logasm
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.9.0
 platform: ruby
 authors:
 - Salemove
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-24 00:00:00.000000000 Z
+date: 2017-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inflecto
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: lru_redux
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark-ips
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: It's logasmic
 email:
 - support@salemove.com
@@ -80,6 +108,7 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- benchmark/whitelisting.rb
 - lib/logasm.rb
 - lib/logasm/adapters.rb
 - lib/logasm/adapters/logstash_adapter.rb
@@ -90,6 +119,7 @@ files:
 - lib/logasm/null_logger.rb
 - lib/logasm/preprocessors.rb
 - lib/logasm/preprocessors/blacklist.rb
+- lib/logasm/preprocessors/json_pointer_trie.rb
 - lib/logasm/preprocessors/whitelist.rb
 - lib/logasm/utils.rb
 - logasm.gemspec
@@ -100,6 +130,7 @@ files:
 - spec/adapters/stdout_json_adapter_spec.rb
 - spec/logasm_spec.rb
 - spec/preprocessors/blacklist_spec.rb
+- spec/preprocessors/json_pointer_trie.rb
 - spec/preprocessors/whitelist_spec.rb
 - spec/spec_helper.rb
 - spec/support/implement_interface.rb
@@ -124,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: What description said
@@ -136,6 +167,7 @@ test_files:
 - spec/adapters/stdout_json_adapter_spec.rb
 - spec/logasm_spec.rb
 - spec/preprocessors/blacklist_spec.rb
+- spec/preprocessors/json_pointer_trie.rb
 - spec/preprocessors/whitelist_spec.rb
 - spec/spec_helper.rb
 - spec/support/implement_interface.rb