lenjador 1.2.1 → 1.3.0

data/lib/lenjador/utils.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
+
 require 'time'
 
 class Lenjador
   module Utils
     DECIMAL_FRACTION_OF_SECOND = 3
+    NO_TRACE_INFORMATION = {}.freeze
 
     # Build logstash json compatible event
     #
@@ -13,8 +16,9 @@ class Lenjador
     # @return [Hash]
     def self.build_event(metadata, level, application_name)
       overwritable_params
-        .merge(metadata)
-        .merge(
+        .merge!(metadata)
+        .merge!(tracing_information)
+        .merge!(
           application: application_name,
           level: level
         )
@@ -37,6 +41,7 @@ class Lenjador
         :@timestamp => Time.now.utc.iso8601(DECIMAL_FRACTION_OF_SECOND)
       }
     end
+    private_class_method :overwritable_params
 
     def self.serialize_time_objects!(object)
       if object.is_a?(Hash)
@@ -67,7 +72,7 @@ class Lenjador
       end
     else
       require 'oj'
-      DUMP_OPTIONS = { mode: :compat, time_format: :ruby }.freeze
+      DUMP_OPTIONS = {mode: :compat, time_format: :ruby}.freeze
 
       def self.generate_json(obj)
         serialize_time_objects!(obj)
@@ -79,13 +84,35 @@ class Lenjador
     def self.underscore(input)
       word = input.to_s.dup
       word.gsub!(/::/, '/')
-      word.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
-      word.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
-      word.tr!("-", "_")
+      word.gsub!(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+      word.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
+      word.tr!('-', '_')
       word.downcase!
       word
     end
 
-    private_class_method :overwritable_params
+    # Tracing information
+    #
+    # Tracing information is included only if OpenTracing is defined and if it
+    # supports method called `active_span` (version >= 0.4.1). We use
+    # SpanContext#trace_id and SpanContext#span_id methods to retrieve tracing
+    # information. These methods are not yet supported by the OpenTracing API,
+    # so we first check if these methods exist. Popular tracing libraries
+    # already implement them. These methods are likely to be added to the API
+    # very soon: https://github.com/opentracing/specification/blob/master/rfc/trace_identifiers.md
+    def self.tracing_information
+      return NO_TRACE_INFORMATION if !defined?(OpenTracing) || !OpenTracing.respond_to?(:active_span)
+
+      context = OpenTracing.active_span&.context
+      if context && context.respond_to?(:trace_id) && context.respond_to?(:span_id)
+        {
+          trace_id: context.trace_id,
+          span_id: context.span_id
+        }
+      else
+        NO_TRACE_INFORMATION
+      end
+    end
+    private_class_method :tracing_information
   end
 end

data/profiler/logs.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Run with `ruby profiler/logs.rb > /dev/null` and then you can read the
+# results using `open /tmp/lenjador.html`
+
+require 'bundler/setup'
+Bundler.require
+logger = Lenjador.build('test_service', stdout: {level: 'info', json: true})
+
+require 'ruby-prof'
+RubyProf.start
+
+100_000.times do
+  logger.info 'hello there', a: 'asdf', b: 'eadsfasdf', c: {hello: 'there'}
+end
+
+result = RubyProf.stop
+printer = RubyProf::GraphHtmlPrinter.new(result)
+File.open('/tmp/lenjador.html', 'w+') { |file| printer.print(file) }

data/spec/{adapters → lenjador/adapters}/stdout_adapter_spec.rb

@@ -1,5 +1,5 @@
 require 'spec_helper'
-require_relative '../../lib/lenjador/adapters/stdout_adapter'
+require 'lenjador/adapters/stdout_adapter'
 
 describe Lenjador::Adapters::StdoutAdapter do
   it 'creates a stdout logger' do

data/spec/{adapters → lenjador/adapters}/stdout_json_adapter_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 require 'json'
-require_relative '../../lib/lenjador/adapters/stdout_json_adapter'
+require 'lenjador/adapters/stdout_json_adapter'
 
 describe Lenjador::Adapters::StdoutJsonAdapter do
   let(:debug_level_code) { 0 }

data/spec/{preprocessors → lenjador/preprocessors}/blacklist_spec.rb

@@ -1,21 +1,25 @@
 require 'spec_helper'
-require_relative '../../lib/lenjador/preprocessors/blacklist'
+require_relative '../../../lib/lenjador/preprocessors/blacklist'
 
 describe Lenjador::Preprocessors::Blacklist do
   subject(:processed_data) { described_class.new(config).process(data) }
 
-  let(:config) {{
-    fields: [{ key: 'field', action: action }]
-  }}
+  let(:config) do
+    {
+      fields: [{key: 'field', action: action}]
+    }
+  end
 
   let(:action) {}
-  let(:data) {{
-    field: value,
-    data: {
-      field: 'secret'
-    },
-    array: [{field: 'secret'}]
-  }}
+  let(:data) do
+    {
+      field: value,
+      data: {
+        field: 'secret'
+      },
+      array: [{field: 'secret'}]
+    }
+  end
 
   let(:value) { 'secret' }
 
@@ -39,7 +43,7 @@ describe Lenjador::Preprocessors::Blacklist do
     end
 
     it 'removes nested in array field' do
-      expect(processed_data[:array]).not_to include({field: 'secret'})
+      expect(processed_data[:array]).not_to include(field: 'secret')
     end
 
     context 'when field is deeply nested' do
@@ -60,7 +64,7 @@ describe Lenjador::Preprocessors::Blacklist do
     end
 
     it 'masks nested in array field' do
-      expect(processed_data[:array]).to include({field: '*****'})
+      expect(processed_data[:array]).to include(field: '*****')
     end
 
     context 'when field is string' do
@@ -88,7 +92,7 @@ describe Lenjador::Preprocessors::Blacklist do
     end
 
     context 'when field is array' do
-      let(:value) { [1,2,3,4] }
+      let(:value) { [1, 2, 3, 4] }
 
       it 'masks value with asterisks' do
         expect(processed_data).to include(field: '*****')

data/spec/{preprocessors/json_pointer_trie.rb → lenjador/preprocessors/json_pointer_trie_spec.rb}

@@ -20,9 +20,9 @@ RSpec.describe Lenjador::Preprocessors::JSONPointerTrie do
     it 'returns false if trie does not contain requested prefix or value' do
       trie.insert('/data/nested/key')
 
-      expect(trie).to_not include('/bad_data')
-      expect(trie).to_not include('/data/bad_nested')
-      expect(trie).to_not include('/data/nested/bad_key')
+      expect(trie).not_to include('/bad_data')
+      expect(trie).not_to include('/data/bad_nested')
+      expect(trie).not_to include('/data/nested/bad_key')
     end
 
     it 'returns true if trie contains requested prefix under wildcard' do
@@ -30,7 +30,7 @@ RSpec.describe Lenjador::Preprocessors::JSONPointerTrie do
 
       expect(trie).to include('/data/arbitrary_key/key')
       expect(trie).to include('/data/another_key/key')
-      expect(trie).to_not include('/data/arbitrary_key/bad_nested_key')
+      expect(trie).not_to include('/data/arbitrary_key/bad_nested_key')
     end
   end
 end

data/spec/lenjador/preprocessors/whitelist_spec.rb

@@ -0,0 +1,335 @@
+require 'spec_helper'
+require_relative '../../../lib/lenjador/preprocessors/whitelist'
+
+RSpec.describe Lenjador::Preprocessors::Whitelist do
+  context 'when :action is :mask or omitted' do
+    subject(:processed_data) { described_class.new(config).process(data) }
+
+    let(:config) { {pointers: pointers} }
+    let(:pointers) { [] }
+    let(:data) do
+      {
+        field: 'secret',
+        data: {
+          field: 'secret'
+        },
+        array: [{field: 'secret'}]
+      }
+    end
+
+    it 'masks all non-whitelisted fields' do
+      expect(processed_data).to eq(
+        field: '*****',
+        data: '*****',
+        array: '*****'
+      )
+    end
+
+    context 'when pointer has trailing slash' do
+      let(:pointers) { ['/field/'] }
+
+      it 'throws exception' do
+        expect { processed_data }.to raise_exception(Lenjador::Preprocessors::Whitelist::InvalidPointerFormatException)
+      end
+    end
+
+    context 'with whitelisted field' do
+      let(:pointers) { ['/field'] }
+
+      it 'includes the field' do
+        expect(processed_data).to eq(
+          field: 'secret',
+          data: '*****',
+          array: '*****'
+        )
+      end
+    end
+
+    context 'with whitelisted nested field' do
+      let(:pointers) { ['/data/field'] }
+
+      it 'includes nested field' do
+        expect(processed_data).to eq(
+          field: '*****',
+          data: {
+            field: 'secret'
+          },
+          array: '*****'
+        )
+      end
+    end
+
+    context 'with whitelisted array element field' do
+      let(:pointers) { ['/array/0/field'] }
+
+      it 'includes array element' do
+        expect(processed_data).to eq(
+          field: '*****',
+          data: '*****',
+          array: [{field: 'secret'}]
+        )
+      end
+    end
+
+    context 'with whitelisted hash' do
+      it 'includes all whitelisted hash elements' do
+        source = {foo: {bar: 'baz'}}
+        target = {foo: {bar: 'baz'}}
+        expect(process(['/foo/~'], source)).to eq(target)
+      end
+
+      it 'does not include nested elements' do
+        source = {foo: {bar: {baz: 'asd'}}}
+        target = {foo: {bar: {baz: '*****'}}}
+        expect(process(['/foo/~'], source)).to eq(target)
+      end
+    end
+
+    context 'with whitelisted array elements field with wildcard' do
+      let(:data) do
+        {
+          array: [
+            {field: 'data1', secret: 'secret1'},
+            {field: 'data2', secret: 'secret2'}
+          ]
+        }
+      end
+      let(:pointers) { ['/array/~/field'] }
+
+      it 'includes array elements field' do
+        expect(processed_data).to include(
+          array: [
+            {field: 'data1', secret: '*****'},
+            {field: 'data2', secret: '*****'}
+          ]
+        )
+      end
+    end
+
+    context 'with whitelisted string array elements with wildcard' do
+      let(:data) do
+        {array: %w[secret secret]}
+      end
+      let(:pointers) { ['/array/~'] }
+
+      it 'includes array elements' do
+        expect(processed_data).to include(array: %w[secret secret])
+      end
+    end
+
+    context 'with whitelisted string array elements in an array with wildcard' do
+      let(:data) do
+        {
+          nested: [{array: %w[secret secret]}]
+        }
+      end
+      let(:pointers) { ['/nested/~/array/~'] }
+
+      it 'includes array elements' do
+        expect(processed_data).to include(nested: [{array: %w[secret secret]}])
+      end
+    end
+
+    context 'with whitelisted array element' do
+      let(:pointers) { ['/array/0'] }
+
+      it 'masks array element' do
+        expect(processed_data).to include(array: [{field: '*****'}])
+      end
+    end
+
+    context 'with whitelisted array' do
+      let(:pointers) { ['/array'] }
+
+      it 'masks array' do
+        expect(processed_data).to include(array: ['*****'])
+      end
+    end
+
+    context 'with whitelisted hash' do
+      let(:pointers) { ['/data'] }
+
+      it 'masks hash' do
+        expect(processed_data).to include(data: {field: '*****'})
+      end
+    end
+
+    context 'when boolean present' do
+      let(:data) { {bool: true} }
+
+      it 'masks it with asteriks' do
+        expect(processed_data).to eq(bool: '*****')
+      end
+    end
+
+    context 'when field has slash in the name' do
+      let(:data) do
+        {'field_with_/' => 'secret'}
+      end
+      let(:pointers) { ['/field_with_~1'] }
+
+      it 'includes field' do
+        expect(processed_data).to include('field_with_/' => 'secret')
+      end
+    end
+
+    context 'when field has tilde in the name' do
+      let(:data) do
+        {'field_with_~' => 'secret'}
+      end
+      let(:pointers) { ['/field_with_~0'] }
+
+      it 'includes field' do
+        expect(processed_data).to include('field_with_~' => 'secret')
+      end
+    end
+
+    context 'when field has tilde and 1' do
+      let(:data) do
+        {'field_with_~1' => 'secret'}
+      end
+      let(:pointers) { ['/field_with_~01'] }
+
+      it 'includes field' do
+        expect(processed_data).to include('field_with_~1' => 'secret')
+      end
+    end
+
+    def process(pointers, data)
+      described_class.new(pointers: pointers).process(data)
+    end
+  end
+
+  context 'when :action is :exclude or :prune' do
+    subject(:processed_data) { described_class.new(config).process(data) }
+
+    let(:config) { {pointers: pointers, action: :prune} }
+    let(:pointers) { [] }
+    let(:data) do
+      {
+        field: 'secret',
+        data: {
+          field: 'secret'
+        },
+        array: [{field: 'secret'}, {field2: 'secret'}]
+      }
+    end
+
+    context 'when pointers is empty' do
+      it 'prunes all fields from the input' do
+        expect(processed_data).to eq({})
+      end
+    end
+
+    context 'with whitelisted field' do
+      let(:pointers) { ['/field'] }
+
+      it 'includes the field' do
+        expect(processed_data).to eq(field: 'secret')
+      end
+    end
+
+    context 'with whitelisted nested field' do
+      let(:pointers) { ['/data/field'] }
+
+      it 'includes nested field' do
+        expect(processed_data).to eq(data: {field: 'secret'})
+      end
+    end
+
+    context 'with whitelisted array element field' do
+      let(:pointers) { ['/array/0/field'] }
+
+      it 'includes array element' do
+        expect(processed_data).to eq(array: [{field: 'secret'}])
+      end
+    end
+
+    context 'with whitelisted hash' do
+      it 'includes all whitelisted hash elements' do
+        source = {foo: {bar: 'baz'}}
+        target = {foo: {bar: 'baz'}}
+        expect(process(['/foo/~'], source)).to eq(target)
+      end
+
+      it 'does not include nested elements' do
+        source = {foo: {bar: {baz: 'asd'}}}
+        target = {foo: {bar: {}}}
+        expect(process(['/foo/~'], source)).to eq(target)
+      end
+    end
+
+    context 'with whitelisted array elements field with wildcard' do
+      let(:data) do
+        {
+          array: [
+            {field: 'data1', secret: 'secret1'},
+            {field: 'data2', secret: 'secret2'}
+          ]
+        }
+      end
+      let(:pointers) { ['/array/~/field'] }
+
+      it 'includes array elements field' do
+        expect(processed_data).to include(
+          array: [
+            {field: 'data1'},
+            {field: 'data2'}
+          ]
+        )
+      end
+    end
+
+    context 'with whitelisted string array elements with wildcard' do
+      let(:data) do
+        {array: %w[secret1 secret2]}
+      end
+      let(:pointers) { ['/array/~'] }
+
+      it 'includes array elements' do
+        expect(processed_data).to include(array: %w[secret1 secret2])
+      end
+    end
+
+    context 'with whitelisted string array elements in an array with wildcard' do
+      let(:data) do
+        {
+          nested: [{array: %w[secret1 secret2]}]
+        }
+      end
+      let(:pointers) { ['/nested/~/array/~'] }
+
+      it 'includes array elements' do
+        expect(processed_data).to include(nested: [{array: %w[secret1 secret2]}])
+      end
+    end
+
+    context 'with whitelisted array element' do
+      let(:pointers) { ['/array/0'] }
+
+      it 'masks array element' do
+        expect(processed_data).to eq(array: [{}])
+      end
+    end
+
+    context 'with whitelisted array' do
+      let(:pointers) { ['/array'] }
+
+      it 'masks array' do
+        expect(processed_data).to include(array: [])
+      end
+    end
+
+    context 'with whitelisted hash' do
+      let(:pointers) { ['/data'] }
+
+      it 'masks hash' do
+        expect(processed_data).to include(data: {})
+      end
+    end
+
+    def process(pointers, data)
+      described_class.new(pointers: pointers, action: :prune).process(data)
+    end
+  end
+end