kiev 4.3.0 → 4.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 077b62ba98a1a883bb9cf7596983156864e1011bdb189ee80cc1d0e8e4039c08
-  data.tar.gz: beb8bdd2f1d31501d5ef4d501d221c1fabe522b9a106e4f7bb150e46aa168029
+  metadata.gz: 5da27510343d0217fedb4e4282cf9fc6468a63be4484935e2495e31205a2b188
+  data.tar.gz: 3a7ac16381946c418fa4265ab440da10397514766e5c091a87847f4451644251
 SHA512:
-  metadata.gz: bb2ff70083d47ac226628cb796f34f607454c6db6b458a63cca06fdbf014a75f28d02ddc29a7b9caf8e4c309148b0adfbfc0a4cdbafcf1030d4e7996a65cf4f5
-  data.tar.gz: 3fb4a342b5d0b475da14cd99f553ee3b5ee02a1f299f11e2d1afb0ab11353672a14373850b657ca9cc755d6d4474439d32d95b09bc94177c749f9b04eda71a3e
+  metadata.gz: ee5e47a70aae2576a0d119c0347fa9707f8a1ff53e8db3f5c1b8ae0bcc39907fd755f1d6112eca317e3b22ce8e2bdb59e83f3e9b70b86014fae995a4377da466
+  data.tar.gz: d976cb714c3e9ca5a2f3aa9c0a505a78d38d4a1c77faddc408555dec2ff3f337dbc9430d881630165933084e2a5c88051ef6fbe530a2c79ebf2f24b64949a329

data/.github/workflows/push.yml

@@ -0,0 +1,80 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Main CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: ['2.5.1', '2.7.2', '2.6.5']
+        gemfile:
+          - gemfiles/ruby_kafka.gemfile
+          - gemfiles/que_0.12.2.gemfile
+          - gemfiles/que_0.12.3.gemfile
+          - gemfiles/rails_5.2.gemfile
+          - gemfiles/shoryuken_4.0.gemfile
+          - gemfiles/sidekiq_4.2.gemfile
+          - gemfiles/sinatra_1.4.gemfile
+          - gemfiles/sinatra_2.0.gemfile
+        allow_failures:
+          - false
+        include:
+          - os: ubuntu
+            ruby-version: ruby-head
+            gemfile: gemfiles/rails_5.2.gemfile
+            allow_failures: true
+    env:
+      BUNDLE_GEMFILE: "${{ matrix.gemfile }}"
+      ALLOW_FAILURES: "${{ matrix.allow_failures }}"
+      REDIS_URL: "redis://localhost:6379/4"
+      DATABASE_URL: ${{ (startsWith(matrix.gemfile,'gemfiles/que') && 'postgres://postgres:postgres@localhost:5432/que_test') || 'sqlite3:db/combustion_test.sqlite'}}
+    continue-on-error: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'debug' }}
+
+    # Service containers to run with `container-job`
+    services:
+      redis:
+        image: redis:latest
+        ports:
+          - 6379:6379
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:9.4
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: que_test
+        # Set health checks to wait until postgres has started
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+        # change this to (see https://github.com/ruby/setup-ruby#versioning):
+        # uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+      - name: Run tests
+        run: bundle exec rake || $ALLOW_FAILURES

data/.rubocop.yml

@@ -1,7 +1,14 @@
 inherit_from:
   - https://raw.githubusercontent.com/blacklane/rubocop/master/rubocop.yml
 
-Lint/HandleExceptions:
+AllCops:
+  TargetRubyVersion: 2.5
+  Exclude:
+    - test/rails_app/**/*.rb # auto-generated
+    - spec/**/*.rb
+    - test/**/*.rb
+    - vendor/bundle/**/*
+Lint/SuppressedException:
   Exclude:
     - test/**/*.rb
     - spec/**/*.rb

data/.ruby-version

@@ -1 +1 @@
-2.3.3
+2.5.1

data/Gemfile

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
-eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_4.1.gemfile"))
+eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_5.2.gemfile"))
 
 gem "wwtd"

data/README.md

@@ -1,4 +1,4 @@
-# Kiev [![Build Status](https://travis-ci.org/blacklane/kiev.svg?branch=master)](https://travis-ci.org/blacklane/kiev)
+# Kiev [![Build Status](https://github.com/blacklane/kiev/workflows/Main%20CI/badge.svg?branch=master)](https://github.com/blacklane/kiev/actions?query=workflow%3A%22Main+CI%22) [![Gem Version](https://badge.fury.io/rb/kiev.svg)](https://badge.fury.io/rb/kiev)
 
 Kiev is a comprehensive logging library aimed at covering a wide range of frameworks and tools from the Ruby ecosystem:
 
@@ -154,6 +154,36 @@ Kiev::Shoryuken.enable
 
 The name of the worker class is not logged by default. Configure [`persistent_log_fields` option](#persistent_log_fields) to include `"shoryuken_class"` if you want this.
 
+### AWS SNS
+
+To enhance messages published to SNS topics you can use the ContextInjector:
+
+```ruby
+sns_message = { topic_arn: "...",  message: "{...}" }
+Kiev::Kafka.inject_context(sns_message[:message_attributes])
+
+```
+
+After this operation the message attributes will also include required context for the Kiev logger.
+
+### Kafka
+
+To enhance messages published to Kafka topics you can use the ContextInjector:
+
+```ruby
+Kiev::Kafka.inject_context(headers)
+```
+
+After this operation the headers variable will also include required context for the Kiev logger.
+
+If you have a consumed `Kafka::FetchedMessage` you can extract logger context with: 
+
+```ruby
+Kiev::Kafka.extract_context(message)
+```
+
+This will work regardless if headers are in HTTP format, e.g. `X-Tracking-Id` or plain field names: `tracking_id`. Plus the `message_key` field will contain the key of processed message. In case you want to log some more fields configure `persistent_log_fields` and `jobs_propagated_fields`.  
+
 ### Que
 
 Add the following lines to your initializer code:
@@ -219,7 +249,9 @@ For web requests the Kiev middleware will log the following information by defau
 
 * `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option.
 
-* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4).
+* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). (currently deprecated in favor of a new name: `tracking_id`)
+
+* `tracking_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). If not provided the value is seeded from deprecated `request_id`. 
 
 * `request_depth` represents the position of the current request within a chain of requests. It starts with 0.
 

data/gemfiles/que_0.12.2.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -11,4 +13,4 @@ gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/que_0.12.3.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 # need it because of bug in https://github.com/chanks/que/issues/191
@@ -12,4 +14,4 @@ gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4.1.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -10,4 +12,4 @@ gem "rspec", require: false
 gem "rspec-rails", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4.2.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -10,4 +12,4 @@ gem "rspec", require: false
 gem "rspec-rails", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "oj"
+
+gem "rails", "~> 5.2.4"
+gem "sqlite3", "1.3.13"
+
+gem "combustion"
+gem "rspec", require: false
+gem "rspec-rails", require: false
+gem "minitest-reporters", require: false
+
+gemspec path: "../"

data/gemfiles/ruby_kafka.gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "ruby-kafka", "~> 0.7.10"
+
+gem "rack-test", require: false
+gem "rspec", require: false
+gem "minitest-reporters", require: false
+
+gemspec path: "../"

data/gemfiles/{shoryuken_3.1.gemfile → shoryuken_4.0.gemfile}

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "aws-sdk", "~> 2.0"
-gem "shoryuken", "~> 3.1.0"
+gem "shoryuken", "~> 4.0"
 
 gem "rack-test", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/sidekiq_4.2.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj", "~> 2"
@@ -10,5 +12,4 @@ gem "minitest-reporters", require: false
 
 gem "her"
 
-gemspec :path => "../"
-
+gemspec path: "../"

data/gemfiles/sinatra_1.4.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -6,10 +8,10 @@ gem "xml-simple"
 
 gem "sinatra", "1.4.7"
 gem "sinatra-contrib"
-gem "rack-parser", :require => "rack/parser"
+gem "rack-parser", require: "rack/parser"
 
 gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/sinatra_2.0.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -6,10 +8,10 @@ gem "xml-simple"
 
 gem "sinatra", "2.0.0"
 gem "sinatra-contrib"
-gem "rack-parser", :require => "rack/parser"
+gem "rack-parser", require: "rack/parser"
 
 gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/kiev.gemspec

@@ -9,7 +9,9 @@ Gem::Specification.new do |spec|
   spec.licenses      = ["MIT"]
 
   spec.summary       = "Distributed logging to JSON integrated with various Ruby frameworks and tools"
-  spec.description   = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing human-readable output in development mode. It integrates nicely with Rails, Sinatra and other Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients."
+  spec.description   = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing "\
+                        "human-readable output in development mode. It integrates nicely with Rails, Sinatra and other"\
+                        " Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients."
   spec.homepage      = "https://github.com/blacklane/kiev"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -17,12 +19,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 2.0.0"
+  spec.required_ruby_version = ">= 2.5"
+  spec.add_dependency "oga", "~> 2.2"
   spec.add_dependency "rack", ">= 1", "< 3"
   spec.add_dependency "request_store", ">= 1.0", "< 1.4"
-  spec.add_dependency "oga", "~> 2.2"
   spec.add_dependency "ruby_dig", "~> 0.0.2" # to support ruby 2.2
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rubocop", "0.49.1"
+  spec.add_development_dependency "rake", "~> 0"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "rubocop", "~> 0.54"
 end

data/lib/kiev/aws_sns/context_injector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module AwsSns
+    class ContextInjector
+      # @param [Hash] message_attributes Injects context headers
+      # @return [Hash]
+      def call(message_attributes = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          message_attributes[key] = {
+            data_type: "String",
+            string_value: value.to_s
+          }
+        end
+        message_attributes
+      end
+    end
+  end
+end

data/lib/kiev/aws_sns.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module AwsSns
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::AwsSns::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/base.rb

@@ -26,8 +26,20 @@ module Kiev
       Config.instance.logger
     end
 
-    def event(event_name, data = EMPTY_OBJ)
-      logger.log(::Logger::Severity::INFO, data, event_name)
+    def filtered_params
+      Config.instance.filtered_params
+    end
+
+    def ignored_params
+      Config.instance.ignored_params
+    end
+
+    def event(log_name, data = EMPTY_OBJ)
+      logger.log(
+        ::Logger::Severity::INFO,
+        ParamFilter.filter(data, filtered_params, ignored_params),
+        log_name
+      )
     end
 
     def []=(name, value)
@@ -47,7 +59,9 @@ module Kiev
     end
 
     def request_id
-      RequestStore.store[:request_id]
+      RequestStore.store[:tracking_id]
     end
+
+    alias_method :tracking_id, :request_id
   end
 end

data/lib/kiev/base52.rb

@@ -3,7 +3,7 @@
 module Kiev
   module Base52
     KEYS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".freeze
-    BASE = KEYS.length.freeze
+    BASE = KEYS.length
 
     def self.encode(num)
       return KEYS[0] if num == 0

data/lib/kiev/config.rb

@@ -63,6 +63,7 @@ module Kiev
       ) << :tempfile).freeze
 
     DEFAULT_HTTP_PROPAGATED_FIELDS = {
+      tracking_id: "X-Tracking-Id",
       request_id: "X-Request-Id",
       request_depth: "X-Request-Depth",
       tree_path: "X-Tree-Path"

data/lib/kiev/context_reader.rb

@@ -6,6 +6,7 @@ module Kiev
   # change field lookup.
   class ContextReader
     REQUEST_ID = "request_id"
+    TRACKING_ID = "tracking_id"
     REQUEST_DEPTH = "request_depth"
     TREE_PATH = "tree_path"
 
@@ -17,12 +18,14 @@ module Kiev
       subject[key]
     end
 
-    def request_id
-      self[REQUEST_ID] || SecureRandom.uuid
+    def tracking_id
+      self[TRACKING_ID] || self[REQUEST_ID] || SecureRandom.uuid
     end
 
+    alias_method :request_id, :tracking_id
+
     def tree_root?
-      !self[REQUEST_ID]
+      !self[TRACKING_ID] && !self[REQUEST_ID]
     end
 
     def request_depth

data/lib/kiev/kafka/context_extractor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "message_context"
+require "kiev/request_id"
+require "kiev/context_reader"
+
+module Kiev
+  module Kafka
+    class ContextExtractor
+      include Kiev::RequestId::Mixin
+
+      # @param [Kafka::FetchedMessage] message
+      def call(message)
+        context = Kiev::Kafka::MessageContext.new(message)
+        context_reader = Kiev::ContextReader.new(context)
+        wrap_request_id(context_reader) {}
+
+        Kiev[:message_key] = message.key
+
+        Config.instance.jobs_propagated_fields.each do |key|
+          Kiev[key] = context_reader[key]
+        end
+      end
+    end
+  end
+end

data/lib/kiev/kafka/context_injector.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module Kafka
+    class ContextInjector
+      # @param [Hash] headers Injects context headers
+      # @return [Hash]
+      def call(headers = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          field_key = Kiev::Config::DEFAULT_HTTP_PROPAGATED_FIELDS.fetch(key.to_sym, key)
+          headers[field_key] = value
+        end
+        headers
+      end
+    end
+  end
+end

data/lib/kiev/kafka/message_context.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Kafka
+    class MessageContext
+      # @param [Kafka::FetchedMessage] message
+      def initialize(message)
+        @headers = message.headers
+      end
+
+      def value(field)
+        headers[header_key(field)] || headers[field.to_s]
+      end
+
+      alias_method :[], :value
+
+      private
+
+      attr_reader :headers
+
+      # @param [String] field
+      def header_key(field)
+        "x_#{field}".gsub("_", " ").split.map(&:capitalize).join("-")
+      end
+    end
+  end
+end

data/lib/kiev/kafka.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module Kafka
+    require_relative "kafka/context_extractor"
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Kafka::FetchedMessage] message
+      def extract_context(message)
+        Kiev::Kafka::ContextExtractor.new.call(message)
+      end
+
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::Kafka::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/logger.rb

@@ -12,17 +12,20 @@ module Kiev
     extend Forwardable
     def_delegators(*([:@logger] + ::Logger.instance_methods(false)))
 
-    DEFAULT_EVENT_NAME = "log"
+    DEFAULT_LOG_NAME = "log"
+    DEFAULT_MESSAGE = "log"
     LOG_ERROR = "ERROR"
     ERROR_STATUS = 500
 
-    FORMATTER = proc do |severity, time, event_name, data|
+    FORMATTER = proc do |severity, time, log_name, data|
       entry =
         {
           application: Config.instance.app,
-          event: event_name || DEFAULT_EVENT_NAME,
+          log_name: log_name || DEFAULT_LOG_NAME,
           level: severity,
           timestamp: time.utc,
+          message: log_name || DEFAULT_MESSAGE,
+          tracking_id: RequestStore.store[:tracking_id],
           request_id: RequestStore.store[:request_id],
           request_depth: RequestStore.store[:request_depth],
           tree_path: RequestStore.store[:tree_path]
@@ -38,12 +41,12 @@ module Kiev
         entry[:jid] = RequestStore.store[:jid]
       end
 
-      if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(event_name)
+      if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(log_name)
         entry[:tree_leaf] = true
       end
 
       if RequestStore.store[:payload]
-        if %i(request_finished job_finished).include?(event_name)
+        if %i(request_finished job_finished).include?(log_name)
           entry.merge!(RequestStore.store[:payload])
         else
           Config.instance.persistent_log_fields.each do |field|
@@ -67,17 +70,17 @@ module Kiev
       JSON.logstash(entry)
     end
 
-    DEVELOPMENT_FORMATTER = proc do |severity, time, event_name, data|
+    DEVELOPMENT_FORMATTER = proc do |severity, time, log_name, data|
       entry = []
 
       entry << time.iso8601
-      entry << (event_name || severity).upcase
+      entry << (log_name || severity).upcase
 
       if data.is_a?(String)
         entry << "#{data}\n"
       end
 
-      if %i(request_finished job_finished).include?(event_name)
+      if %i(request_finished job_finished).include?(log_name)
         verb = RequestStore.store[:request_verb]
         path = RequestStore.store[:request_path]
         entry << "#{verb} #{path}" if verb && path
@@ -92,10 +95,8 @@ module Kiev
         entry << "(#{duration}ms)" if duration
         entry << "\n"
 
-        meta = {
-          request_id: RequestStore.store[:request_id],
-          request_depth: RequestStore.store[:request_depth]
-        }.merge!(Hash(RequestStore.store[:payload]))
+        meta = RequestStore.store.slice(:trakcing_id, :request_id, :request_depth)
+                           .reverse_merge!(Hash(RequestStore.store[:payload]))
 
         meta.reject! { |_, value| value.nil? }
 

data/lib/kiev/param_filter.rb

@@ -1,12 +1,23 @@
 # frozen_string_literal: true
 
 module Kiev
-  module ParamFilter
+  class ParamFilter
     FILTERED = "[FILTERED]"
 
     def self.filter(params, filtered_params, ignored_params)
+      new(filtered_params, ignored_params).call(params)
+    end
+
+    def initialize(filtered_params, ignored_params)
+      @filtered_params = normalize(filtered_params)
+      @ignored_params = normalize(ignored_params)
+    end
+
+    def call(params)
+      return params unless filterable?(params)
+
       params.each_with_object({}) do |(key, value), acc|
-        next if ignored_params.include?(key)
+        next if ignored_params.include?(key.to_s)
 
         if defined?(ActionDispatch) && value.is_a?(ActionDispatch::Http::UploadedFile)
           value = {
@@ -17,14 +28,26 @@ module Kiev
         end
 
         acc[key] =
-          if filtered_params.include?(key) && !value.is_a?(Hash)
+          if filtered_params.include?(key.to_s) && !value.is_a?(Hash)
             FILTERED
           elsif value.is_a?(Hash)
-            filter(value, filtered_params, ignored_params)
+            call(value)
           else
             value
           end
       end
     end
+
+    private
+
+    attr_reader :filtered_params, :ignored_params
+
+    def filterable?(params)
+      params.respond_to?(:each_with_object)
+    end
+
+    def normalize(params)
+      Set.new(params.map(&:to_s))
+    end
   end
 end

data/lib/kiev/que/job.rb

@@ -47,7 +47,8 @@ module Kiev
             Kiev[key] = payload[key]
           end
           request_store = Kiev::RequestStore.store
-          request_store[:request_id] = payload[:request_id]
+          request_store[:tracking_id] = payload[:tracking_id]
+          request_store[:request_id] = payload[:tracking_id] || payload[:request_id]
           request_store[:request_depth] = payload[:request_depth].to_i + 1
           request_store[:tree_path] = payload[:tree_path]
 

data/lib/kiev/rack/request_id.rb

@@ -14,24 +14,26 @@ module Kiev
 
       def call(env)
         request_id_header_out = to_rack(:request_id)
-        request_id_header_in = to_http(:request_id)
+        tracking_id_header_out = to_rack(:tracking_id)
 
-        request_id = make_request_id(env[RAILS_REQUEST_ID] || env[request_id_header_in])
-        RequestStore.store[:request_id] = request_id
+        tracking_id = make_tracking_id(env[to_http(:tracking_id)] || env[RAILS_REQUEST_ID] || env[to_http(:request_id)])
+        RequestStore.store[:tracking_id] = tracking_id
+        RequestStore.store[:request_id] = tracking_id
         RequestStore.store[:request_depth] = request_depth(env)
         RequestStore.store[:tree_path] = tree_path(env)
 
-        @app.call(env).tap { |_status, headers, _body| headers[request_id_header_out] = request_id }
+        @app.call(env).tap do |_status, headers, _body|
+          headers[tracking_id_header_out] = tracking_id
+          headers[request_id_header_out] = tracking_id
+        end
       end
 
       private
 
-      # TODO: in Rails 5 they set `headers[X_REQUEST_ID]`, so this will not work
-      # https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/request_id.rb
-      # https://github.com/interagent/pliny/blob/master/lib/pliny/middleware/request_id.rb
       def tree_root?(env)
+        tracking_id_header_in = to_http(:tracking_id)
         request_id_header_in = to_http(:request_id)
-        !env[request_id_header_in]
+        !env[tracking_id_header_in] && !env[request_id_header_in]
       end
 
       def request_depth(env)
@@ -52,15 +54,15 @@ module Kiev
         Config.instance.all_http_propagated_fields[value]
       end
 
-      def make_request_id(request_id)
-        if request_id.nil? || request_id.empty?
-          internal_request_id
+      def make_tracking_id(tracking_id)
+        if tracking_id.nil? || tracking_id.empty?
+          internal_tracking_id
         else
-          Util.sanitize(request_id)
+          Util.sanitize(tracking_id)
         end
       end
 
-      def internal_request_id
+      def internal_tracking_id
         SecureRandom.uuid
       end
     end

data/lib/kiev/rack/request_logger.rb

@@ -85,8 +85,6 @@ module Kiev
             request.params
           end
 
-        params = ParamFilter.filter(params, config.filtered_params, config.ignored_params)
-
         data = {
           host: request.host, # env["HTTP_HOST"] || env["HTTPS_HOST"],
           params: params.empty? ? nil : params, # env[Rack::QUERY_STRING],
@@ -125,8 +123,8 @@ module Kiev
               sio = StringIO.new(data[:body])
               gz = Zlib::GzipReader.new(sio)
               data[:body] = gz.read
-            rescue Zlib::GzipFile::Error => err
-              data[:gzip_parse_error] = err.message
+            rescue Zlib::GzipFile::Error => e
+              data[:gzip_parse_error] = e.message
             end
           end
         end

data/lib/kiev/request_id.rb

@@ -7,7 +7,8 @@ module Kiev
 
       def wrap_request_id(context_reader, &_block)
         request_store = Kiev::RequestStore.store
-        request_store[:request_id] = context_reader.request_id
+        request_store[:tracking_id] = context_reader.tracking_id || context_reader.request_id
+        request_store[:request_id] = request_store[:tracking_id]
         request_store[:request_depth] = context_reader.request_depth
         request_store[:tree_path] = context_reader.tree_path
         yield

data/lib/kiev/request_logger.rb

@@ -12,8 +12,8 @@ module Kiev
 
         begin
           return_value = yield
-        rescue StandardError => exception
-          error = exception
+        rescue StandardError => e
+          error = e
         end
 
         begin
@@ -28,6 +28,7 @@ module Kiev
           Kiev.event(event, data)
         ensure
           raise error if error
+
           return_value
         end
       end

data/lib/kiev/shoryuken/context_reader.rb

@@ -12,8 +12,10 @@ module Kiev
 
       def [](key)
         return unless @message_attributes.key?(key)
+
         attribute_value = @message_attributes[key]
         return unless attribute_value.data_type == "String"
+
         attribute_value.string_value
       end
     end

data/lib/kiev/shoryuken/middleware/message_tracer.rb

@@ -1,17 +1,14 @@
 # frozen_string_literal: true
 
+require "kiev/aws_sns/context_injector"
+
 module Kiev
   module Shoryuken
     module Middleware
       class MessageTracer
         def call(options)
-          attrbutes = options[:message_attributes] ||= {}
-          SubrequestHelper.payload.each do |key, value|
-            attrbutes[key] = {
-              data_type: "String",
-              string_value: value.to_s
-            }
-          end
+          options[:message_attributes] ||= {}
+          Kiev::AwsSns::ContextInjector.new.call(options[:message_attributes])
           yield
         end
       end

data/lib/kiev/test.rb

@@ -25,8 +25,9 @@ module Kiev
 
       def entries
         return @logs unless @logs.empty?
+
         @logs = raw_logs.each_line.map(&::JSON.method(:parse))
-      rescue
+      rescue StandardError
         puts raw_logs
         raise
       end

data/lib/kiev/util.rb

@@ -4,6 +4,7 @@ module Kiev
   module Util
     def self.sanitize(value)
       return unless value
+
       value.gsub(/[^\w\-]/, "")[0...255]
     end
 

data/lib/kiev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kiev
-  VERSION = "4.3.0"
+  VERSION = "4.7.0"
 end

data/lib/kiev.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "kiev/base"
+require_relative "kiev/aws_sns" if defined?(AWS::SNS)
+require_relative "kiev/kafka" if defined?(Kafka)
 require_relative "kiev/rack" if defined?(Rack)
 require_relative "kiev/railtie" if defined?(Rails)
 require_relative "kiev/sidekiq" if defined?(Sidekiq)

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: kiev
 version: !ruby/object:Gem::Version
-  version: 4.3.0
+  version: 4.7.0
 platform: ruby
 authors:
 - Blacklane
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2021-10-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -50,20 +64,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.4'
-- !ruby/object:Gem::Dependency
-  name: oga
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: ruby_dig
   requirement: !ruby/object:Gem::Requirement
@@ -82,58 +82,58 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.49.1
+        version: '0.54'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.49.1
+        version: '0.54'
 description: Kiev is a logging tool aimed at distributed environments. It logs to
   JSON, while providing human-readable output in development mode. It integrates nicely
   with Rails, Sinatra and other Rack-based frameworks, Sidekiq, Que, HTTParty, Her
   and other Faraday-based HTTP clients.
-email:
+email: 
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/push.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
-- ".travis.yml"
 - Gemfile
 - LICENSE.md
 - README.md
@@ -144,13 +144,17 @@ files:
 - gemfiles/que_0.12.3.gemfile
 - gemfiles/rails_4.1.gemfile
 - gemfiles/rails_4.2.gemfile
-- gemfiles/shoryuken_3.1.gemfile
+- gemfiles/rails_5.2.gemfile
+- gemfiles/ruby_kafka.gemfile
+- gemfiles/shoryuken_4.0.gemfile
 - gemfiles/sidekiq_4.2.gemfile
 - gemfiles/sinatra_1.4.gemfile
 - gemfiles/sinatra_2.0.gemfile
 - kiev.gemspec
 - lib/ext/rack/common_logger.rb
 - lib/kiev.rb
+- lib/kiev/aws_sns.rb
+- lib/kiev/aws_sns/context_injector.rb
 - lib/kiev/base.rb
 - lib/kiev/base52.rb
 - lib/kiev/config.rb
@@ -159,6 +163,10 @@ files:
 - lib/kiev/her_ext/client_request_id.rb
 - lib/kiev/httparty.rb
 - lib/kiev/json.rb
+- lib/kiev/kafka.rb
+- lib/kiev/kafka/context_extractor.rb
+- lib/kiev/kafka/context_injector.rb
+- lib/kiev/kafka/message_context.rb
 - lib/kiev/logger.rb
 - lib/kiev/param_filter.rb
 - lib/kiev/que/job.rb
@@ -199,7 +207,7 @@ homepage: https://github.com/blacklane/kiev
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -207,15 +215,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
 specification_version: 4
 summary: Distributed logging to JSON integrated with various Ruby frameworks and tools
 test_files: []

data/.travis.yml

@@ -1,28 +0,0 @@
-sudo: false
-branches:
-  only:
-    - master
-cache:
-  - bundler
-language: ruby
-rvm:
-  - 2.3.3
-  - 2.2.3
-addons:
-  postgresql: "9.4"
-services:
-  - postgresql
-  - redis-server
-before_script:
-  - psql -c 'create database que_test;' -U postgres
-gemfile:
-  - gemfiles/que_0.12.2.gemfile
-  - gemfiles/que_0.12.3.gemfile
-  - gemfiles/rails_4.1.gemfile
-  - gemfiles/rails_4.2.gemfile
-  - gemfiles/shoryuken_3.1.gemfile
-  - gemfiles/sidekiq_4.2.gemfile
-  - gemfiles/sinatra_1.4.gemfile
-  - gemfiles/sinatra_2.0.gemfile
-matrix:
-  fast_finish: true