kiev 4.1.0 → 4.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 48e8f7e0dc2f9c76295426b0d18a44853797d1aa3e9a776ed319853cd5eca0af
-  data.tar.gz: edd058019e6602d95412a3f00c8fa48ce4147cab70cee51223f25ce575fe61c6
+  metadata.gz: 227c980f7b73aba9b41e37bba7eed1becbaab0508c4bce76cbcb1eb7a533a702
+  data.tar.gz: 5be6d241640e6a8d7384927be193c1a0b79dbd5bd781c16281703407464039b5
 SHA512:
-  metadata.gz: 50a73b6d2af85c8807c7fe41008c7ae3c260d0883c5bea0024c81d7bbb839563a1392a56f6967e784678d5c2491e33631bded930916d320792160d47db3432c1
-  data.tar.gz: 1300a13faebba70414cced00ba99bee90a686487b1b4f950800e60eb10b0a11b6f9f969a3591e718a5b164a79011fdb39c97ddf56d2923152614448f7b243d65
+  metadata.gz: a3760058c93f466e9ddf1e2040581dded64adf75d317265ae6e47f8ee913dd4d8b079c204ef99be2c5e1da2b40729944467a12e65f240daee9083298ae4d1015
+  data.tar.gz: 8c4dcdf1fcbb3b8a2c5e255a3fb3505cb4558076970b6dbb8a5e0c2ca79f59d816699fe1a8eaead9fa0445501f54d3df296aaac8f78d2ec21b40a02508899d25

data/.github/workflows/push.yml

@@ -0,0 +1,80 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Main CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: ['2.5.1', '2.7.2', '2.6.5']
+        gemfile:
+          - gemfiles/ruby_kafka.gemfile
+          - gemfiles/que_0.12.2.gemfile
+          - gemfiles/que_0.12.3.gemfile
+          - gemfiles/rails_5.2.gemfile
+          - gemfiles/shoryuken_4.0.gemfile
+          - gemfiles/sidekiq_4.2.gemfile
+          - gemfiles/sinatra_1.4.gemfile
+          - gemfiles/sinatra_2.0.gemfile
+        allow_failures:
+          - false
+        include:
+          - os: ubuntu
+            ruby-version: ruby-head
+            gemfile: gemfiles/rails_5.2.gemfile
+            allow_failures: true
+    env:
+      BUNDLE_GEMFILE: "${{ matrix.gemfile }}"
+      ALLOW_FAILURES: "${{ matrix.allow_failures }}"
+      REDIS_URL: "redis://localhost:6379/4"
+      DATABASE_URL: ${{ (startsWith(matrix.gemfile,'gemfiles/que') && 'postgres://postgres:postgres@localhost:5432/que_test') || 'sqlite3:db/combustion_test.sqlite'}}
+    continue-on-error: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'debug' }}
+
+    # Service containers to run with `container-job`
+    services:
+      redis:
+        image: redis:latest
+        ports:
+          - 6379:6379
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:9.4
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: que_test
+        # Set health checks to wait until postgres has started
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+        # change this to (see https://github.com/ruby/setup-ruby#versioning):
+        # uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+      - name: Run tests
+        run: bundle exec rake || $ALLOW_FAILURES

data/.rubocop.yml

@@ -1,10 +1,17 @@
 inherit_from:
   - https://raw.githubusercontent.com/blacklane/rubocop/master/rubocop.yml
 
-Lint/HandleExceptions:
+AllCops:
+  TargetRubyVersion: 2.5
   Exclude:
-    - test/rails_integration_test.rb
-    - test/sinatra_integration_test.rb
+    - test/rails_app/**/*.rb # auto-generated
+    - spec/**/*.rb
+    - test/**/*.rb
+    - vendor/bundle/**/*
+Lint/SuppressedException:
+  Exclude:
+    - test/**/*.rb
+    - spec/**/*.rb
 Lint/RescueException:
   Exclude:
     - lib/kiev/request_body_filter/json.rb

data/.ruby-version

@@ -1 +1 @@
-2.3.3
+2.5.1

data/Gemfile

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
-eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_4.1.gemfile"))
+eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_5.2.gemfile"))
 
 gem "wwtd"

data/README.md

@@ -1,4 +1,4 @@
-# Kiev [![Build Status](https://travis-ci.org/blacklane/kiev.svg?branch=master)](https://travis-ci.org/blacklane/kiev)
+# Kiev [![Build Status](https://github.com/blacklane/kiev/workflows/Main%20CI/badge.svg?branch=master)](https://github.com/blacklane/kiev/actions?query=workflow%3A%22Main+CI%22) [![Gem Version](https://badge.fury.io/rb/kiev.svg)](https://badge.fury.io/rb/kiev)
 
 Kiev is a comprehensive logging library aimed at covering a wide range of frameworks and tools from the Ruby ecosystem:
 
@@ -154,6 +154,36 @@ Kiev::Shoryuken.enable
 
 The name of the worker class is not logged by default. Configure [`persistent_log_fields` option](#persistent_log_fields) to include `"shoryuken_class"` if you want this.
 
+### AWS SNS
+
+To enhance messages published to SNS topics you can use the ContextInjector:
+
+```ruby
+sns_message = { topic_arn: "...",  message: "{...}" }
+Kiev::Kafka.inject_context(sns_message[:message_attributes])
+
+```
+
+After this operation the message attributes will also include required context for the Kiev logger.
+
+### Kafka
+
+To enhance messages published to Kafka topics you can use the ContextInjector:
+
+```ruby
+Kiev::Kafka.inject_context(headers)
+```
+
+After this operation the headers variable will also include required context for the Kiev logger.
+
+If you have a consumed `Kafka::FetchedMessage` you can extract logger context with: 
+
+```ruby
+Kiev::Kafka.extract_context(message)
+```
+
+This will work regardless if headers are in HTTP format, e.g. `X-Tracking-Id` or plain field names: `tracking_id`. Plus the `message_key` field will contain the key of processed message. In case you want to log some more fields configure `persistent_log_fields` and `jobs_propagated_fields`.  
+
 ### Que
 
 Add the following lines to your initializer code:
@@ -219,7 +249,9 @@ For web requests the Kiev middleware will log the following information by defau
 
 * `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option.
 
-* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4).
+* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). (currently deprecated in favor of a new name: `tracking_id`)
+
+* `tracking_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). If not provided the value is seeded from deprecated `request_id`. 
 
 * `request_depth` represents the position of the current request within a chain of requests. It starts with 0.
 

data/gemfiles/que_0.12.2.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -11,4 +13,4 @@ gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/que_0.12.3.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 # need it because of bug in https://github.com/chanks/que/issues/191
@@ -12,4 +14,4 @@ gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4.1.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -10,4 +12,4 @@ gem "rspec", require: false
 gem "rspec-rails", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4.2.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -10,4 +12,4 @@ gem "rspec", require: false
 gem "rspec-rails", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "oj"
+
+gem "rails", "~> 5.2.4"
+gem "sqlite3", "1.3.13"
+
+gem "combustion"
+gem "rspec", require: false
+gem "rspec-rails", require: false
+gem "minitest-reporters", require: false
+
+gemspec path: "../"

data/gemfiles/ruby_kafka.gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "ruby-kafka", "~> 0.7.10"
+
+gem "rack-test", require: false
+gem "rspec", require: false
+gem "minitest-reporters", require: false
+
+gemspec path: "../"

data/gemfiles/{shoryuken_3.1.gemfile → shoryuken_4.0.gemfile}

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "aws-sdk", "~> 2.0"
-gem "shoryuken", "~> 3.1.0"
+gem "shoryuken", "~> 4.0"
 
 gem "rack-test", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/sidekiq_4.2.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj", "~> 2"
@@ -10,5 +12,4 @@ gem "minitest-reporters", require: false
 
 gem "her"
 
-gemspec :path => "../"
-
+gemspec path: "../"

data/gemfiles/sinatra_1.4.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -6,10 +8,10 @@ gem "xml-simple"
 
 gem "sinatra", "1.4.7"
 gem "sinatra-contrib"
-gem "rack-parser", :require => "rack/parser"
+gem "rack-parser", require: "rack/parser"
 
 gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/sinatra_2.0.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -6,10 +8,10 @@ gem "xml-simple"
 
 gem "sinatra", "2.0.0"
 gem "sinatra-contrib"
-gem "rack-parser", :require => "rack/parser"
+gem "rack-parser", require: "rack/parser"
 
 gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/kiev.gemspec

@@ -9,7 +9,9 @@ Gem::Specification.new do |spec|
   spec.licenses      = ["MIT"]
 
   spec.summary       = "Distributed logging to JSON integrated with various Ruby frameworks and tools"
-  spec.description   = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing human-readable output in development mode. It integrates nicely with Rails, Sinatra and other Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients."
+  spec.description   = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing "\
+                        "human-readable output in development mode. It integrates nicely with Rails, Sinatra and other"\
+                        " Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients."
   spec.homepage      = "https://github.com/blacklane/kiev"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -17,12 +19,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 2.0.0"
+  spec.required_ruby_version = ">= 2.5"
+  spec.add_dependency "oga", "~> 2.2"
   spec.add_dependency "rack", ">= 1", "< 3"
   spec.add_dependency "request_store", ">= 1.0", "< 1.4"
-  spec.add_dependency "oga", "~> 2.2"
   spec.add_dependency "ruby_dig", "~> 0.0.2" # to support ruby 2.2
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rubocop", "0.49.1"
+  spec.add_development_dependency "rake", "~> 0"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "rubocop", "~> 0.54"
 end

data/lib/kiev.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "kiev/base"
+require_relative "kiev/aws_sns" if defined?(AWS::SNS)
+require_relative "kiev/kafka" if defined?(Kafka)
 require_relative "kiev/rack" if defined?(Rack)
 require_relative "kiev/railtie" if defined?(Rails)
 require_relative "kiev/sidekiq" if defined?(Sidekiq)

data/lib/kiev/aws_sns.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module AwsSns
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::AwsSns::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/aws_sns/context_injector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module AwsSns
+    class ContextInjector
+      # @param [Hash] message_attributes Injects context headers
+      # @return [Hash]
+      def call(message_attributes = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          message_attributes[key] = {
+            data_type: "String",
+            string_value: value.to_s
+          }
+        end
+        message_attributes
+      end
+    end
+  end
+end

data/lib/kiev/base.rb

@@ -26,8 +26,20 @@ module Kiev
       Config.instance.logger
     end
 
+    def filtered_params
+      Config.instance.filtered_params
+    end
+
+    def ignored_params
+      Config.instance.ignored_params
+    end
+
     def event(event_name, data = EMPTY_OBJ)
-      logger.log(::Logger::Severity::INFO, data, event_name)
+      logger.log(
+        ::Logger::Severity::INFO,
+        ParamFilter.filter(data, filtered_params, ignored_params),
+        event_name
+      )
     end
 
     def []=(name, value)
@@ -47,7 +59,9 @@ module Kiev
     end
 
     def request_id
-      RequestStore.store[:request_id]
+      RequestStore.store[:tracking_id]
     end
+
+    alias_method :tracking_id, :request_id
   end
 end

data/lib/kiev/base52.rb

@@ -3,7 +3,7 @@
 module Kiev
   module Base52
     KEYS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".freeze
-    BASE = KEYS.length.freeze
+    BASE = KEYS.length
 
     def self.encode(num)
       return KEYS[0] if num == 0

data/lib/kiev/config.rb

@@ -63,6 +63,7 @@ module Kiev
       ) << :tempfile).freeze
 
     DEFAULT_HTTP_PROPAGATED_FIELDS = {
+      tracking_id: "X-Tracking-Id",
       request_id: "X-Request-Id",
       request_depth: "X-Request-Depth",
       tree_path: "X-Tree-Path"

data/lib/kiev/context_reader.rb

@@ -6,6 +6,7 @@ module Kiev
   # change field lookup.
   class ContextReader
     REQUEST_ID = "request_id"
+    TRACKING_ID = "tracking_id"
     REQUEST_DEPTH = "request_depth"
     TREE_PATH = "tree_path"
 
@@ -17,12 +18,14 @@ module Kiev
       subject[key]
     end
 
-    def request_id
-      self[REQUEST_ID] || SecureRandom.uuid
+    def tracking_id
+      self[TRACKING_ID] || self[REQUEST_ID] || SecureRandom.uuid
     end
 
+    alias_method :request_id, :tracking_id
+
     def tree_root?
-      !self[REQUEST_ID]
+      !self[TRACKING_ID] && !self[REQUEST_ID]
     end
 
     def request_depth

data/lib/kiev/kafka.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module Kafka
+    require_relative "kafka/context_extractor"
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Kafka::FetchedMessage] message
+      def extract_context(message)
+        Kiev::Kafka::ContextExtractor.new.call(message)
+      end
+
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::Kafka::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/kafka/context_extractor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "message_context"
+require "kiev/request_id"
+require "kiev/context_reader"
+
+module Kiev
+  module Kafka
+    class ContextExtractor
+      include Kiev::RequestId::Mixin
+
+      # @param [Kafka::FetchedMessage] message
+      def call(message)
+        context = Kiev::Kafka::MessageContext.new(message)
+        context_reader = Kiev::ContextReader.new(context)
+        wrap_request_id(context_reader) {}
+
+        Kiev[:message_key] = message.key
+
+        Config.instance.jobs_propagated_fields.each do |key|
+          Kiev[key] = context_reader[key]
+        end
+      end
+    end
+  end
+end

data/lib/kiev/kafka/context_injector.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module Kafka
+    class ContextInjector
+      # @param [Hash] headers Injects context headers
+      # @return [Hash]
+      def call(headers = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          field_key = Kiev::Config::DEFAULT_HTTP_PROPAGATED_FIELDS.fetch(key.to_sym, key)
+          headers[field_key] = value
+        end
+        headers
+      end
+    end
+  end
+end

data/lib/kiev/kafka/message_context.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Kafka
+    class MessageContext
+      # @param [Kafka::FetchedMessage] message
+      def initialize(message)
+        @headers = message.headers
+      end
+
+      def value(field)
+        headers[header_key(field)] || headers[field.to_s]
+      end
+
+      alias_method :[], :value
+
+      private
+
+      attr_reader :headers
+
+      # @param [String] field
+      def header_key(field)
+        "x_#{field}".gsub("_", " ").split.map(&:capitalize).join("-")
+      end
+    end
+  end
+end

data/lib/kiev/logger.rb

@@ -13,6 +13,8 @@ module Kiev
     def_delegators(*([:@logger] + ::Logger.instance_methods(false)))
 
     DEFAULT_EVENT_NAME = "log"
+    LOG_ERROR = "ERROR"
+    ERROR_STATUS = 500
 
     FORMATTER = proc do |severity, time, event_name, data|
       entry =
@@ -21,6 +23,7 @@ module Kiev
           event: event_name || DEFAULT_EVENT_NAME,
           level: severity,
           timestamp: time.utc,
+          tracking_id: RequestStore.store[:tracking_id],
           request_id: RequestStore.store[:request_id],
           request_depth: RequestStore.store[:request_depth],
           tree_path: RequestStore.store[:tree_path]
@@ -54,8 +57,11 @@ module Kiev
         entry.merge!(data)
       elsif !data.nil?
         entry[:message] = data.to_s
+        entry[:status] = ERROR_STATUS if data.to_s.downcase.include?(LOG_ERROR)
       end
 
+      entry[:level] = LOG_ERROR if entry[:status].to_i.between?(400, 599)
+
       # Save some disk space
       entry.reject! { |_, value| value.nil? }
 
@@ -87,10 +93,8 @@ module Kiev
         entry << "(#{duration}ms)" if duration
         entry << "\n"
 
-        meta = {
-          request_id: RequestStore.store[:request_id],
-          request_depth: RequestStore.store[:request_depth]
-        }.merge!(Hash(RequestStore.store[:payload]))
+        meta = RequestStore.store.slice(:trakcing_id, :request_id, :request_depth)
+                           .reverse_merge!(Hash(RequestStore.store[:payload]))
 
         meta.reject! { |_, value| value.nil? }
 

data/lib/kiev/param_filter.rb

@@ -1,12 +1,21 @@
 # frozen_string_literal: true
 
 module Kiev
-  module ParamFilter
+  class ParamFilter
     FILTERED = "[FILTERED]"
 
     def self.filter(params, filtered_params, ignored_params)
+      new(filtered_params, ignored_params).call(params)
+    end
+
+    def initialize(filtered_params, ignored_params)
+      @filtered_params = normalize(filtered_params)
+      @ignored_params = normalize(ignored_params)
+    end
+
+    def call(params)
       params.each_with_object({}) do |(key, value), acc|
-        next if ignored_params.include?(key)
+        next if ignored_params.include?(key.to_s)
 
         if defined?(ActionDispatch) && value.is_a?(ActionDispatch::Http::UploadedFile)
           value = {
@@ -17,14 +26,22 @@ module Kiev
         end
 
         acc[key] =
-          if filtered_params.include?(key) && !value.is_a?(Hash)
+          if filtered_params.include?(key.to_s) && !value.is_a?(Hash)
             FILTERED
           elsif value.is_a?(Hash)
-            filter(value, filtered_params, ignored_params)
+            call(value)
           else
             value
           end
       end
     end
+
+    private
+
+    attr_reader :filtered_params, :ignored_params
+
+    def normalize(params)
+      Set.new(params.map(&:to_s))
+    end
   end
 end

data/lib/kiev/que/job.rb

@@ -27,6 +27,7 @@ module Kiev
       private
 
       NEW_LINE = "\n"
+      LOG_ERROR = "ERROR"
 
       def kiev_run
         args = attrs[:args]
@@ -46,7 +47,8 @@ module Kiev
             Kiev[key] = payload[key]
           end
           request_store = Kiev::RequestStore.store
-          request_store[:request_id] = payload[:request_id]
+          request_store[:tracking_id] = payload[:tracking_id]
+          request_store[:request_id] = payload[:tracking_id] || payload[:request_id]
           request_store[:request_depth] = payload[:request_depth].to_i + 1
           request_store[:tree_path] = payload[:tree_path]
 
@@ -69,6 +71,7 @@ module Kiev
           data[:error_class] = error.class.name
           data[:error_message] = error.message[0..5000]
           data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000]
+          data[:level] = LOG_ERROR
         end
 
         Kiev.event(:job_finished, data)

data/lib/kiev/rack/request_id.rb

@@ -14,24 +14,26 @@ module Kiev
 
       def call(env)
         request_id_header_out = to_rack(:request_id)
-        request_id_header_in = to_http(:request_id)
+        tracking_id_header_out = to_rack(:tracking_id)
 
-        request_id = make_request_id(env[RAILS_REQUEST_ID] || env[request_id_header_in])
-        RequestStore.store[:request_id] = request_id
+        tracking_id = make_tracking_id(env[to_http(:tracking_id)] || env[RAILS_REQUEST_ID] || env[to_http(:request_id)])
+        RequestStore.store[:tracking_id] = tracking_id
+        RequestStore.store[:request_id] = tracking_id
         RequestStore.store[:request_depth] = request_depth(env)
         RequestStore.store[:tree_path] = tree_path(env)
 
-        @app.call(env).tap { |_status, headers, _body| headers[request_id_header_out] = request_id }
+        @app.call(env).tap do |_status, headers, _body|
+          headers[tracking_id_header_out] = tracking_id
+          headers[request_id_header_out] = tracking_id
+        end
       end
 
       private
 
-      # TODO: in Rails 5 they set `headers[X_REQUEST_ID]`, so this will not work
-      # https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/request_id.rb
-      # https://github.com/interagent/pliny/blob/master/lib/pliny/middleware/request_id.rb
       def tree_root?(env)
+        tracking_id_header_in = to_http(:tracking_id)
         request_id_header_in = to_http(:request_id)
-        !env[request_id_header_in]
+        !env[tracking_id_header_in] && !env[request_id_header_in]
       end
 
       def request_depth(env)
@@ -52,15 +54,15 @@ module Kiev
         Config.instance.all_http_propagated_fields[value]
       end
 
-      def make_request_id(request_id)
-        if request_id.nil? || request_id.empty?
-          internal_request_id
+      def make_tracking_id(tracking_id)
+        if tracking_id.nil? || tracking_id.empty?
+          internal_tracking_id
         else
-          Util.sanitize(request_id)
+          Util.sanitize(tracking_id)
         end
       end
 
-      def internal_request_id
+      def internal_tracking_id
         SecureRandom.uuid
       end
     end

data/lib/kiev/rack/request_logger.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
+require "zlib"
+
 module Kiev
   module Rack
     class RequestLogger
       ERROR_STATUS = 500
       ERROR_HEADERS = [].freeze
       ERROR_BODY = [""].freeze
+      LOG_ERROR = "ERROR"
 
       def initialize(app)
         @app = app
@@ -82,8 +85,6 @@ module Kiev
             request.params
           end
 
-        params = ParamFilter.filter(params, config.filtered_params, config.ignored_params)
-
         data = {
           host: request.host, # env["HTTP_HOST"] || env["HTTPS_HOST"],
           params: params.empty? ? nil : params, # env[Rack::QUERY_STRING],
@@ -94,6 +95,8 @@ module Kiev
           route: extract_route(env)
         }
 
+        data[:level] = LOG_ERROR if data[:status].to_i.between?(400, 599)
+
         if env[HTTP_X_REQUEST_START]
           data[:request_latency] = ((began_at - env[HTTP_X_REQUEST_START].to_f) * 1000).round(3)
         end
@@ -115,6 +118,15 @@ module Kiev
             full_body << str
           end
           data[:body] = full_body.join
+          if data[:body] && !data[:body].empty? && response.headers["Content-Encoding"] == "gzip"
+            begin
+              sio = StringIO.new(data[:body])
+              gz = Zlib::GzipReader.new(sio)
+              data[:body] = gz.read
+            rescue Zlib::GzipFile::Error => e
+              data[:gzip_parse_error] = e.message
+            end
+          end
         end
 
         should_log_errors = config.log_request_error_condition.call(request, response)
@@ -122,8 +134,8 @@ module Kiev
           data[:error_class] = exception.class.name
           data[:error_message] = exception.message[0..5000]
           data[:error_backtrace] = Array(exception.backtrace).join(NEW_LINE)[0..5000]
+          data[:level] = LOG_ERROR
         end
-
         data
       end
 

data/lib/kiev/request_id.rb

@@ -7,7 +7,8 @@ module Kiev
 
       def wrap_request_id(context_reader, &_block)
         request_store = Kiev::RequestStore.store
-        request_store[:request_id] = context_reader.request_id
+        request_store[:tracking_id] = context_reader.tracking_id || context_reader.request_id
+        request_store[:request_id] = request_store[:tracking_id]
         request_store[:request_depth] = context_reader.request_depth
         request_store[:tree_path] = context_reader.tree_path
         yield

data/lib/kiev/request_logger.rb

@@ -4,6 +4,7 @@ module Kiev
   module RequestLogger
     module Mixin
       NEW_LINE = "\n"
+      LOG_ERROR = "ERROR"
 
       def wrap_request_logger(event, **data, &_block)
         began_at = Time.now
@@ -11,8 +12,8 @@ module Kiev
 
         begin
           return_value = yield
-        rescue StandardError => exception
-          error = exception
+        rescue StandardError => e
+          error = e
         end
 
         begin
@@ -21,11 +22,13 @@ module Kiev
             data[:error_class] = error.class.name
             data[:error_message] = error.message[0..5000]
             data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000]
+            data[:level] = LOG_ERROR
           end
 
           Kiev.event(event, data)
         ensure
           raise error if error
+
           return_value
         end
       end

data/lib/kiev/shoryuken/context_reader.rb

@@ -12,8 +12,10 @@ module Kiev
 
       def [](key)
         return unless @message_attributes.key?(key)
+
         attribute_value = @message_attributes[key]
         return unless attribute_value.data_type == "String"
+
         attribute_value.string_value
       end
     end

data/lib/kiev/shoryuken/middleware/message_tracer.rb

@@ -1,17 +1,14 @@
 # frozen_string_literal: true
 
+require "kiev/aws_sns/context_injector"
+
 module Kiev
   module Shoryuken
     module Middleware
       class MessageTracer
         def call(options)
-          attrbutes = options[:message_attributes] ||= {}
-          SubrequestHelper.payload.each do |key, value|
-            attrbutes[key] = {
-              data_type: "String",
-              string_value: value.to_s
-            }
-          end
+          options[:message_attributes] ||= {}
+          Kiev::AwsSns::ContextInjector.new.call(options[:message_attributes])
           yield
         end
       end

data/lib/kiev/test.rb

@@ -25,8 +25,9 @@ module Kiev
 
       def entries
         return @logs unless @logs.empty?
+
         @logs = raw_logs.each_line.map(&::JSON.method(:parse))
-      rescue
+      rescue StandardError
         puts raw_logs
         raise
       end

data/lib/kiev/util.rb

@@ -4,6 +4,7 @@ module Kiev
   module Util
     def self.sanitize(value)
       return unless value
+
       value.gsub(/[^\w\-]/, "")[0...255]
     end
 

data/lib/kiev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kiev
-  VERSION = "4.1.0"
+  VERSION = "4.6.0"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: kiev
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.6.0
 platform: ruby
 authors:
 - Blacklane
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-12-05 00:00:00.000000000 Z
+date: 2021-06-23 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -50,20 +64,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.4'
-- !ruby/object:Gem::Dependency
-  name: oga
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: ruby_dig
   requirement: !ruby/object:Gem::Requirement
@@ -82,58 +82,58 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.49.1
+        version: '0.54'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.49.1
+        version: '0.54'
 description: Kiev is a logging tool aimed at distributed environments. It logs to
   JSON, while providing human-readable output in development mode. It integrates nicely
   with Rails, Sinatra and other Rack-based frameworks, Sidekiq, Que, HTTParty, Her
   and other Faraday-based HTTP clients.
-email: 
+email:
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/push.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
-- ".travis.yml"
 - Gemfile
 - LICENSE.md
 - README.md
@@ -144,13 +144,17 @@ files:
 - gemfiles/que_0.12.3.gemfile
 - gemfiles/rails_4.1.gemfile
 - gemfiles/rails_4.2.gemfile
-- gemfiles/shoryuken_3.1.gemfile
+- gemfiles/rails_5.2.gemfile
+- gemfiles/ruby_kafka.gemfile
+- gemfiles/shoryuken_4.0.gemfile
 - gemfiles/sidekiq_4.2.gemfile
 - gemfiles/sinatra_1.4.gemfile
 - gemfiles/sinatra_2.0.gemfile
 - kiev.gemspec
 - lib/ext/rack/common_logger.rb
 - lib/kiev.rb
+- lib/kiev/aws_sns.rb
+- lib/kiev/aws_sns/context_injector.rb
 - lib/kiev/base.rb
 - lib/kiev/base52.rb
 - lib/kiev/config.rb
@@ -159,6 +163,10 @@ files:
 - lib/kiev/her_ext/client_request_id.rb
 - lib/kiev/httparty.rb
 - lib/kiev/json.rb
+- lib/kiev/kafka.rb
+- lib/kiev/kafka/context_extractor.rb
+- lib/kiev/kafka/context_injector.rb
+- lib/kiev/kafka/message_context.rb
 - lib/kiev/logger.rb
 - lib/kiev/param_filter.rb
 - lib/kiev/que/job.rb
@@ -199,7 +207,7 @@ homepage: https://github.com/blacklane/kiev
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -207,16 +215,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
 specification_version: 4
 summary: Distributed logging to JSON integrated with various Ruby frameworks and tools
 test_files: []

data/.travis.yml

@@ -1,28 +0,0 @@
-sudo: false
-branches:
-  only:
-    - master
-cache:
-  - bundler
-language: ruby
-rvm:
-  - 2.3.3
-  - 2.2.3
-addons:
-  postgresql: "9.4"
-services:
-  - postgresql
-  - redis-server
-before_script:
-  - psql -c 'create database que_test;' -U postgres
-gemfile:
-  - gemfiles/que_0.12.2.gemfile
-  - gemfiles/que_0.12.3.gemfile
-  - gemfiles/rails_4.1.gemfile
-  - gemfiles/rails_4.2.gemfile
-  - gemfiles/shoryuken_3.1.gemfile
-  - gemfiles/sidekiq_4.2.gemfile
-  - gemfiles/sinatra_1.4.gemfile
-  - gemfiles/sinatra_2.0.gemfile
-matrix:
-  fast_finish: true