kiev 4.0.0 → 4.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d931febf4f2ad051147ce2b796985482fe2510de337882f4a0ddf5405dec6dfa
-  data.tar.gz: c05aa2ff341c312313d2e10da5dbb2d552d7266827ea4f7f62529080013bc34b
+  metadata.gz: f160c0e287ac552fab279ebc1974bf2de5704af494766872e1e2abfa11b48766
+  data.tar.gz: f2601c9be33ecc63b3741e76832b2b409250a4b5e7365ede3a40cc6a59c0eb6f
 SHA512:
-  metadata.gz: 3c57d6b5a18cc053e3e6fdde63d7e9c83fc3348b050d7ffee4bfcc8dbede1ddd843974faf3256e524a0eca3ec2aa0075a25d1917becdaa263c367a1a1d1b660b
-  data.tar.gz: 9b95396a7c5ed4f1a0a90d607b9bae3eaf1254aa4dbab23e8d7da0a3b2d5e37256d84d19ec8366d317a7194b68cdae3f69dd6f46f90c2e2c04c62d3de336cd2d
+  metadata.gz: 9560e9929db4c64e968dc39d2efadb6b9a09cff5ced0a7e5ffacd823c63273bcd05be4e1918b5e4b70586c73f8520e4d62dccc9e83795908f0981d0ed14ca6e1
+  data.tar.gz: 4caa6b83331e522867ec3254ac5a3f0356c3336a452ade39e9c8df022f5409a17cc1a024c10cee80353252bec8da66b36aea5430cc4947c59acc0dab0fa1eccd

data/.github/workflows/push.yml

@@ -0,0 +1,80 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Main CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: ['2.5.1', '2.7.2', '2.6.5']
+        gemfile:
+          - gemfiles/ruby_kafka.gemfile
+          - gemfiles/que_0.12.2.gemfile
+          - gemfiles/que_0.12.3.gemfile
+          - gemfiles/rails_5.2.gemfile
+          - gemfiles/shoryuken_4.0.gemfile
+          - gemfiles/sidekiq_4.2.gemfile
+          - gemfiles/sinatra_1.4.gemfile
+          - gemfiles/sinatra_2.0.gemfile
+        allow_failures:
+          - false
+        include:
+          - os: ubuntu
+            ruby-version: ruby-head
+            gemfile: gemfiles/rails_5.2.gemfile
+            allow_failures: true
+    env:
+      BUNDLE_GEMFILE: "${{ matrix.gemfile }}"
+      ALLOW_FAILURES: "${{ matrix.allow_failures }}"
+      REDIS_URL: "redis://localhost:6379/4"
+      DATABASE_URL: ${{ (startsWith(matrix.gemfile,'gemfiles/que') && 'postgres://postgres:postgres@localhost:5432/que_test') || 'sqlite3:db/combustion_test.sqlite'}}
+    continue-on-error: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'debug' }}
+
+    # Service containers to run with `container-job`
+    services:
+      redis:
+        image: redis:latest
+        ports:
+          - 6379:6379
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:9.4
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: que_test
+        # Set health checks to wait until postgres has started
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+        # change this to (see https://github.com/ruby/setup-ruby#versioning):
+        # uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+
+      - name: Run tests
+        run: bundle exec rake || $ALLOW_FAILURES

data/.rubocop.yml

@@ -1,10 +1,17 @@
 inherit_from:
   - https://raw.githubusercontent.com/blacklane/rubocop/master/rubocop.yml
 
-Lint/HandleExceptions:
+AllCops:
+  TargetRubyVersion: 2.5
   Exclude:
-    - test/rails_integration_test.rb
-    - test/sinatra_integration_test.rb
+    - test/rails_app/**/*.rb # auto-generated
+    - spec/**/*.rb
+    - test/**/*.rb
+    - vendor/bundle/**/*
+Lint/SuppressedException:
+  Exclude:
+    - test/**/*.rb
+    - spec/**/*.rb
 Lint/RescueException:
   Exclude:
     - lib/kiev/request_body_filter/json.rb

data/.ruby-version

@@ -1 +1 @@
-2.3.3
+2.5.1

data/Gemfile

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
-eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_4.1.gemfile"))
+eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_5.2.gemfile"))
 
 gem "wwtd"

data/README.md

@@ -1,4 +1,4 @@
-# Kiev [![Build Status](https://travis-ci.org/blacklane/kiev.svg?branch=master)](https://travis-ci.org/blacklane/kiev)
+# Kiev [![Build Status](https://github.com/blacklane/kiev/workflows/Main%20CI/badge.svg?branch=master)](https://github.com/blacklane/kiev/actions?query=workflow%3A%22Main+CI%22) [![Gem Version](https://badge.fury.io/rb/kiev.svg)](https://badge.fury.io/rb/kiev)
 
 Kiev is a comprehensive logging library aimed at covering a wide range of frameworks and tools from the Ruby ecosystem:
 
@@ -107,6 +107,35 @@ end
 run(app)
 ```
 
+### Hanami
+
+Place your configuration under `config/initializers/kiev.rb`:
+
+```ruby
+require "kiev"
+
+Kiev.configure do |config|
+  config.app = :my_app
+  config.development_mode = Hanami.env?(:development)
+  config.log_path = File.join("log", "structured.log")
+end
+```
+
+Within your `MyApp::Application` file, include the `Kiev::Hanami` module, in order to register the middleware stack.
+The `include` should be added before `configure` block.
+
+```ruby
+module MyApp
+  class Application < Hanami::Application
+    include Kiev::Hanami
+
+    configure do
+      # ...
+    end
+  end
+end
+```
+
 ### Sidekiq
 
 Add the following lines to your initializer code:
@@ -125,6 +154,36 @@ Kiev::Shoryuken.enable
 
 The name of the worker class is not logged by default. Configure [`persistent_log_fields` option](#persistent_log_fields) to include `"shoryuken_class"` if you want this.
 
+### AWS SNS
+
+To enhance messages published to SNS topics you can use the ContextInjector:
+
+```ruby
+sns_message = { topic_arn: "...",  message: "{...}" }
+Kiev::Kafka.inject_context(sns_message[:message_attributes])
+
+```
+
+After this operation the message attributes will also include required context for the Kiev logger.
+
+### Kafka
+
+To enhance messages published to Kafka topics you can use the ContextInjector:
+
+```ruby
+Kiev::Kafka.inject_context(headers)
+```
+
+After this operation the headers variable will also include required context for the Kiev logger.
+
+If you have a consumed `Kafka::FetchedMessage` you can extract logger context with: 
+
+```ruby
+Kiev::Kafka.extract_context(message)
+```
+
+This will work regardless if headers are in HTTP format, e.g. `X-Tracking-Id` or plain field names: `tracking_id`. Plus the `message_key` field will contain the key of processed message. In case you want to log some more fields configure `persistent_log_fields` and `jobs_propagated_fields`.  
+
 ### Que
 
 Add the following lines to your initializer code:
@@ -190,7 +249,9 @@ For web requests the Kiev middleware will log the following information by defau
 
 * `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option.
 
-* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4).
+* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). (currently deprecated in favor of a new name: `tracking_id`)
+
+* `tracking_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). If not provided the value is seeded from deprecated `request_id`. 
 
 * `request_depth` represents the position of the current request within a chain of requests. It starts with 0.
 

data/gemfiles/que_0.12.2.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -11,4 +13,4 @@ gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/que_0.12.3.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 # need it because of bug in https://github.com/chanks/que/issues/191
@@ -12,4 +14,4 @@ gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4.1.gemfile

@@ -1,13 +1,15 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
 
 gem "rails", "4.1.16"
-gem "sqlite3"
+gem "sqlite3", "1.3.13"
 
 gem "combustion"
 gem "rspec", require: false
 gem "rspec-rails", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_4.2.gemfile

@@ -1,13 +1,15 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
 
 gem "rails", "4.2.7"
-gem "sqlite3"
+gem "sqlite3", "1.3.13"
 
 gem "combustion"
 gem "rspec", require: false
 gem "rspec-rails", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "oj"
+
+gem "rails", "~> 5.2.4"
+gem "sqlite3", "1.3.13"
+
+gem "combustion"
+gem "rspec", require: false
+gem "rspec-rails", require: false
+gem "minitest-reporters", require: false
+
+gemspec path: "../"

data/gemfiles/ruby_kafka.gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "ruby-kafka", "~> 0.7.10"
+
+gem "rack-test", require: false
+gem "rspec", require: false
+gem "minitest-reporters", require: false
+
+gemspec path: "../"

data/gemfiles/{shoryuken_3.1.gemfile → shoryuken_4.0.gemfile}

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "aws-sdk", "~> 2.0"
-gem "shoryuken", "~> 3.1.0"
+gem "shoryuken", "~> 4.0"
 
 gem "rack-test", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/sidekiq_4.2.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj", "~> 2"
@@ -10,5 +12,4 @@ gem "minitest-reporters", require: false
 
 gem "her"
 
-gemspec :path => "../"
-
+gemspec path: "../"

data/gemfiles/sinatra_1.4.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -6,10 +8,10 @@ gem "xml-simple"
 
 gem "sinatra", "1.4.7"
 gem "sinatra-contrib"
-gem "rack-parser", :require => "rack/parser"
+gem "rack-parser", require: "rack/parser"
 
 gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/gemfiles/sinatra_2.0.gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
 gem "oj"
@@ -6,10 +8,10 @@ gem "xml-simple"
 
 gem "sinatra", "2.0.0"
 gem "sinatra-contrib"
-gem "rack-parser", :require => "rack/parser"
+gem "rack-parser", require: "rack/parser"
 
 gem "rack-test", require: false
 gem "rspec", require: false
 gem "minitest-reporters", require: false
 
-gemspec :path => "../"
+gemspec path: "../"

data/kiev.gemspec

@@ -9,7 +9,9 @@ Gem::Specification.new do |spec|
   spec.licenses      = ["MIT"]
 
   spec.summary       = "Distributed logging to JSON integrated with various Ruby frameworks and tools"
-  spec.description   = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing human-readable output in development mode. It integrates nicely with Rails, Sinatra and other Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients."
+  spec.description   = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing "\
+                        "human-readable output in development mode. It integrates nicely with Rails, Sinatra and other"\
+                        " Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients."
   spec.homepage      = "https://github.com/blacklane/kiev"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -17,12 +19,12 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.required_ruby_version = ">= 2.0.0"
+  spec.required_ruby_version = ">= 2.5"
+  spec.add_dependency "oga", "~> 2.2"
   spec.add_dependency "rack", ">= 1", "< 3"
   spec.add_dependency "request_store", ">= 1.0", "< 1.4"
-  spec.add_dependency "oga", "~> 2.2"
   spec.add_dependency "ruby_dig", "~> 0.0.2" # to support ruby 2.2
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rubocop", "0.49.1"
+  spec.add_development_dependency "rake", "~> 0"
+  spec.add_development_dependency "rspec", "~> 3.10"
+  spec.add_development_dependency "rubocop", "~> 0.54"
 end

data/lib/kiev.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "kiev/base"
+require_relative "kiev/aws_sns" if defined?(AWS::SNS)
+require_relative "kiev/kafka" if defined?(Kafka)
 require_relative "kiev/rack" if defined?(Rack)
 require_relative "kiev/railtie" if defined?(Rails)
 require_relative "kiev/sidekiq" if defined?(Sidekiq)

data/lib/kiev/aws_sns.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module AwsSns
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::AwsSns::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/aws_sns/context_injector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module AwsSns
+    class ContextInjector
+      # @param [Hash] message_attributes Injects context headers
+      # @return [Hash]
+      def call(message_attributes = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          message_attributes[key] = {
+            data_type: "String",
+            string_value: value.to_s
+          }
+        end
+        message_attributes
+      end
+    end
+  end
+end

data/lib/kiev/base.rb

@@ -12,6 +12,7 @@ require_relative "version"
 require_relative "config"
 require_relative "util"
 require_relative "subrequest_helper"
+require_relative "hanami"
 
 module Kiev
   class << self
@@ -46,7 +47,9 @@ module Kiev
     end
 
     def request_id
-      RequestStore.store[:request_id]
+      RequestStore.store[:tracking_id]
     end
+
+    alias_method :tracking_id, :request_id
   end
 end

data/lib/kiev/base52.rb

@@ -3,7 +3,7 @@
 module Kiev
   module Base52
     KEYS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".freeze
-    BASE = KEYS.length.freeze
+    BASE = KEYS.length
 
     def self.encode(num)
       return KEYS[0] if num == 0

data/lib/kiev/config.rb

@@ -63,6 +63,7 @@ module Kiev
       ) << :tempfile).freeze
 
     DEFAULT_HTTP_PROPAGATED_FIELDS = {
+      tracking_id: "X-Tracking-Id",
       request_id: "X-Request-Id",
       request_depth: "X-Request-Depth",
       tree_path: "X-Tree-Path"

data/lib/kiev/context_reader.rb

@@ -6,6 +6,7 @@ module Kiev
   # change field lookup.
   class ContextReader
     REQUEST_ID = "request_id"
+    TRACKING_ID = "tracking_id"
     REQUEST_DEPTH = "request_depth"
     TREE_PATH = "tree_path"
 
@@ -17,12 +18,14 @@ module Kiev
       subject[key]
     end
 
-    def request_id
-      self[REQUEST_ID] || SecureRandom.uuid
+    def tracking_id
+      self[TRACKING_ID] || self[REQUEST_ID] || SecureRandom.uuid
     end
 
+    alias_method :request_id, :tracking_id
+
     def tree_root?
-      !self[REQUEST_ID]
+      !self[TRACKING_ID] && !self[REQUEST_ID]
     end
 
     def request_depth

data/lib/kiev/hanami.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "rack/request_logger"
+require_relative "rack/store_request_details"
+require_relative "rack/request_id"
+
+module Kiev
+  module Hanami
+    def self.included(base)
+      base.configure do
+        # The order is important
+        middleware.use(::RequestStore::Middleware)
+        middleware.use(Kiev::Rack::RequestLogger)
+        middleware.use(Kiev::Rack::StoreRequestDetails)
+        middleware.use(Kiev::Rack::RequestId)
+      end
+    end
+  end
+end

data/lib/kiev/kafka.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module Kafka
+    require_relative "kafka/context_extractor"
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Kafka::FetchedMessage] message
+      def extract_context(message)
+        Kiev::Kafka::ContextExtractor.new.call(message)
+      end
+
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::Kafka::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/kafka/context_extractor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "message_context"
+require "kiev/request_id"
+require "kiev/context_reader"
+
+module Kiev
+  module Kafka
+    class ContextExtractor
+      include Kiev::RequestId::Mixin
+
+      # @param [Kafka::FetchedMessage] message
+      def call(message)
+        context = Kiev::Kafka::MessageContext.new(message)
+        context_reader = Kiev::ContextReader.new(context)
+        wrap_request_id(context_reader) {}
+
+        Kiev[:message_key] = message.key
+
+        Config.instance.jobs_propagated_fields.each do |key|
+          Kiev[key] = context_reader[key]
+        end
+      end
+    end
+  end
+end

data/lib/kiev/kafka/context_injector.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module Kafka
+    class ContextInjector
+      # @param [Hash] headers Injects context headers
+      # @return [Hash]
+      def call(headers = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          field_key = Kiev::Config::DEFAULT_HTTP_PROPAGATED_FIELDS.fetch(key.to_sym, key)
+          headers[field_key] = value
+        end
+        headers
+      end
+    end
+  end
+end

data/lib/kiev/kafka/message_context.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Kafka
+    class MessageContext
+      # @param [Kafka::FetchedMessage] message
+      def initialize(message)
+        @headers = message.headers
+      end
+
+      def value(field)
+        headers[header_key(field)] || headers[field.to_s]
+      end
+
+      alias_method :[], :value
+
+      private
+
+      attr_reader :headers
+
+      # @param [String] field
+      def header_key(field)
+        "x_#{field}".gsub("_", " ").split.map(&:capitalize).join("-")
+      end
+    end
+  end
+end

data/lib/kiev/logger.rb

@@ -13,6 +13,8 @@ module Kiev
     def_delegators(*([:@logger] + ::Logger.instance_methods(false)))
 
     DEFAULT_EVENT_NAME = "log"
+    LOG_ERROR = "ERROR"
+    ERROR_STATUS = 500
 
     FORMATTER = proc do |severity, time, event_name, data|
       entry =
@@ -21,6 +23,7 @@ module Kiev
           event: event_name || DEFAULT_EVENT_NAME,
           level: severity,
           timestamp: time.utc,
+          tracking_id: RequestStore.store[:tracking_id],
           request_id: RequestStore.store[:request_id],
           request_depth: RequestStore.store[:request_depth],
           tree_path: RequestStore.store[:tree_path]
@@ -54,8 +57,11 @@ module Kiev
         entry.merge!(data)
       elsif !data.nil?
         entry[:message] = data.to_s
+        entry[:status] = ERROR_STATUS if data.to_s.downcase.include?(LOG_ERROR)
       end
 
+      entry[:level] = LOG_ERROR if entry[:status].to_i.between?(400, 599)
+
       # Save some disk space
       entry.reject! { |_, value| value.nil? }
 
@@ -87,10 +93,8 @@ module Kiev
         entry << "(#{duration}ms)" if duration
         entry << "\n"
 
-        meta = {
-          request_id: RequestStore.store[:request_id],
-          request_depth: RequestStore.store[:request_depth]
-        }.merge!(Hash(RequestStore.store[:payload]))
+        meta = RequestStore.store.slice(:trakcing_id, :request_id, :request_depth)
+                           .reverse_merge!(Hash(RequestStore.store[:payload]))
 
         meta.reject! { |_, value| value.nil? }
 

data/lib/kiev/que/job.rb

@@ -27,6 +27,7 @@ module Kiev
       private
 
       NEW_LINE = "\n"
+      LOG_ERROR = "ERROR"
 
       def kiev_run
         args = attrs[:args]
@@ -46,7 +47,8 @@ module Kiev
             Kiev[key] = payload[key]
           end
           request_store = Kiev::RequestStore.store
-          request_store[:request_id] = payload[:request_id]
+          request_store[:tracking_id] = payload[:tracking_id]
+          request_store[:request_id] = payload[:tracking_id] || payload[:request_id]
           request_store[:request_depth] = payload[:request_depth].to_i + 1
           request_store[:tree_path] = payload[:tree_path]
 
@@ -69,6 +71,7 @@ module Kiev
           data[:error_class] = error.class.name
           data[:error_message] = error.message[0..5000]
           data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000]
+          data[:level] = LOG_ERROR
         end
 
         Kiev.event(:job_finished, data)

data/lib/kiev/rack/request_id.rb

@@ -14,24 +14,26 @@ module Kiev
 
       def call(env)
         request_id_header_out = to_rack(:request_id)
-        request_id_header_in = to_http(:request_id)
+        tracking_id_header_out = to_rack(:tracking_id)
 
-        request_id = make_request_id(env[RAILS_REQUEST_ID] || env[request_id_header_in])
-        RequestStore.store[:request_id] = request_id
+        tracking_id = make_tracking_id(env[to_http(:tracking_id)] || env[RAILS_REQUEST_ID] || env[to_http(:request_id)])
+        RequestStore.store[:tracking_id] = tracking_id
+        RequestStore.store[:request_id] = tracking_id
         RequestStore.store[:request_depth] = request_depth(env)
         RequestStore.store[:tree_path] = tree_path(env)
 
-        @app.call(env).tap { |_status, headers, _body| headers[request_id_header_out] = request_id }
+        @app.call(env).tap do |_status, headers, _body|
+          headers[tracking_id_header_out] = tracking_id
+          headers[request_id_header_out] = tracking_id
+        end
       end
 
       private
 
-      # TODO: in Rails 5 they set `headers[X_REQUEST_ID]`, so this will not work
-      # https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/request_id.rb
-      # https://github.com/interagent/pliny/blob/master/lib/pliny/middleware/request_id.rb
       def tree_root?(env)
+        tracking_id_header_in = to_http(:tracking_id)
         request_id_header_in = to_http(:request_id)
-        !env[request_id_header_in]
+        !env[tracking_id_header_in] && !env[request_id_header_in]
       end
 
       def request_depth(env)
@@ -52,15 +54,15 @@ module Kiev
         Config.instance.all_http_propagated_fields[value]
       end
 
-      def make_request_id(request_id)
-        if request_id.nil? || request_id.empty?
-          internal_request_id
+      def make_tracking_id(tracking_id)
+        if tracking_id.nil? || tracking_id.empty?
+          internal_tracking_id
         else
-          Util.sanitize(request_id)
+          Util.sanitize(tracking_id)
         end
       end
 
-      def internal_request_id
+      def internal_tracking_id
         SecureRandom.uuid
       end
     end

data/lib/kiev/rack/request_logger.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 
+require "zlib"
+
 module Kiev
   module Rack
     class RequestLogger
       ERROR_STATUS = 500
       ERROR_HEADERS = [].freeze
       ERROR_BODY = [""].freeze
+      LOG_ERROR = "ERROR"
 
       def initialize(app)
         @app = app
@@ -94,6 +97,8 @@ module Kiev
           route: extract_route(env)
         }
 
+        data[:level] = LOG_ERROR if data[:status].to_i.between?(400, 599)
+
         if env[HTTP_X_REQUEST_START]
           data[:request_latency] = ((began_at - env[HTTP_X_REQUEST_START].to_f) * 1000).round(3)
         end
@@ -115,6 +120,15 @@ module Kiev
             full_body << str
           end
           data[:body] = full_body.join
+          if data[:body] && !data[:body].empty? && response.headers["Content-Encoding"] == "gzip"
+            begin
+              sio = StringIO.new(data[:body])
+              gz = Zlib::GzipReader.new(sio)
+              data[:body] = gz.read
+            rescue Zlib::GzipFile::Error => e
+              data[:gzip_parse_error] = e.message
+            end
+          end
         end
 
         should_log_errors = config.log_request_error_condition.call(request, response)
@@ -122,8 +136,8 @@ module Kiev
           data[:error_class] = exception.class.name
           data[:error_message] = exception.message[0..5000]
           data[:error_backtrace] = Array(exception.backtrace).join(NEW_LINE)[0..5000]
+          data[:level] = LOG_ERROR
         end
-
         data
       end
 

data/lib/kiev/request_id.rb

@@ -7,7 +7,8 @@ module Kiev
 
       def wrap_request_id(context_reader, &_block)
         request_store = Kiev::RequestStore.store
-        request_store[:request_id] = context_reader.request_id
+        request_store[:tracking_id] = context_reader.tracking_id || context_reader.request_id
+        request_store[:request_id] = request_store[:tracking_id]
         request_store[:request_depth] = context_reader.request_depth
         request_store[:tree_path] = context_reader.tree_path
         yield

data/lib/kiev/request_logger.rb

@@ -4,6 +4,7 @@ module Kiev
   module RequestLogger
     module Mixin
       NEW_LINE = "\n"
+      LOG_ERROR = "ERROR"
 
       def wrap_request_logger(event, **data, &_block)
         began_at = Time.now
@@ -11,8 +12,8 @@ module Kiev
 
         begin
           return_value = yield
-        rescue StandardError => exception
-          error = exception
+        rescue StandardError => e
+          error = e
         end
 
         begin
@@ -21,11 +22,13 @@ module Kiev
             data[:error_class] = error.class.name
             data[:error_message] = error.message[0..5000]
             data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000]
+            data[:level] = LOG_ERROR
           end
 
           Kiev.event(event, data)
         ensure
           raise error if error
+
           return_value
         end
       end

data/lib/kiev/shoryuken/context_reader.rb

@@ -12,8 +12,10 @@ module Kiev
 
       def [](key)
         return unless @message_attributes.key?(key)
+
         attribute_value = @message_attributes[key]
         return unless attribute_value.data_type == "String"
+
         attribute_value.string_value
       end
     end

data/lib/kiev/shoryuken/middleware/message_tracer.rb

@@ -1,17 +1,14 @@
 # frozen_string_literal: true
 
+require "kiev/aws_sns/context_injector"
+
 module Kiev
   module Shoryuken
     module Middleware
       class MessageTracer
         def call(options)
-          attrbutes = options[:message_attributes] ||= {}
-          SubrequestHelper.payload.each do |key, value|
-            attrbutes[key] = {
-              data_type: "String",
-              string_value: value.to_s
-            }
-          end
+          options[:message_attributes] ||= {}
+          Kiev::AwsSns::ContextInjector.new.call(options[:message_attributes])
           yield
         end
       end

data/lib/kiev/test.rb

@@ -25,8 +25,9 @@ module Kiev
 
       def entries
         return @logs unless @logs.empty?
+
         @logs = raw_logs.each_line.map(&::JSON.method(:parse))
-      rescue
+      rescue StandardError
         puts raw_logs
         raise
       end

data/lib/kiev/util.rb

@@ -4,6 +4,7 @@ module Kiev
   module Util
     def self.sanitize(value)
       return unless value
+
       value.gsub(/[^\w\-]/, "")[0...255]
     end
 

data/lib/kiev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kiev
-  VERSION = "4.0.0"
+  VERSION = "4.5.0"
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: kiev
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.5.0
 platform: ruby
 authors:
 - Blacklane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-19 00:00:00.000000000 Z
+date: 2021-02-08 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -50,20 +64,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '1.4'
-- !ruby/object:Gem::Dependency
-  name: oga
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: ruby_dig
   requirement: !ruby/object:Gem::Requirement
@@ -82,44 +82,44 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.49.1
+        version: '0.54'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.49.1
+        version: '0.54'
 description: Kiev is a logging tool aimed at distributed environments. It logs to
   JSON, while providing human-readable output in development mode. It integrates nicely
   with Rails, Sinatra and other Rack-based frameworks, Sidekiq, Que, HTTParty, Her
@@ -129,11 +129,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/push.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
-- ".travis.yml"
 - Gemfile
 - LICENSE.md
 - README.md
@@ -144,20 +144,29 @@ files:
 - gemfiles/que_0.12.3.gemfile
 - gemfiles/rails_4.1.gemfile
 - gemfiles/rails_4.2.gemfile
-- gemfiles/shoryuken_3.1.gemfile
+- gemfiles/rails_5.2.gemfile
+- gemfiles/ruby_kafka.gemfile
+- gemfiles/shoryuken_4.0.gemfile
 - gemfiles/sidekiq_4.2.gemfile
 - gemfiles/sinatra_1.4.gemfile
 - gemfiles/sinatra_2.0.gemfile
 - kiev.gemspec
 - lib/ext/rack/common_logger.rb
 - lib/kiev.rb
+- lib/kiev/aws_sns.rb
+- lib/kiev/aws_sns/context_injector.rb
 - lib/kiev/base.rb
 - lib/kiev/base52.rb
 - lib/kiev/config.rb
 - lib/kiev/context_reader.rb
+- lib/kiev/hanami.rb
 - lib/kiev/her_ext/client_request_id.rb
 - lib/kiev/httparty.rb
 - lib/kiev/json.rb
+- lib/kiev/kafka.rb
+- lib/kiev/kafka/context_extractor.rb
+- lib/kiev/kafka/context_injector.rb
+- lib/kiev/kafka/message_context.rb
 - lib/kiev/logger.rb
 - lib/kiev/param_filter.rb
 - lib/kiev/que/job.rb
@@ -206,7 +215,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -214,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Distributed logging to JSON integrated with various Ruby frameworks and tools

data/.travis.yml

@@ -1,28 +0,0 @@
-sudo: false
-branches:
-  only:
-    - master
-cache:
-  - bundler
-language: ruby
-rvm:
-  - 2.3.3
-  - 2.2.3
-addons:
-  postgresql: "9.4"
-services:
-  - postgresql
-  - redis-server
-before_script:
-  - psql -c 'create database que_test;' -U postgres
-gemfile:
-  - gemfiles/que_0.12.2.gemfile
-  - gemfiles/que_0.12.3.gemfile
-  - gemfiles/rails_4.1.gemfile
-  - gemfiles/rails_4.2.gemfile
-  - gemfiles/shoryuken_3.1.gemfile
-  - gemfiles/sidekiq_4.2.gemfile
-  - gemfiles/sinatra_1.4.gemfile
-  - gemfiles/sinatra_2.0.gemfile
-matrix:
-  fast_finish: true