kiev 2.7.3 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4993218a36a9d91204d3af2a87327a3394494e3f
-  data.tar.gz: f446c0d1c31a6aacf5d12d47488cafb19756dabd
+  metadata.gz: c95641b776dbecacfb3c20627bdca29ed253a5a5
+  data.tar.gz: 04e91ef07a297c90b94011b115e429a188b2df5e
 SHA512:
-  metadata.gz: 66768446011943507c8741451239becbbaa9397b1d4cf96214f3bc0bc31e524d407821d55566ea998953092ae5bcf869c8e3a74622ad06f9a0cca446f1fcf0d2
-  data.tar.gz: d11e2427f18cb51feb4ccddcb270b840638f37531819685e9913c6a1825a0cc8dc2d5fd6e44cfd2799a27b47caaa94178edabad127b135cbc6d5c40161c288c5
+  metadata.gz: 84dd23df61159eac642eb93dfa14d97b4cfb354159dc3afa51626640ecd66106282d0414b508764ff40b2be04ef4d5e71852bd974ab7973d380b78f6bac4ff0a
+  data.tar.gz: 016ed238aa2aabf377acd7796577d95c13b10ac0d065a5ef02560fef5520ac30b5e24ebfbed3613479ade39cf88eaded5526115683e08de4ac8b69830ae37c8d

data/.rubocop.yml

@@ -9,9 +9,9 @@ Lint/RescueException:
   Exclude:
     - lib/kiev/request_body_filter/json.rb
     - lib/kiev/sidekiq/request_logger.rb
+    - lib/kiev/shoryuken/request_logger.rb
     - lib/kiev/rack/request_logger.rb
     - lib/kiev/json.rb
-    - test/sidekiq_test.rb
 Style/GlobalVars:
   Exclude:
     - test/helper.rb

data/.travis.yml

@@ -20,6 +20,7 @@ gemfile:
   - gemfiles/que_0.12.3.gemfile
   - gemfiles/rails_4.1.gemfile
   - gemfiles/rails_4.2.gemfile
+  - gemfiles/shoryuken_3.1.gemfile
   - gemfiles/sidekiq_4.2.gemfile
   - gemfiles/sinatra_1.4.gemfile
   - gemfiles/sinatra_2.0.gemfile

data/README.md

@@ -1,4 +1,4 @@
-# Kiev [![Build Status](https://travis-ci.com/blacklane/kiev.svg?token=j5tcq3Fz9ERKZ2HhzC8T&branch=master)](https://travis-ci.com/blacklane/kiev)
+# Kiev [![Build Status](https://travis-ci.org/blacklane/kiev.svg?branch=master)](https://travis-ci.org/blacklane/kiev)
 
 Kiev is a comprehensive logging library aimed at covering a wide range of frameworks and tools from the Ruby ecosystem:
 
@@ -7,6 +7,7 @@ Kiev is a comprehensive logging library aimed at covering a wide range of framew
 - Rack and other Rack-based frameworks
 - Sidekiq
 - Que
+- Shoryuken
 - Her and other Faraday-based libraries
 - HTTParty
 
@@ -114,6 +115,16 @@ Add the following lines to your initializer code:
 Kiev::Sidekiq.enable
 ```
 
+### Shoryuken
+
+Add the following lines to your initializer code:
+
+```ruby
+Kiev::Shoryuken.enable
+```
+
+The name of the worker class is not logged by default. Configure [`persistent_log_fields` option](#persistent_log_fields) to include `"shoryuken_class"` if you want this.
+
 ### Que
 
 Add the following lines to your initializer code:
@@ -177,21 +188,21 @@ For web requests the Kiev middleware will log the following information by defau
 }
 ```
 
-The `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option.
+* `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option.
 
-The `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4).
+* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4).
 
-The `request_depth` represents the position of the current request within a chain of requests. It starts with 0.
+* `request_depth` represents the position of the current request within a chain of requests. It starts with 0.
 
-The `route` attribute will be set to either the Rails route (`RootController#index`) or Sinatra route (`/`) or the path, depending on the context.
+* `route` attribute will be set to either the Rails route (`RootController#index`) or Sinatra route (`/`) or the path, depending on the context.
 
-The `request_duration` is measured in miliseconds.
+* `request_duration` is measured in miliseconds.
 
-The `body` attribute coresponds to the response body and will be logged depending on the `#log_response_body_condition` option.
+* `body` attribute coresponds to the response body and will be logged depending on the `#log_response_body_condition` option.
 
-The `tree_path` attribute can be used to follow the branching of requests within a chain of requests. It's a lexicographically sortable string.
+* `tree_path` attribute can be used to follow the branching of requests within a chain of requests. It's a lexicographically sortable string.
 
-The `tree_leaf` points out that this request is a leaf in the request chain tree structure.
+* `tree_leaf` points out that this request is a leaf in the request chain tree structure.
 
 ### Background jobs
 
@@ -422,6 +433,53 @@ When storing logs on disk, we recommend using Logrotate in truncate mode.
 
 You can use [jq](https://stedolan.github.io/jq/) to traverse JSON log files, when you're not running Kiev in *development mode*.
 
+## Suffixing `tree_path`
+
+Kiev is built upon the assumption that one request is handled once. This isn't always true.
+
+A practical example: multiple Amazon SQS queues subscribed to one Amazon SNS topic. You send one message to SNS and queues receive identical copies that are impossible to distinguish in the trace without any help from the outside.
+
+You can solve this by adding a fixed unique suffix inside each queue processor. Preferably a single character with an even number in the alphabet (B, D, F and so on), to maintain the notion of "asynchronous processing" used throughout Kiev.
+
+For a combination of SNS and [Shoryuken](https://github.com/phstc/shoryuken) (SQS consumer). Here's how you can use it:
+
+* Enable "Raw Message Delivery" in your SQS-to-SNS subscriptions
+* On sender, write `Kiev::SubrequestHelper.payload` into the message attributes
+* On each receiver, use `Kiev::Shoryuken::suffix_tree_path` with a unique tag, like this:
+
+  ```ruby
+  # Suffix a single worker class:
+  class MyWorker
+    include Shoryuken::Worker
+    Kiev::Shoryuken.suffix_tree_path(self, "B")
+    # ...
+  end
+
+  # Or use a suffix process-wide:
+  Shoryuken.configure_server do |config|
+    Kiev::Shoryuken.suffix_tree_path(config, "B")
+  end
+  ```
+
+Here's an example of the possble `tree_path` sequence you could get by configuring two consumers with suffixes `1` and `2` (note ordering by `tree_path`):
+
+| `tree_path` |                             Meaning                             |
+|-------------|-----------------------------------------------------------------|
+| `A`         | An entry point into the system, a synchronous request           |
+| `AB`        | Background job caused by `A` executed                           |
+| `ABA`       | Synchoronous request made from `AB`                             |
+| `ABD`       | _(Not logged by Kiev itself)_ `AB` sends out an SNS message     |
+| `ABD1`      | Message `ABD` handled by susbcriber `1`                         |
+| `ABD1A`     | Synchronous request sent by `1` when handling the message `ABD` |
+| `ABD1C`     | Synchronous request sent by `1` when handling the message `ABD` |
+| `ABD2`      | Message `ABD` handled by susbcriber `2`                         |
+| `ABD2A`     | Synchronous request sent by `2` when handling the message `ABD` |
+| `ABF`       | Another backgound job from `AB` executed                        |
+| `AD`        | Background job caused by `A` executed                           |
+| `AE`        | Synchronous request made from `A`                               |
+
+Without suffixing you won't see at a glance who made the request `ABDC` and you will have two entries for both `ABD` and `ABDA`. As different subscribers may log different fields, you might be able to tell apart `ABD`s. But both `ABDA`s could happen on the same node and be logged with the same lines of code.
+
 ## Alternatives
 
 ### Logging

data/gemfiles/shoryuken_3.1.gemfile

@@ -0,0 +1,9 @@
+source "https://rubygems.org"
+
+gem "aws-sdk", "~> 2.0"
+gem "shoryuken", "~> 3.1.0"
+
+gem "rack-test", require: false
+gem "minitest-reporters", require: false
+
+gemspec :path => "../"

data/lib/kiev.rb

@@ -4,6 +4,7 @@ require_relative "kiev/base"
 require_relative "kiev/rack" if defined?(Rack)
 require_relative "kiev/railtie" if defined?(Rails)
 require_relative "kiev/sidekiq" if defined?(Sidekiq)
+require_relative "kiev/shoryuken" if defined?(Shoryuken)
 require_relative "kiev/her_ext/client_request_id" if defined?(Faraday)
 require_relative "kiev/httparty" if defined?(HTTParty)
 require_relative "kiev/que/job" if defined?(Que::Job)

data/lib/kiev/base.rb

@@ -3,6 +3,7 @@
 require "request_store"
 require "ruby_dig"
 require_relative "request_store"
+require_relative "request_logger"
 require_relative "logger"
 require_relative "param_filter"
 require_relative "request_body_filter"

data/lib/kiev/context_reader.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Kiev
+  # Abstracts common details about reading tracing context
+  # into Kiev's request store. Subclass and override #[] to
+  # change field lookup.
+  class ContextReader
+    REQUEST_ID = "request_id"
+    REQUEST_DEPTH = "request_depth"
+    TREE_PATH = "tree_path"
+
+    def initialize(subject)
+      @subject = subject
+    end
+
+    def [](key)
+      subject[key]
+    end
+
+    def request_id
+      self[REQUEST_ID] || SecureRandom.uuid
+    end
+
+    def tree_root?
+      !self[REQUEST_ID]
+    end
+
+    def request_depth
+      tree_root? ? 0 : (self[REQUEST_DEPTH].to_i + 1)
+    end
+
+    def tree_path
+      if tree_root?
+        SubrequestHelper.root_path(synchronous: false)
+      else
+        self[TREE_PATH]
+      end
+    end
+
+    private
+
+    attr_reader :subject
+  end
+end

data/lib/kiev/request_id.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Kiev
+  module RequestId
+    module Mixin
+      NEW_LINE = "\n"
+
+      def wrap_request_id(context_reader, &_block)
+        request_store = Kiev::RequestStore.store
+        request_store[:request_id] = context_reader.request_id
+        request_store[:request_depth] = context_reader.request_depth
+        request_store[:tree_path] = context_reader.tree_path
+        yield
+      end
+    end
+  end
+end

data/lib/kiev/request_logger.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Kiev
+  module RequestLogger
+    module Mixin
+      NEW_LINE = "\n"
+
+      def wrap_request_logger(event, **data, &_block)
+        began_at = Time.now
+        error = nil
+
+        begin
+          return_value = yield
+        rescue StandardError => exception
+          error = exception
+        end
+
+        begin
+          data[:request_duration] = ((Time.now - began_at) * 1000).round(3)
+          if error
+            data[:error_class] = error.class.name
+            data[:error_message] = error.message[0..5000]
+            data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000]
+          end
+
+          Kiev.event(event, data)
+        ensure
+          raise error if error
+          return_value
+        end
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module Shoryuken
+    require_relative "shoryuken/middleware"
+
+    class << self
+      def enable(base = nil)
+        base ||= ::Shoryuken
+        base.configure_client do |config|
+          enable_client_middleware(config)
+        end
+        base.configure_server do |config|
+          enable_client_middleware(config)
+          enable_server_middleware(config)
+        end
+      end
+
+      def enable_server_middleware(config)
+        server_mw_enabled = false
+        config.server_middleware do |chain|
+          chain.add(Middleware::RequestStore)
+          chain.add(Middleware::RequestId)
+          chain.add(Middleware::StoreRequestDetails)
+          chain.add(Middleware::RequestLogger)
+          server_mw_enabled = true
+        end
+        server_mw_enabled # Shoryuken configuration may skip that block in non-worker setups
+      end
+
+      def enable_client_middleware(config)
+        config.client_middleware do |chain|
+          chain.add(Middleware::MessageTracer)
+        end
+      end
+
+      def suffix_tree_path(config, tag)
+        config.server_middleware do |chain|
+          chain.insert_after(Middleware::RequestId, Middleware::TreePathSuffix, tag)
+        end
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken/context_reader.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "kiev/context_reader"
+
+module Kiev
+  module Shoryuken
+    class ContextReader < Kiev::ContextReader
+      def initialize(message)
+        super
+        @message_attributes = message.message_attributes
+      end
+
+      def [](key)
+        return unless @message_attributes.key?(key)
+        attribute_value = @message_attributes[key]
+        return unless attribute_value.data_type == "String"
+        attribute_value.string_value
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken/middleware.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Client middleware
+require_relative "middleware/message_tracer"
+
+# Server middleware
+require_relative "middleware/request_id"
+require_relative "middleware/request_logger"
+require_relative "middleware/request_store"
+require_relative "middleware/store_request_details"
+require_relative "middleware/tree_path_suffix"

data/lib/kiev/shoryuken/middleware/message_tracer.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Shoryuken
+    module Middleware
+      class MessageTracer
+        def call(options)
+          attrbutes = options[:message_attributes] ||= {}
+          SubrequestHelper.payload.each do |key, value|
+            attrbutes[key] = {
+              data_type: "String",
+              string_value: value.to_s
+            }
+          end
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken/middleware/request_id.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "kiev/request_id"
+require "kiev/shoryuken/context_reader"
+
+module Kiev
+  module Shoryuken
+    module Middleware
+      class RequestId
+        include Kiev::RequestId::Mixin
+
+        def call(_worker, _queue, message, _body, &block)
+          context_reader = Kiev::Shoryuken::ContextReader.new(message)
+          wrap_request_id(context_reader, &block)
+        end
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken/middleware/request_logger.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Shoryuken
+    module Middleware
+      class RequestLogger
+        include Kiev::RequestLogger::Mixin
+
+        def call(_worker, _queue, _message, body, &block)
+          wrap_request_logger(:job_finished, body: body, &block)
+        end
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken/middleware/request_store.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Shoryuken
+    module Middleware
+      class RequestStore
+        include Kiev::RequestStore::Mixin
+
+        def call(_worker, _queue, _message, _body, &block)
+          wrap_request_store(&block)
+        end
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken/middleware/store_request_details.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "kiev/shoryuken/context_reader"
+
+module Kiev
+  module Shoryuken
+    module Middleware
+      class StoreRequestDetails
+        def call(_worker, _queue, message, _body)
+          context_reader = Kiev::Shoryuken::ContextReader.new(message)
+          Config.instance.jobs_propagated_fields.each do |key|
+            Kiev[key] = context_reader[key]
+          end
+          request_store = Kiev::RequestStore.store
+          request_store[:background_job] = true
+          request_store[:message_id] = message.message_id
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken/middleware/tree_path_suffix.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Shoryuken
+    module Middleware
+      class TreePathSuffix
+        def initialize(tag)
+          @tag = tag.dup.freeze
+        end
+
+        def call(_worker, _queue, _message, _body)
+          request_store = Kiev::RequestStore.store
+          request_store[:tree_path] ||= ""
+          request_store[:tree_path] += @tag
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/kiev/sidekiq/request_id.rb

@@ -1,38 +1,17 @@
 # frozen_string_literal: true
 
 require "securerandom"
+require "kiev/request_id"
+require "kiev/context_reader"
 
 module Kiev
   module Sidekiq
     class RequestId
-      REQUEST_ID = "request_id"
-      REQUEST_DEPTH = "request_depth"
-      TREE_PATH = "tree_path"
+      include Kiev::RequestId::Mixin
 
-      def call(_worker, job, _queue)
-        Kiev::RequestStore.store[:request_id] = request_id(job)
-        Kiev::RequestStore.store[:request_depth] = request_depth(job)
-        Kiev::RequestStore.store[:tree_path] = tree_path(job)
-        yield
-      end
-
-      private
-
-      def request_id(job)
-        # cron jobs will be triggered without request_id
-        job[REQUEST_ID] || SecureRandom.uuid
-      end
-
-      def tree_root?(job)
-        !job[REQUEST_ID]
-      end
-
-      def request_depth(job)
-        tree_root?(job) ? 0 : (job[REQUEST_DEPTH].to_i + 1)
-      end
-
-      def tree_path(job)
-        tree_root?(job) ? SubrequestHelper.root_path(synchronous: false) : job[TREE_PATH]
+      def call(_worker, job, _queue, &block)
+        context_reader = Kiev::ContextReader.new(job)
+        wrap_request_id(context_reader, &block)
       end
     end
   end

data/lib/kiev/sidekiq/request_logger.rb

@@ -3,36 +3,12 @@
 module Kiev
   module Sidekiq
     class RequestLogger
-      NEW_LINE = "\n"
-      ARGS = "args"
-
-      def call(_worker, job, _queue)
-        began_at = Time.now
-        error = nil
-
-        begin
-          return_value = yield
-        rescue Exception => exception
-          error = exception
-        end
+      include Kiev::RequestLogger::Mixin
 
-        begin
-          data = {
-            params: job[ARGS],
-            request_duration: ((Time.now - began_at) * 1000).round(3)
-          }
-
-          if error
-            data[:error_class] = error.class.name
-            data[:error_message] = error.message[0..5000]
-            data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000]
-          end
+      ARGS = "args"
 
-          Kiev.event(:job_finished, data)
-        ensure
-          raise error if error
-          return_value
-        end
+      def call(_worker, job, _queue, &block)
+        wrap_request_logger(:job_finished, params: job[ARGS], &block)
       end
     end
   end

data/lib/kiev/test.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Kiev
+  # Test helpers for testing both Kiev itself and products that use it.
+  module Test
+    module Log
+      STREAM = StringIO.new
+
+      module_function
+
+      def configure
+        @logs = []
+        Kiev.configure do |c|
+          c.log_path = STREAM
+        end
+      end
+
+      def clear
+        STREAM.rewind
+        STREAM.truncate(0)
+        @logs = []
+      end
+
+      def entries
+        return @logs unless @logs.empty?
+        @logs = raw_logs.each_line.map(&::JSON.method(:parse))
+      rescue
+        puts raw_logs
+        raise
+      end
+
+      def raw_logs
+        STREAM.string
+      end
+    end
+  end
+end

data/lib/kiev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kiev
-  VERSION = "2.7.3"
+  VERSION = "2.8.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiev
 version: !ruby/object:Gem::Version
-  version: 2.7.3
+  version: 2.8.0
 platform: ruby
 authors:
 - Blacklane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-10-18 00:00:00.000000000 Z
+date: 2018-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -144,6 +144,7 @@ files:
 - gemfiles/que_0.12.3.gemfile
 - gemfiles/rails_4.1.gemfile
 - gemfiles/rails_4.2.gemfile
+- gemfiles/shoryuken_3.1.gemfile
 - gemfiles/sidekiq_4.2.gemfile
 - gemfiles/sinatra_1.4.gemfile
 - gemfiles/sinatra_2.0.gemfile
@@ -153,6 +154,7 @@ files:
 - lib/kiev/base.rb
 - lib/kiev/base52.rb
 - lib/kiev/config.rb
+- lib/kiev/context_reader.rb
 - lib/kiev/her_ext/client_request_id.rb
 - lib/kiev/httparty.rb
 - lib/kiev/json.rb
@@ -170,7 +172,18 @@ files:
 - lib/kiev/request_body_filter/form_data.rb
 - lib/kiev/request_body_filter/json.rb
 - lib/kiev/request_body_filter/xml.rb
+- lib/kiev/request_id.rb
+- lib/kiev/request_logger.rb
 - lib/kiev/request_store.rb
+- lib/kiev/shoryuken.rb
+- lib/kiev/shoryuken/context_reader.rb
+- lib/kiev/shoryuken/middleware.rb
+- lib/kiev/shoryuken/middleware/message_tracer.rb
+- lib/kiev/shoryuken/middleware/request_id.rb
+- lib/kiev/shoryuken/middleware/request_logger.rb
+- lib/kiev/shoryuken/middleware/request_store.rb
+- lib/kiev/shoryuken/middleware/store_request_details.rb
+- lib/kiev/shoryuken/middleware/tree_path_suffix.rb
 - lib/kiev/sidekiq.rb
 - lib/kiev/sidekiq/client_request_id.rb
 - lib/kiev/sidekiq/request_id.rb
@@ -178,6 +191,7 @@ files:
 - lib/kiev/sidekiq/request_store.rb
 - lib/kiev/sidekiq/store_request_details.rb
 - lib/kiev/subrequest_helper.rb
+- lib/kiev/test.rb
 - lib/kiev/util.rb
 - lib/kiev/version.rb
 homepage: https://github.com/blacklane/kiev