kiev 4.4.0 → 4.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f25b13c7dfe2a466ce7f9b0f03f3b4078d70158d3241aa589568124ab8b5501
-  data.tar.gz: 3665a4859072ec37b624a89c218ec362e019a2994236901a2ce2dfc55a2a60ca
+  metadata.gz: f160c0e287ac552fab279ebc1974bf2de5704af494766872e1e2abfa11b48766
+  data.tar.gz: f2601c9be33ecc63b3741e76832b2b409250a4b5e7365ede3a40cc6a59c0eb6f
 SHA512:
-  metadata.gz: 35a76a1baf47a1c857ef9c0a6b39f9adb513dd7366c372d6cbcb148980b25772759a529c82559af5779d61bf2eb51706016463fda47aaa85c8a0a3604d5667c0
-  data.tar.gz: b7830138907f5a65d062c3e5f2c7e2d3a9bced6595ac428b9e14ab47f11ebbef94abd35bd590a9dc5890e1045d8fdd86a85a643b1aa29c1dcb545d921045c2d6
+  metadata.gz: 9560e9929db4c64e968dc39d2efadb6b9a09cff5ced0a7e5ffacd823c63273bcd05be4e1918b5e4b70586c73f8520e4d62dccc9e83795908f0981d0ed14ca6e1
+  data.tar.gz: 4caa6b83331e522867ec3254ac5a3f0356c3336a452ade39e9c8df022f5409a17cc1a024c10cee80353252bec8da66b36aea5430cc4947c59acc0dab0fa1eccd

data/.github/workflows/push.yml

@@ -21,6 +21,7 @@ jobs:
       matrix:
         ruby: ['2.5.1', '2.7.2', '2.6.5']
         gemfile:
+          - gemfiles/ruby_kafka.gemfile
           - gemfiles/que_0.12.2.gemfile
           - gemfiles/que_0.12.3.gemfile
           - gemfiles/rails_5.2.gemfile

data/README.md

@@ -154,6 +154,36 @@ Kiev::Shoryuken.enable
 
 The name of the worker class is not logged by default. Configure [`persistent_log_fields` option](#persistent_log_fields) to include `"shoryuken_class"` if you want this.
 
+### AWS SNS
+
+To enhance messages published to SNS topics you can use the ContextInjector:
+
+```ruby
+sns_message = { topic_arn: "...",  message: "{...}" }
+Kiev::Kafka.inject_context(sns_message[:message_attributes])
+
+```
+
+After this operation the message attributes will also include required context for the Kiev logger.
+
+### Kafka
+
+To enhance messages published to Kafka topics you can use the ContextInjector:
+
+```ruby
+Kiev::Kafka.inject_context(headers)
+```
+
+After this operation the headers variable will also include required context for the Kiev logger.
+
+If you have a consumed `Kafka::FetchedMessage` you can extract logger context with: 
+
+```ruby
+Kiev::Kafka.extract_context(message)
+```
+
+This will work regardless if headers are in HTTP format, e.g. `X-Tracking-Id` or plain field names: `tracking_id`. Plus the `message_key` field will contain the key of processed message. In case you want to log some more fields configure `persistent_log_fields` and `jobs_propagated_fields`.  
+
 ### Que
 
 Add the following lines to your initializer code:
@@ -219,7 +249,9 @@ For web requests the Kiev middleware will log the following information by defau
 
 * `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option.
 
-* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4).
+* `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). (currently deprecated in favor of a new name: `tracking_id`)
+
+* `tracking_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). If not provided the value is seeded from deprecated `request_id`. 
 
 * `request_depth` represents the position of the current request within a chain of requests. It starts with 0.
 

data/gemfiles/ruby_kafka.gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "ruby-kafka", "~> 0.7.10"
+
+gem "rack-test", require: false
+gem "rspec", require: false
+gem "minitest-reporters", require: false
+
+gemspec path: "../"

data/kiev.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rack", ">= 1", "< 3"
   spec.add_dependency "request_store", ">= 1.0", "< 1.4"
   spec.add_dependency "ruby_dig", "~> 0.0.2" # to support ruby 2.2
-  spec.add_development_dependency "rake", '~> 0'
-  spec.add_development_dependency "rspec", '~> 0'
+  spec.add_development_dependency "rake", "~> 0"
+  spec.add_development_dependency "rspec", "~> 3.10"
   spec.add_development_dependency "rubocop", "~> 0.54"
 end

data/lib/kiev.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require_relative "kiev/base"
+require_relative "kiev/aws_sns" if defined?(AWS::SNS)
+require_relative "kiev/kafka" if defined?(Kafka)
 require_relative "kiev/rack" if defined?(Rack)
 require_relative "kiev/railtie" if defined?(Rails)
 require_relative "kiev/sidekiq" if defined?(Sidekiq)

data/lib/kiev/aws_sns.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module AwsSns
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::AwsSns::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/aws_sns/context_injector.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module AwsSns
+    class ContextInjector
+      # @param [Hash] message_attributes Injects context headers
+      # @return [Hash]
+      def call(message_attributes = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          message_attributes[key] = {
+            data_type: "String",
+            string_value: value.to_s
+          }
+        end
+        message_attributes
+      end
+    end
+  end
+end

data/lib/kiev/base.rb

@@ -47,7 +47,9 @@ module Kiev
     end
 
     def request_id
-      RequestStore.store[:request_id]
+      RequestStore.store[:tracking_id]
     end
+
+    alias_method :tracking_id, :request_id
   end
 end

data/lib/kiev/kafka.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module Kiev
+  module Kafka
+    require_relative "kafka/context_extractor"
+    require_relative "kafka/context_injector"
+
+    class << self
+      # @param [Kafka::FetchedMessage] message
+      def extract_context(message)
+        Kiev::Kafka::ContextExtractor.new.call(message)
+      end
+
+      # @param [Hash] headers
+      def inject_context(headers = {})
+        Kiev::Kafka::ContextInjector.new.call(headers)
+      end
+    end
+  end
+end

data/lib/kiev/kafka/context_extractor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "message_context"
+require "kiev/request_id"
+require "kiev/context_reader"
+
+module Kiev
+  module Kafka
+    class ContextExtractor
+      include Kiev::RequestId::Mixin
+
+      # @param [Kafka::FetchedMessage] message
+      def call(message)
+        context = Kiev::Kafka::MessageContext.new(message)
+        context_reader = Kiev::ContextReader.new(context)
+        wrap_request_id(context_reader) {}
+
+        Kiev[:message_key] = message.key
+
+        Config.instance.jobs_propagated_fields.each do |key|
+          Kiev[key] = context_reader[key]
+        end
+      end
+    end
+  end
+end

data/lib/kiev/kafka/context_injector.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "kiev/config"
+require "kiev/subrequest_helper"
+
+module Kiev
+  module Kafka
+    class ContextInjector
+      # @param [Hash] headers Injects context headers
+      # @return [Hash]
+      def call(headers = {})
+        Kiev::SubrequestHelper.payload.each do |key, value|
+          field_key = Kiev::Config::DEFAULT_HTTP_PROPAGATED_FIELDS.fetch(key.to_sym, key)
+          headers[field_key] = value
+        end
+        headers
+      end
+    end
+  end
+end

data/lib/kiev/kafka/message_context.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Kiev
+  module Kafka
+    class MessageContext
+      # @param [Kafka::FetchedMessage] message
+      def initialize(message)
+        @headers = message.headers
+      end
+
+      def value(field)
+        headers[header_key(field)] || headers[field.to_s]
+      end
+
+      alias_method :[], :value
+
+      private
+
+      attr_reader :headers
+
+      # @param [String] field
+      def header_key(field)
+        "x_#{field}".gsub("_", " ").split.map(&:capitalize).join("-")
+      end
+    end
+  end
+end

data/lib/kiev/shoryuken/middleware/message_tracer.rb

@@ -1,17 +1,14 @@
 # frozen_string_literal: true
 
+require "kiev/aws_sns/context_injector"
+
 module Kiev
   module Shoryuken
     module Middleware
       class MessageTracer
         def call(options)
-          attrbutes = options[:message_attributes] ||= {}
-          SubrequestHelper.payload.each do |key, value|
-            attrbutes[key] = {
-              data_type: "String",
-              string_value: value.to_s
-            }
-          end
+          options[:message_attributes] ||= {}
+          Kiev::AwsSns::ContextInjector.new.call(options[:message_attributes])
           yield
         end
       end

data/lib/kiev/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kiev
-  VERSION = "4.4.0"
+  VERSION = "4.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kiev
 version: !ruby/object:Gem::Version
-  version: 4.4.0
+  version: 4.5.0
 platform: ruby
 authors:
 - Blacklane
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-03 00:00:00.000000000 Z
+date: 2021-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: oga
@@ -98,14 +98,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -145,6 +145,7 @@ files:
 - gemfiles/rails_4.1.gemfile
 - gemfiles/rails_4.2.gemfile
 - gemfiles/rails_5.2.gemfile
+- gemfiles/ruby_kafka.gemfile
 - gemfiles/shoryuken_4.0.gemfile
 - gemfiles/sidekiq_4.2.gemfile
 - gemfiles/sinatra_1.4.gemfile
@@ -152,6 +153,8 @@ files:
 - kiev.gemspec
 - lib/ext/rack/common_logger.rb
 - lib/kiev.rb
+- lib/kiev/aws_sns.rb
+- lib/kiev/aws_sns/context_injector.rb
 - lib/kiev/base.rb
 - lib/kiev/base52.rb
 - lib/kiev/config.rb
@@ -160,6 +163,10 @@ files:
 - lib/kiev/her_ext/client_request_id.rb
 - lib/kiev/httparty.rb
 - lib/kiev/json.rb
+- lib/kiev/kafka.rb
+- lib/kiev/kafka/context_extractor.rb
+- lib/kiev/kafka/context_injector.rb
+- lib/kiev/kafka/message_context.rb
 - lib/kiev/logger.rb
 - lib/kiev/param_filter.rb
 - lib/kiev/que/job.rb