kiev 4.3.0 → 4.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/push.yml +80 -0
  3. data/.rubocop.yml +8 -1
  4. data/.ruby-version +1 -1
  5. data/Gemfile +1 -1
  6. data/README.md +34 -2
  7. data/gemfiles/que_0.12.2.gemfile +3 -1
  8. data/gemfiles/que_0.12.3.gemfile +3 -1
  9. data/gemfiles/rails_4.1.gemfile +3 -1
  10. data/gemfiles/rails_4.2.gemfile +3 -1
  11. data/gemfiles/rails_5.2.gemfile +15 -0
  12. data/gemfiles/ruby_kafka.gemfile +11 -0
  13. data/gemfiles/{shoryuken_3.1.gemfile → shoryuken_4.0.gemfile} +4 -2
  14. data/gemfiles/sidekiq_4.2.gemfile +3 -2
  15. data/gemfiles/sinatra_1.4.gemfile +4 -2
  16. data/gemfiles/sinatra_2.0.gemfile +4 -2
  17. data/kiev.gemspec +8 -6
  18. data/lib/kiev/aws_sns/context_injector.rb +22 -0
  19. data/lib/kiev/aws_sns.rb +16 -0
  20. data/lib/kiev/base.rb +17 -3
  21. data/lib/kiev/base52.rb +1 -1
  22. data/lib/kiev/config.rb +1 -0
  23. data/lib/kiev/context_reader.rb +6 -3
  24. data/lib/kiev/kafka/context_extractor.rb +26 -0
  25. data/lib/kiev/kafka/context_injector.rb +20 -0
  26. data/lib/kiev/kafka/message_context.rb +27 -0
  27. data/lib/kiev/kafka.rb +22 -0
  28. data/lib/kiev/logger.rb +13 -12
  29. data/lib/kiev/param_filter.rb +27 -4
  30. data/lib/kiev/que/job.rb +2 -1
  31. data/lib/kiev/rack/request_id.rb +15 -13
  32. data/lib/kiev/rack/request_logger.rb +2 -4
  33. data/lib/kiev/request_id.rb +2 -1
  34. data/lib/kiev/request_logger.rb +3 -2
  35. data/lib/kiev/shoryuken/context_reader.rb +2 -0
  36. data/lib/kiev/shoryuken/middleware/message_tracer.rb +4 -7
  37. data/lib/kiev/test.rb +2 -1
  38. data/lib/kiev/util.rb +1 -0
  39. data/lib/kiev/version.rb +1 -1
  40. data/lib/kiev.rb +2 -0
  41. metadata +43 -34
  42. data/.travis.yml +0 -28
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 077b62ba98a1a883bb9cf7596983156864e1011bdb189ee80cc1d0e8e4039c08
4
- data.tar.gz: beb8bdd2f1d31501d5ef4d501d221c1fabe522b9a106e4f7bb150e46aa168029
3
+ metadata.gz: 5da27510343d0217fedb4e4282cf9fc6468a63be4484935e2495e31205a2b188
4
+ data.tar.gz: 3a7ac16381946c418fa4265ab440da10397514766e5c091a87847f4451644251
5
5
  SHA512:
6
- metadata.gz: bb2ff70083d47ac226628cb796f34f607454c6db6b458a63cca06fdbf014a75f28d02ddc29a7b9caf8e4c309148b0adfbfc0a4cdbafcf1030d4e7996a65cf4f5
7
- data.tar.gz: 3fb4a342b5d0b475da14cd99f553ee3b5ee02a1f299f11e2d1afb0ab11353672a14373850b657ca9cc755d6d4474439d32d95b09bc94177c749f9b04eda71a3e
6
+ metadata.gz: ee5e47a70aae2576a0d119c0347fa9707f8a1ff53e8db3f5c1b8ae0bcc39907fd755f1d6112eca317e3b22ce8e2bdb59e83f3e9b70b86014fae995a4377da466
7
+ data.tar.gz: d976cb714c3e9ca5a2f3aa9c0a505a78d38d4a1c77faddc408555dec2ff3f337dbc9430d881630165933084e2a5c88051ef6fbe530a2c79ebf2f24b64949a329
@@ -0,0 +1,80 @@
1
+ # This workflow uses actions that are not certified by GitHub.
2
+ # They are provided by a third-party and are governed by
3
+ # separate terms of service, privacy policy, and support
4
+ # documentation.
5
+ # This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
6
+ # For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
7
+
8
+ name: Main CI
9
+
10
+ on:
11
+ push:
12
+ branches: [ master ]
13
+ pull_request:
14
+ branches: [ master ]
15
+
16
+ jobs:
17
+ test:
18
+
19
+ runs-on: ubuntu-latest
20
+ strategy:
21
+ matrix:
22
+ ruby: ['2.5.1', '2.7.2', '2.6.5']
23
+ gemfile:
24
+ - gemfiles/ruby_kafka.gemfile
25
+ - gemfiles/que_0.12.2.gemfile
26
+ - gemfiles/que_0.12.3.gemfile
27
+ - gemfiles/rails_5.2.gemfile
28
+ - gemfiles/shoryuken_4.0.gemfile
29
+ - gemfiles/sidekiq_4.2.gemfile
30
+ - gemfiles/sinatra_1.4.gemfile
31
+ - gemfiles/sinatra_2.0.gemfile
32
+ allow_failures:
33
+ - false
34
+ include:
35
+ - os: ubuntu
36
+ ruby-version: ruby-head
37
+ gemfile: gemfiles/rails_5.2.gemfile
38
+ allow_failures: true
39
+ env:
40
+ BUNDLE_GEMFILE: "${{ matrix.gemfile }}"
41
+ ALLOW_FAILURES: "${{ matrix.allow_failures }}"
42
+ REDIS_URL: "redis://localhost:6379/4"
43
+ DATABASE_URL: ${{ (startsWith(matrix.gemfile,'gemfiles/que') && 'postgres://postgres:postgres@localhost:5432/que_test') || 'sqlite3:db/combustion_test.sqlite'}}
44
+ continue-on-error: ${{ endsWith(matrix.ruby, 'head') || matrix.ruby == 'debug' }}
45
+
46
+ # Service containers to run with `container-job`
47
+ services:
48
+ redis:
49
+ image: redis:latest
50
+ ports:
51
+ - 6379:6379
52
+ # Label used to access the service container
53
+ postgres:
54
+ # Docker Hub image
55
+ image: postgres:9.4
56
+ # Provide the password for postgres
57
+ env:
58
+ POSTGRES_PASSWORD: postgres
59
+ POSTGRES_DB: que_test
60
+ # Set health checks to wait until postgres has started
61
+ ports:
62
+ - 5432:5432
63
+ options: >-
64
+ --health-cmd pg_isready
65
+ --health-interval 10s
66
+ --health-timeout 5s
67
+ --health-retries 5
68
+ steps:
69
+ - uses: actions/checkout@v2
70
+ - name: Set up Ruby
71
+ # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
72
+ # change this to (see https://github.com/ruby/setup-ruby#versioning):
73
+ # uses: ruby/setup-ruby@v1
74
+ uses: ruby/setup-ruby@473e4d8fe5dd94ee328fdfca9f8c9c7afc9dae5e
75
+ with:
76
+ ruby-version: ${{ matrix.ruby }}
77
+ bundler-cache: true # runs 'bundle install' and caches installed gems automatically
78
+
79
+ - name: Run tests
80
+ run: bundle exec rake || $ALLOW_FAILURES
data/.rubocop.yml CHANGED
@@ -1,7 +1,14 @@
1
1
  inherit_from:
2
2
  - https://raw.githubusercontent.com/blacklane/rubocop/master/rubocop.yml
3
3
 
4
- Lint/HandleExceptions:
4
+ AllCops:
5
+ TargetRubyVersion: 2.5
6
+ Exclude:
7
+ - test/rails_app/**/*.rb # auto-generated
8
+ - spec/**/*.rb
9
+ - test/**/*.rb
10
+ - vendor/bundle/**/*
11
+ Lint/SuppressedException:
5
12
  Exclude:
6
13
  - test/**/*.rb
7
14
  - spec/**/*.rb
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 2.3.3
1
+ 2.5.1
data/Gemfile CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_4.1.gemfile"))
3
+ eval_gemfile(File.join(File.dirname(__FILE__), "gemfiles/rails_5.2.gemfile"))
4
4
 
5
5
  gem "wwtd"
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # Kiev [![Build Status](https://travis-ci.org/blacklane/kiev.svg?branch=master)](https://travis-ci.org/blacklane/kiev)
1
+ # Kiev [![Build Status](https://github.com/blacklane/kiev/workflows/Main%20CI/badge.svg?branch=master)](https://github.com/blacklane/kiev/actions?query=workflow%3A%22Main+CI%22) [![Gem Version](https://badge.fury.io/rb/kiev.svg)](https://badge.fury.io/rb/kiev)
2
2
 
3
3
  Kiev is a comprehensive logging library aimed at covering a wide range of frameworks and tools from the Ruby ecosystem:
4
4
 
@@ -154,6 +154,36 @@ Kiev::Shoryuken.enable
154
154
 
155
155
  The name of the worker class is not logged by default. Configure [`persistent_log_fields` option](#persistent_log_fields) to include `"shoryuken_class"` if you want this.
156
156
 
157
+ ### AWS SNS
158
+
159
+ To enhance messages published to SNS topics you can use the ContextInjector:
160
+
161
+ ```ruby
162
+ sns_message = { topic_arn: "...", message: "{...}" }
163
+ Kiev::Kafka.inject_context(sns_message[:message_attributes])
164
+
165
+ ```
166
+
167
+ After this operation the message attributes will also include required context for the Kiev logger.
168
+
169
+ ### Kafka
170
+
171
+ To enhance messages published to Kafka topics you can use the ContextInjector:
172
+
173
+ ```ruby
174
+ Kiev::Kafka.inject_context(headers)
175
+ ```
176
+
177
+ After this operation the headers variable will also include required context for the Kiev logger.
178
+
179
+ If you have a consumed `Kafka::FetchedMessage` you can extract logger context with:
180
+
181
+ ```ruby
182
+ Kiev::Kafka.extract_context(message)
183
+ ```
184
+
185
+ This will work regardless if headers are in HTTP format, e.g. `X-Tracking-Id` or plain field names: `tracking_id`. Plus the `message_key` field will contain the key of processed message. In case you want to log some more fields configure `persistent_log_fields` and `jobs_propagated_fields`.
186
+
157
187
  ### Que
158
188
 
159
189
  Add the following lines to your initializer code:
@@ -219,7 +249,9 @@ For web requests the Kiev middleware will log the following information by defau
219
249
 
220
250
  * `params` attribute will store both query parameters and request body fields (as long as they are parseable). Sensitive fields will be filtered out - see the `#filtered_params` option.
221
251
 
222
- * `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4).
252
+ * `request_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). (currently deprecated in favor of a new name: `tracking_id`)
253
+
254
+ * `tracking_id` is the correlation ID and will be the same across all requests within a chain of requests. It's represented as a UUID (version 4). If not provided the value is seeded from deprecated `request_id`.
223
255
 
224
256
  * `request_depth` represents the position of the current request within a chain of requests. It starts with 0.
225
257
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source "https://rubygems.org"
2
4
 
3
5
  gem "oj"
@@ -11,4 +13,4 @@ gem "rack-test", require: false
11
13
  gem "rspec", require: false
12
14
  gem "minitest-reporters", require: false
13
15
 
14
- gemspec :path => "../"
16
+ gemspec path: "../"
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source "https://rubygems.org"
2
4
 
3
5
  # need it because of bug in https://github.com/chanks/que/issues/191
@@ -12,4 +14,4 @@ gem "rack-test", require: false
12
14
  gem "rspec", require: false
13
15
  gem "minitest-reporters", require: false
14
16
 
15
- gemspec :path => "../"
17
+ gemspec path: "../"
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source "https://rubygems.org"
2
4
 
3
5
  gem "oj"
@@ -10,4 +12,4 @@ gem "rspec", require: false
10
12
  gem "rspec-rails", require: false
11
13
  gem "minitest-reporters", require: false
12
14
 
13
- gemspec :path => "../"
15
+ gemspec path: "../"
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source "https://rubygems.org"
2
4
 
3
5
  gem "oj"
@@ -10,4 +12,4 @@ gem "rspec", require: false
10
12
  gem "rspec-rails", require: false
11
13
  gem "minitest-reporters", require: false
12
14
 
13
- gemspec :path => "../"
15
+ gemspec path: "../"
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "oj"
6
+
7
+ gem "rails", "~> 5.2.4"
8
+ gem "sqlite3", "1.3.13"
9
+
10
+ gem "combustion"
11
+ gem "rspec", require: false
12
+ gem "rspec-rails", require: false
13
+ gem "minitest-reporters", require: false
14
+
15
+ gemspec path: "../"
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "ruby-kafka", "~> 0.7.10"
6
+
7
+ gem "rack-test", require: false
8
+ gem "rspec", require: false
9
+ gem "minitest-reporters", require: false
10
+
11
+ gemspec path: "../"
@@ -1,9 +1,11 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source "https://rubygems.org"
2
4
 
3
5
  gem "aws-sdk", "~> 2.0"
4
- gem "shoryuken", "~> 3.1.0"
6
+ gem "shoryuken", "~> 4.0"
5
7
 
6
8
  gem "rack-test", require: false
7
9
  gem "minitest-reporters", require: false
8
10
 
9
- gemspec :path => "../"
11
+ gemspec path: "../"
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source "https://rubygems.org"
2
4
 
3
5
  gem "oj", "~> 2"
@@ -10,5 +12,4 @@ gem "minitest-reporters", require: false
10
12
 
11
13
  gem "her"
12
14
 
13
- gemspec :path => "../"
14
-
15
+ gemspec path: "../"
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source "https://rubygems.org"
2
4
 
3
5
  gem "oj"
@@ -6,10 +8,10 @@ gem "xml-simple"
6
8
 
7
9
  gem "sinatra", "1.4.7"
8
10
  gem "sinatra-contrib"
9
- gem "rack-parser", :require => "rack/parser"
11
+ gem "rack-parser", require: "rack/parser"
10
12
 
11
13
  gem "rack-test", require: false
12
14
  gem "rspec", require: false
13
15
  gem "minitest-reporters", require: false
14
16
 
15
- gemspec :path => "../"
17
+ gemspec path: "../"
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source "https://rubygems.org"
2
4
 
3
5
  gem "oj"
@@ -6,10 +8,10 @@ gem "xml-simple"
6
8
 
7
9
  gem "sinatra", "2.0.0"
8
10
  gem "sinatra-contrib"
9
- gem "rack-parser", :require => "rack/parser"
11
+ gem "rack-parser", require: "rack/parser"
10
12
 
11
13
  gem "rack-test", require: false
12
14
  gem "rspec", require: false
13
15
  gem "minitest-reporters", require: false
14
16
 
15
- gemspec :path => "../"
17
+ gemspec path: "../"
data/kiev.gemspec CHANGED
@@ -9,7 +9,9 @@ Gem::Specification.new do |spec|
9
9
  spec.licenses = ["MIT"]
10
10
 
11
11
  spec.summary = "Distributed logging to JSON integrated with various Ruby frameworks and tools"
12
- spec.description = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing human-readable output in development mode. It integrates nicely with Rails, Sinatra and other Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients."
12
+ spec.description = "Kiev is a logging tool aimed at distributed environments. It logs to JSON, while providing "\
13
+ "human-readable output in development mode. It integrates nicely with Rails, Sinatra and other"\
14
+ " Rack-based frameworks, Sidekiq, Que, HTTParty, Her and other Faraday-based HTTP clients."
13
15
  spec.homepage = "https://github.com/blacklane/kiev"
14
16
 
15
17
  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
@@ -17,12 +19,12 @@ Gem::Specification.new do |spec|
17
19
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
18
20
  spec.require_paths = ["lib"]
19
21
 
20
- spec.required_ruby_version = ">= 2.0.0"
22
+ spec.required_ruby_version = ">= 2.5"
23
+ spec.add_dependency "oga", "~> 2.2"
21
24
  spec.add_dependency "rack", ">= 1", "< 3"
22
25
  spec.add_dependency "request_store", ">= 1.0", "< 1.4"
23
- spec.add_dependency "oga", "~> 2.2"
24
26
  spec.add_dependency "ruby_dig", "~> 0.0.2" # to support ruby 2.2
25
- spec.add_development_dependency "rake"
26
- spec.add_development_dependency "rspec"
27
- spec.add_development_dependency "rubocop", "0.49.1"
27
+ spec.add_development_dependency "rake", "~> 0"
28
+ spec.add_development_dependency "rspec", "~> 3.10"
29
+ spec.add_development_dependency "rubocop", "~> 0.54"
28
30
  end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "kiev/config"
4
+ require "kiev/subrequest_helper"
5
+
6
+ module Kiev
7
+ module AwsSns
8
+ class ContextInjector
9
+ # @param [Hash] message_attributes Injects context headers
10
+ # @return [Hash]
11
+ def call(message_attributes = {})
12
+ Kiev::SubrequestHelper.payload.each do |key, value|
13
+ message_attributes[key] = {
14
+ data_type: "String",
15
+ string_value: value.to_s
16
+ }
17
+ end
18
+ message_attributes
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,16 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Kiev
6
+ module AwsSns
7
+ require_relative "kafka/context_injector"
8
+
9
+ class << self
10
+ # @param [Hash] headers
11
+ def inject_context(headers = {})
12
+ Kiev::AwsSns::ContextInjector.new.call(headers)
13
+ end
14
+ end
15
+ end
16
+ end
data/lib/kiev/base.rb CHANGED
@@ -26,8 +26,20 @@ module Kiev
26
26
  Config.instance.logger
27
27
  end
28
28
 
29
- def event(event_name, data = EMPTY_OBJ)
30
- logger.log(::Logger::Severity::INFO, data, event_name)
29
+ def filtered_params
30
+ Config.instance.filtered_params
31
+ end
32
+
33
+ def ignored_params
34
+ Config.instance.ignored_params
35
+ end
36
+
37
+ def event(log_name, data = EMPTY_OBJ)
38
+ logger.log(
39
+ ::Logger::Severity::INFO,
40
+ ParamFilter.filter(data, filtered_params, ignored_params),
41
+ log_name
42
+ )
31
43
  end
32
44
 
33
45
  def []=(name, value)
@@ -47,7 +59,9 @@ module Kiev
47
59
  end
48
60
 
49
61
  def request_id
50
- RequestStore.store[:request_id]
62
+ RequestStore.store[:tracking_id]
51
63
  end
64
+
65
+ alias_method :tracking_id, :request_id
52
66
  end
53
67
  end
data/lib/kiev/base52.rb CHANGED
@@ -3,7 +3,7 @@
3
3
  module Kiev
4
4
  module Base52
5
5
  KEYS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".freeze
6
- BASE = KEYS.length.freeze
6
+ BASE = KEYS.length
7
7
 
8
8
  def self.encode(num)
9
9
  return KEYS[0] if num == 0
data/lib/kiev/config.rb CHANGED
@@ -63,6 +63,7 @@ module Kiev
63
63
  ) << :tempfile).freeze
64
64
 
65
65
  DEFAULT_HTTP_PROPAGATED_FIELDS = {
66
+ tracking_id: "X-Tracking-Id",
66
67
  request_id: "X-Request-Id",
67
68
  request_depth: "X-Request-Depth",
68
69
  tree_path: "X-Tree-Path"
@@ -6,6 +6,7 @@ module Kiev
6
6
  # change field lookup.
7
7
  class ContextReader
8
8
  REQUEST_ID = "request_id"
9
+ TRACKING_ID = "tracking_id"
9
10
  REQUEST_DEPTH = "request_depth"
10
11
  TREE_PATH = "tree_path"
11
12
 
@@ -17,12 +18,14 @@ module Kiev
17
18
  subject[key]
18
19
  end
19
20
 
20
- def request_id
21
- self[REQUEST_ID] || SecureRandom.uuid
21
+ def tracking_id
22
+ self[TRACKING_ID] || self[REQUEST_ID] || SecureRandom.uuid
22
23
  end
23
24
 
25
+ alias_method :request_id, :tracking_id
26
+
24
27
  def tree_root?
25
- !self[REQUEST_ID]
28
+ !self[TRACKING_ID] && !self[REQUEST_ID]
26
29
  end
27
30
 
28
31
  def request_depth
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "message_context"
4
+ require "kiev/request_id"
5
+ require "kiev/context_reader"
6
+
7
+ module Kiev
8
+ module Kafka
9
+ class ContextExtractor
10
+ include Kiev::RequestId::Mixin
11
+
12
+ # @param [Kafka::FetchedMessage] message
13
+ def call(message)
14
+ context = Kiev::Kafka::MessageContext.new(message)
15
+ context_reader = Kiev::ContextReader.new(context)
16
+ wrap_request_id(context_reader) {}
17
+
18
+ Kiev[:message_key] = message.key
19
+
20
+ Config.instance.jobs_propagated_fields.each do |key|
21
+ Kiev[key] = context_reader[key]
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "kiev/config"
4
+ require "kiev/subrequest_helper"
5
+
6
+ module Kiev
7
+ module Kafka
8
+ class ContextInjector
9
+ # @param [Hash] headers Injects context headers
10
+ # @return [Hash]
11
+ def call(headers = {})
12
+ Kiev::SubrequestHelper.payload.each do |key, value|
13
+ field_key = Kiev::Config::DEFAULT_HTTP_PROPAGATED_FIELDS.fetch(key.to_sym, key)
14
+ headers[field_key] = value
15
+ end
16
+ headers
17
+ end
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,27 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module Kafka
5
+ class MessageContext
6
+ # @param [Kafka::FetchedMessage] message
7
+ def initialize(message)
8
+ @headers = message.headers
9
+ end
10
+
11
+ def value(field)
12
+ headers[header_key(field)] || headers[field.to_s]
13
+ end
14
+
15
+ alias_method :[], :value
16
+
17
+ private
18
+
19
+ attr_reader :headers
20
+
21
+ # @param [String] field
22
+ def header_key(field)
23
+ "x_#{field}".gsub("_", " ").split.map(&:capitalize).join("-")
24
+ end
25
+ end
26
+ end
27
+ end
data/lib/kiev/kafka.rb ADDED
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Kiev
6
+ module Kafka
7
+ require_relative "kafka/context_extractor"
8
+ require_relative "kafka/context_injector"
9
+
10
+ class << self
11
+ # @param [Kafka::FetchedMessage] message
12
+ def extract_context(message)
13
+ Kiev::Kafka::ContextExtractor.new.call(message)
14
+ end
15
+
16
+ # @param [Hash] headers
17
+ def inject_context(headers = {})
18
+ Kiev::Kafka::ContextInjector.new.call(headers)
19
+ end
20
+ end
21
+ end
22
+ end
data/lib/kiev/logger.rb CHANGED
@@ -12,17 +12,20 @@ module Kiev
12
12
  extend Forwardable
13
13
  def_delegators(*([:@logger] + ::Logger.instance_methods(false)))
14
14
 
15
- DEFAULT_EVENT_NAME = "log"
15
+ DEFAULT_LOG_NAME = "log"
16
+ DEFAULT_MESSAGE = "log"
16
17
  LOG_ERROR = "ERROR"
17
18
  ERROR_STATUS = 500
18
19
 
19
- FORMATTER = proc do |severity, time, event_name, data|
20
+ FORMATTER = proc do |severity, time, log_name, data|
20
21
  entry =
21
22
  {
22
23
  application: Config.instance.app,
23
- event: event_name || DEFAULT_EVENT_NAME,
24
+ log_name: log_name || DEFAULT_LOG_NAME,
24
25
  level: severity,
25
26
  timestamp: time.utc,
27
+ message: log_name || DEFAULT_MESSAGE,
28
+ tracking_id: RequestStore.store[:tracking_id],
26
29
  request_id: RequestStore.store[:request_id],
27
30
  request_depth: RequestStore.store[:request_depth],
28
31
  tree_path: RequestStore.store[:tree_path]
@@ -38,12 +41,12 @@ module Kiev
38
41
  entry[:jid] = RequestStore.store[:jid]
39
42
  end
40
43
 
41
- if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(event_name)
44
+ if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(log_name)
42
45
  entry[:tree_leaf] = true
43
46
  end
44
47
 
45
48
  if RequestStore.store[:payload]
46
- if %i(request_finished job_finished).include?(event_name)
49
+ if %i(request_finished job_finished).include?(log_name)
47
50
  entry.merge!(RequestStore.store[:payload])
48
51
  else
49
52
  Config.instance.persistent_log_fields.each do |field|
@@ -67,17 +70,17 @@ module Kiev
67
70
  JSON.logstash(entry)
68
71
  end
69
72
 
70
- DEVELOPMENT_FORMATTER = proc do |severity, time, event_name, data|
73
+ DEVELOPMENT_FORMATTER = proc do |severity, time, log_name, data|
71
74
  entry = []
72
75
 
73
76
  entry << time.iso8601
74
- entry << (event_name || severity).upcase
77
+ entry << (log_name || severity).upcase
75
78
 
76
79
  if data.is_a?(String)
77
80
  entry << "#{data}\n"
78
81
  end
79
82
 
80
- if %i(request_finished job_finished).include?(event_name)
83
+ if %i(request_finished job_finished).include?(log_name)
81
84
  verb = RequestStore.store[:request_verb]
82
85
  path = RequestStore.store[:request_path]
83
86
  entry << "#{verb} #{path}" if verb && path
@@ -92,10 +95,8 @@ module Kiev
92
95
  entry << "(#{duration}ms)" if duration
93
96
  entry << "\n"
94
97
 
95
- meta = {
96
- request_id: RequestStore.store[:request_id],
97
- request_depth: RequestStore.store[:request_depth]
98
- }.merge!(Hash(RequestStore.store[:payload]))
98
+ meta = RequestStore.store.slice(:trakcing_id, :request_id, :request_depth)
99
+ .reverse_merge!(Hash(RequestStore.store[:payload]))
99
100
 
100
101
  meta.reject! { |_, value| value.nil? }
101
102
 
@@ -1,12 +1,23 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kiev
4
- module ParamFilter
4
+ class ParamFilter
5
5
  FILTERED = "[FILTERED]"
6
6
 
7
7
  def self.filter(params, filtered_params, ignored_params)
8
+ new(filtered_params, ignored_params).call(params)
9
+ end
10
+
11
+ def initialize(filtered_params, ignored_params)
12
+ @filtered_params = normalize(filtered_params)
13
+ @ignored_params = normalize(ignored_params)
14
+ end
15
+
16
+ def call(params)
17
+ return params unless filterable?(params)
18
+
8
19
  params.each_with_object({}) do |(key, value), acc|
9
- next if ignored_params.include?(key)
20
+ next if ignored_params.include?(key.to_s)
10
21
 
11
22
  if defined?(ActionDispatch) && value.is_a?(ActionDispatch::Http::UploadedFile)
12
23
  value = {
@@ -17,14 +28,26 @@ module Kiev
17
28
  end
18
29
 
19
30
  acc[key] =
20
- if filtered_params.include?(key) && !value.is_a?(Hash)
31
+ if filtered_params.include?(key.to_s) && !value.is_a?(Hash)
21
32
  FILTERED
22
33
  elsif value.is_a?(Hash)
23
- filter(value, filtered_params, ignored_params)
34
+ call(value)
24
35
  else
25
36
  value
26
37
  end
27
38
  end
28
39
  end
40
+
41
+ private
42
+
43
+ attr_reader :filtered_params, :ignored_params
44
+
45
+ def filterable?(params)
46
+ params.respond_to?(:each_with_object)
47
+ end
48
+
49
+ def normalize(params)
50
+ Set.new(params.map(&:to_s))
51
+ end
29
52
  end
30
53
  end
data/lib/kiev/que/job.rb CHANGED
@@ -47,7 +47,8 @@ module Kiev
47
47
  Kiev[key] = payload[key]
48
48
  end
49
49
  request_store = Kiev::RequestStore.store
50
- request_store[:request_id] = payload[:request_id]
50
+ request_store[:tracking_id] = payload[:tracking_id]
51
+ request_store[:request_id] = payload[:tracking_id] || payload[:request_id]
51
52
  request_store[:request_depth] = payload[:request_depth].to_i + 1
52
53
  request_store[:tree_path] = payload[:tree_path]
53
54
 
@@ -14,24 +14,26 @@ module Kiev
14
14
 
15
15
  def call(env)
16
16
  request_id_header_out = to_rack(:request_id)
17
- request_id_header_in = to_http(:request_id)
17
+ tracking_id_header_out = to_rack(:tracking_id)
18
18
 
19
- request_id = make_request_id(env[RAILS_REQUEST_ID] || env[request_id_header_in])
20
- RequestStore.store[:request_id] = request_id
19
+ tracking_id = make_tracking_id(env[to_http(:tracking_id)] || env[RAILS_REQUEST_ID] || env[to_http(:request_id)])
20
+ RequestStore.store[:tracking_id] = tracking_id
21
+ RequestStore.store[:request_id] = tracking_id
21
22
  RequestStore.store[:request_depth] = request_depth(env)
22
23
  RequestStore.store[:tree_path] = tree_path(env)
23
24
 
24
- @app.call(env).tap { |_status, headers, _body| headers[request_id_header_out] = request_id }
25
+ @app.call(env).tap do |_status, headers, _body|
26
+ headers[tracking_id_header_out] = tracking_id
27
+ headers[request_id_header_out] = tracking_id
28
+ end
25
29
  end
26
30
 
27
31
  private
28
32
 
29
- # TODO: in Rails 5 they set `headers[X_REQUEST_ID]`, so this will not work
30
- # https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/request_id.rb
31
- # https://github.com/interagent/pliny/blob/master/lib/pliny/middleware/request_id.rb
32
33
  def tree_root?(env)
34
+ tracking_id_header_in = to_http(:tracking_id)
33
35
  request_id_header_in = to_http(:request_id)
34
- !env[request_id_header_in]
36
+ !env[tracking_id_header_in] && !env[request_id_header_in]
35
37
  end
36
38
 
37
39
  def request_depth(env)
@@ -52,15 +54,15 @@ module Kiev
52
54
  Config.instance.all_http_propagated_fields[value]
53
55
  end
54
56
 
55
- def make_request_id(request_id)
56
- if request_id.nil? || request_id.empty?
57
- internal_request_id
57
+ def make_tracking_id(tracking_id)
58
+ if tracking_id.nil? || tracking_id.empty?
59
+ internal_tracking_id
58
60
  else
59
- Util.sanitize(request_id)
61
+ Util.sanitize(tracking_id)
60
62
  end
61
63
  end
62
64
 
63
- def internal_request_id
65
+ def internal_tracking_id
64
66
  SecureRandom.uuid
65
67
  end
66
68
  end
@@ -85,8 +85,6 @@ module Kiev
85
85
  request.params
86
86
  end
87
87
 
88
- params = ParamFilter.filter(params, config.filtered_params, config.ignored_params)
89
-
90
88
  data = {
91
89
  host: request.host, # env["HTTP_HOST"] || env["HTTPS_HOST"],
92
90
  params: params.empty? ? nil : params, # env[Rack::QUERY_STRING],
@@ -125,8 +123,8 @@ module Kiev
125
123
  sio = StringIO.new(data[:body])
126
124
  gz = Zlib::GzipReader.new(sio)
127
125
  data[:body] = gz.read
128
- rescue Zlib::GzipFile::Error => err
129
- data[:gzip_parse_error] = err.message
126
+ rescue Zlib::GzipFile::Error => e
127
+ data[:gzip_parse_error] = e.message
130
128
  end
131
129
  end
132
130
  end
@@ -7,7 +7,8 @@ module Kiev
7
7
 
8
8
  def wrap_request_id(context_reader, &_block)
9
9
  request_store = Kiev::RequestStore.store
10
- request_store[:request_id] = context_reader.request_id
10
+ request_store[:tracking_id] = context_reader.tracking_id || context_reader.request_id
11
+ request_store[:request_id] = request_store[:tracking_id]
11
12
  request_store[:request_depth] = context_reader.request_depth
12
13
  request_store[:tree_path] = context_reader.tree_path
13
14
  yield
@@ -12,8 +12,8 @@ module Kiev
12
12
 
13
13
  begin
14
14
  return_value = yield
15
- rescue StandardError => exception
16
- error = exception
15
+ rescue StandardError => e
16
+ error = e
17
17
  end
18
18
 
19
19
  begin
@@ -28,6 +28,7 @@ module Kiev
28
28
  Kiev.event(event, data)
29
29
  ensure
30
30
  raise error if error
31
+
31
32
  return_value
32
33
  end
33
34
  end
@@ -12,8 +12,10 @@ module Kiev
12
12
 
13
13
  def [](key)
14
14
  return unless @message_attributes.key?(key)
15
+
15
16
  attribute_value = @message_attributes[key]
16
17
  return unless attribute_value.data_type == "String"
18
+
17
19
  attribute_value.string_value
18
20
  end
19
21
  end
@@ -1,17 +1,14 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "kiev/aws_sns/context_injector"
4
+
3
5
  module Kiev
4
6
  module Shoryuken
5
7
  module Middleware
6
8
  class MessageTracer
7
9
  def call(options)
8
- attrbutes = options[:message_attributes] ||= {}
9
- SubrequestHelper.payload.each do |key, value|
10
- attrbutes[key] = {
11
- data_type: "String",
12
- string_value: value.to_s
13
- }
14
- end
10
+ options[:message_attributes] ||= {}
11
+ Kiev::AwsSns::ContextInjector.new.call(options[:message_attributes])
15
12
  yield
16
13
  end
17
14
  end
data/lib/kiev/test.rb CHANGED
@@ -25,8 +25,9 @@ module Kiev
25
25
 
26
26
  def entries
27
27
  return @logs unless @logs.empty?
28
+
28
29
  @logs = raw_logs.each_line.map(&::JSON.method(:parse))
29
- rescue
30
+ rescue StandardError
30
31
  puts raw_logs
31
32
  raise
32
33
  end
data/lib/kiev/util.rb CHANGED
@@ -4,6 +4,7 @@ module Kiev
4
4
  module Util
5
5
  def self.sanitize(value)
6
6
  return unless value
7
+
7
8
  value.gsub(/[^\w\-]/, "")[0...255]
8
9
  end
9
10
 
data/lib/kiev/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Kiev
4
- VERSION = "4.3.0"
4
+ VERSION = "4.7.0"
5
5
  end
data/lib/kiev.rb CHANGED
@@ -1,6 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "kiev/base"
4
+ require_relative "kiev/aws_sns" if defined?(AWS::SNS)
5
+ require_relative "kiev/kafka" if defined?(Kafka)
4
6
  require_relative "kiev/rack" if defined?(Rack)
5
7
  require_relative "kiev/railtie" if defined?(Rails)
6
8
  require_relative "kiev/sidekiq" if defined?(Sidekiq)
metadata CHANGED
@@ -1,15 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kiev
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.3.0
4
+ version: 4.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Blacklane
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-09-08 00:00:00.000000000 Z
11
+ date: 2021-10-20 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: oga
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.2'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.2'
13
27
  - !ruby/object:Gem::Dependency
14
28
  name: rack
15
29
  requirement: !ruby/object:Gem::Requirement
@@ -50,20 +64,6 @@ dependencies:
50
64
  - - "<"
51
65
  - !ruby/object:Gem::Version
52
66
  version: '1.4'
53
- - !ruby/object:Gem::Dependency
54
- name: oga
55
- requirement: !ruby/object:Gem::Requirement
56
- requirements:
57
- - - "~>"
58
- - !ruby/object:Gem::Version
59
- version: '2.2'
60
- type: :runtime
61
- prerelease: false
62
- version_requirements: !ruby/object:Gem::Requirement
63
- requirements:
64
- - - "~>"
65
- - !ruby/object:Gem::Version
66
- version: '2.2'
67
67
  - !ruby/object:Gem::Dependency
68
68
  name: ruby_dig
69
69
  requirement: !ruby/object:Gem::Requirement
@@ -82,58 +82,58 @@ dependencies:
82
82
  name: rake
83
83
  requirement: !ruby/object:Gem::Requirement
84
84
  requirements:
85
- - - ">="
85
+ - - "~>"
86
86
  - !ruby/object:Gem::Version
87
87
  version: '0'
88
88
  type: :development
89
89
  prerelease: false
90
90
  version_requirements: !ruby/object:Gem::Requirement
91
91
  requirements:
92
- - - ">="
92
+ - - "~>"
93
93
  - !ruby/object:Gem::Version
94
94
  version: '0'
95
95
  - !ruby/object:Gem::Dependency
96
96
  name: rspec
97
97
  requirement: !ruby/object:Gem::Requirement
98
98
  requirements:
99
- - - ">="
99
+ - - "~>"
100
100
  - !ruby/object:Gem::Version
101
- version: '0'
101
+ version: '3.10'
102
102
  type: :development
103
103
  prerelease: false
104
104
  version_requirements: !ruby/object:Gem::Requirement
105
105
  requirements:
106
- - - ">="
106
+ - - "~>"
107
107
  - !ruby/object:Gem::Version
108
- version: '0'
108
+ version: '3.10'
109
109
  - !ruby/object:Gem::Dependency
110
110
  name: rubocop
111
111
  requirement: !ruby/object:Gem::Requirement
112
112
  requirements:
113
- - - '='
113
+ - - "~>"
114
114
  - !ruby/object:Gem::Version
115
- version: 0.49.1
115
+ version: '0.54'
116
116
  type: :development
117
117
  prerelease: false
118
118
  version_requirements: !ruby/object:Gem::Requirement
119
119
  requirements:
120
- - - '='
120
+ - - "~>"
121
121
  - !ruby/object:Gem::Version
122
- version: 0.49.1
122
+ version: '0.54'
123
123
  description: Kiev is a logging tool aimed at distributed environments. It logs to
124
124
  JSON, while providing human-readable output in development mode. It integrates nicely
125
125
  with Rails, Sinatra and other Rack-based frameworks, Sidekiq, Que, HTTParty, Her
126
126
  and other Faraday-based HTTP clients.
127
- email:
127
+ email:
128
128
  executables: []
129
129
  extensions: []
130
130
  extra_rdoc_files: []
131
131
  files:
132
+ - ".github/workflows/push.yml"
132
133
  - ".gitignore"
133
134
  - ".rspec"
134
135
  - ".rubocop.yml"
135
136
  - ".ruby-version"
136
- - ".travis.yml"
137
137
  - Gemfile
138
138
  - LICENSE.md
139
139
  - README.md
@@ -144,13 +144,17 @@ files:
144
144
  - gemfiles/que_0.12.3.gemfile
145
145
  - gemfiles/rails_4.1.gemfile
146
146
  - gemfiles/rails_4.2.gemfile
147
- - gemfiles/shoryuken_3.1.gemfile
147
+ - gemfiles/rails_5.2.gemfile
148
+ - gemfiles/ruby_kafka.gemfile
149
+ - gemfiles/shoryuken_4.0.gemfile
148
150
  - gemfiles/sidekiq_4.2.gemfile
149
151
  - gemfiles/sinatra_1.4.gemfile
150
152
  - gemfiles/sinatra_2.0.gemfile
151
153
  - kiev.gemspec
152
154
  - lib/ext/rack/common_logger.rb
153
155
  - lib/kiev.rb
156
+ - lib/kiev/aws_sns.rb
157
+ - lib/kiev/aws_sns/context_injector.rb
154
158
  - lib/kiev/base.rb
155
159
  - lib/kiev/base52.rb
156
160
  - lib/kiev/config.rb
@@ -159,6 +163,10 @@ files:
159
163
  - lib/kiev/her_ext/client_request_id.rb
160
164
  - lib/kiev/httparty.rb
161
165
  - lib/kiev/json.rb
166
+ - lib/kiev/kafka.rb
167
+ - lib/kiev/kafka/context_extractor.rb
168
+ - lib/kiev/kafka/context_injector.rb
169
+ - lib/kiev/kafka/message_context.rb
162
170
  - lib/kiev/logger.rb
163
171
  - lib/kiev/param_filter.rb
164
172
  - lib/kiev/que/job.rb
@@ -199,7 +207,7 @@ homepage: https://github.com/blacklane/kiev
199
207
  licenses:
200
208
  - MIT
201
209
  metadata: {}
202
- post_install_message:
210
+ post_install_message:
203
211
  rdoc_options: []
204
212
  require_paths:
205
213
  - lib
@@ -207,15 +215,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
207
215
  requirements:
208
216
  - - ">="
209
217
  - !ruby/object:Gem::Version
210
- version: 2.0.0
218
+ version: '2.5'
211
219
  required_rubygems_version: !ruby/object:Gem::Requirement
212
220
  requirements:
213
221
  - - ">="
214
222
  - !ruby/object:Gem::Version
215
223
  version: '0'
216
224
  requirements: []
217
- rubygems_version: 3.0.3
218
- signing_key:
225
+ rubyforge_project:
226
+ rubygems_version: 2.7.6
227
+ signing_key:
219
228
  specification_version: 4
220
229
  summary: Distributed logging to JSON integrated with various Ruby frameworks and tools
221
230
  test_files: []
data/.travis.yml DELETED
@@ -1,28 +0,0 @@
1
- sudo: false
2
- branches:
3
- only:
4
- - master
5
- cache:
6
- - bundler
7
- language: ruby
8
- rvm:
9
- - 2.3.3
10
- - 2.2.3
11
- addons:
12
- postgresql: "9.4"
13
- services:
14
- - postgresql
15
- - redis-server
16
- before_script:
17
- - psql -c 'create database que_test;' -U postgres
18
- gemfile:
19
- - gemfiles/que_0.12.2.gemfile
20
- - gemfiles/que_0.12.3.gemfile
21
- - gemfiles/rails_4.1.gemfile
22
- - gemfiles/rails_4.2.gemfile
23
- - gemfiles/shoryuken_3.1.gemfile
24
- - gemfiles/sidekiq_4.2.gemfile
25
- - gemfiles/sinatra_1.4.gemfile
26
- - gemfiles/sinatra_2.0.gemfile
27
- matrix:
28
- fast_finish: true