kiev 2.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.rspec +2 -0
  4. data/.rubocop.yml +25 -0
  5. data/.ruby-version +1 -0
  6. data/.travis.yml +27 -0
  7. data/Gemfile +5 -0
  8. data/LICENSE.md +7 -0
  9. data/README.md +461 -0
  10. data/Rakefile +18 -0
  11. data/bin/console +8 -0
  12. data/config.ru +9 -0
  13. data/gemfiles/que_0.12.2.gemfile +14 -0
  14. data/gemfiles/que_0.12.3.gemfile +15 -0
  15. data/gemfiles/rails_4.1.gemfile +13 -0
  16. data/gemfiles/rails_4.2.gemfile +13 -0
  17. data/gemfiles/sidekiq_4.2.gemfile +14 -0
  18. data/gemfiles/sinatra_1.4.gemfile +15 -0
  19. data/gemfiles/sinatra_2.0.gemfile +15 -0
  20. data/kiev.gemspec +28 -0
  21. data/lib/ext/rack/common_logger.rb +12 -0
  22. data/lib/kiev.rb +9 -0
  23. data/lib/kiev/base.rb +51 -0
  24. data/lib/kiev/base52.rb +20 -0
  25. data/lib/kiev/config.rb +164 -0
  26. data/lib/kiev/her_ext/client_request_id.rb +14 -0
  27. data/lib/kiev/httparty.rb +11 -0
  28. data/lib/kiev/json.rb +118 -0
  29. data/lib/kiev/logger.rb +122 -0
  30. data/lib/kiev/param_filter.rb +30 -0
  31. data/lib/kiev/que/job.rb +78 -0
  32. data/lib/kiev/rack.rb +20 -0
  33. data/lib/kiev/rack/request_id.rb +68 -0
  34. data/lib/kiev/rack/request_logger.rb +140 -0
  35. data/lib/kiev/rack/silence_action_dispatch_logger.rb +22 -0
  36. data/lib/kiev/rack/store_request_details.rb +21 -0
  37. data/lib/kiev/railtie.rb +55 -0
  38. data/lib/kiev/request_body_filter.rb +36 -0
  39. data/lib/kiev/request_body_filter/default.rb +11 -0
  40. data/lib/kiev/request_body_filter/form_data.rb +12 -0
  41. data/lib/kiev/request_body_filter/json.rb +14 -0
  42. data/lib/kiev/request_body_filter/xml.rb +18 -0
  43. data/lib/kiev/request_store.rb +32 -0
  44. data/lib/kiev/sidekiq.rb +41 -0
  45. data/lib/kiev/sidekiq/client_request_id.rb +12 -0
  46. data/lib/kiev/sidekiq/request_id.rb +39 -0
  47. data/lib/kiev/sidekiq/request_logger.rb +39 -0
  48. data/lib/kiev/sidekiq/request_store.rb +13 -0
  49. data/lib/kiev/sidekiq/store_request_details.rb +27 -0
  50. data/lib/kiev/subrequest_helper.rb +61 -0
  51. data/lib/kiev/util.rb +14 -0
  52. data/lib/kiev/version.rb +5 -0
  53. metadata +208 -0
@@ -0,0 +1,140 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module Rack
5
+ class RequestLogger
6
+ ERROR_STATUS = 500
7
+ ERROR_HEADERS = [].freeze
8
+ ERROR_BODY = [""].freeze
9
+
10
+ def initialize(app)
11
+ @app = app
12
+ end
13
+
14
+ def call(env)
15
+ rescued_exception = nil
16
+ began_at = Time.now.to_f
17
+
18
+ request = ::Rack::Request.new(env)
19
+
20
+ begin
21
+ status, headers, body = @app.call(env)
22
+ rescue Exception => e
23
+ rescued_exception = e
24
+
25
+ status = ERROR_STATUS
26
+ headers = ERROR_HEADERS
27
+ body = ERROR_BODY
28
+
29
+ if defined?(ActionDispatch::ExceptionWrapper)
30
+ status = ::ActionDispatch::ExceptionWrapper.status_code_for_exception(rescued_exception.class.name)
31
+ end
32
+ end
33
+
34
+ response = ::Rack::Response.new(body, status, headers)
35
+
36
+ rack_exception = log_rack_exception?(env[SINATRA_ERROR]) ? env[SINATRA_ERROR] : nil
37
+ log_exception = RequestStore.store[:error]
38
+ exception = rescued_exception || rack_exception || log_exception
39
+
40
+ if exception || Config.instance.log_request_condition.call(request, response)
41
+ Kiev.event(
42
+ :request_finished,
43
+ form_data(
44
+ began_at: began_at,
45
+ env: env,
46
+ request: request,
47
+ response: response,
48
+ status: status,
49
+ body: body,
50
+ exception: exception
51
+ )
52
+ )
53
+ end
54
+
55
+ raise rescued_exception if rescued_exception
56
+
57
+ [status, headers, body]
58
+ end
59
+
60
+ private
61
+
62
+ HTTP_USER_AGENT = "HTTP_USER_AGENT"
63
+ ACTION_REQUEST_PARAMETERS = "action_dispatch.request.request_parameters"
64
+ ACTION_QUERY_PARAMETERS = "action_dispatch.request.query_parameters"
65
+ NEW_LINE = "\n"
66
+ HTTP_X_REQUEST_START = "HTTP_X_REQUEST_START"
67
+ SINATRA_ERROR = "sinatra.error"
68
+
69
+ def log_rack_exception?(exception)
70
+ !Config.instance.ignored_rack_exceptions.include?(exception.class.name)
71
+ end
72
+
73
+ def form_data(request:, began_at:, status:, env:, body:, response:, exception:)
74
+ config = Config.instance
75
+
76
+ params =
77
+ if env[ACTION_REQUEST_PARAMETERS] && env[ACTION_QUERY_PARAMETERS]
78
+ env[ACTION_REQUEST_PARAMETERS].merge(env[ACTION_QUERY_PARAMETERS])
79
+ elsif env[ACTION_REQUEST_PARAMETERS]
80
+ env[ACTION_REQUEST_PARAMETERS]
81
+ else
82
+ request.params
83
+ end
84
+
85
+ params = ParamFilter.filter(params, config.filtered_params, config.ignored_params)
86
+
87
+ data = {
88
+ host: request.host, # env["HTTP_HOST"] || env["HTTPS_HOST"],
89
+ params: params.empty? ? nil : params, # env[Rack::QUERY_STRING],
90
+ ip: request.ip, # split_http_x_forwarded_headers(env) || env["REMOTE_ADDR"]
91
+ user_agent: env[HTTP_USER_AGENT],
92
+ status: status,
93
+ request_duration: ((Time.now.to_f - began_at) * 1000).round(3),
94
+ route: extract_route(env)
95
+ }
96
+
97
+ if env[HTTP_X_REQUEST_START]
98
+ data[:request_latency] = ((began_at - env[HTTP_X_REQUEST_START].to_f) * 1000).round(3)
99
+ end
100
+
101
+ if config.log_request_body_condition.call(request, response)
102
+ data[:request_body] =
103
+ RequestBodyFilter.filter(
104
+ request.content_type,
105
+ request.body,
106
+ config.filtered_params,
107
+ config.ignored_params
108
+ )
109
+ end
110
+
111
+ if config.log_response_body_condition.call(request, response)
112
+ # it should always respond to each, but this code is not streaming friendly
113
+ full_body = []
114
+ body.each do |str|
115
+ full_body << str
116
+ end
117
+ data[:body] = full_body.join
118
+ end
119
+
120
+ should_log_errors = config.log_request_error_condition.call(request, response)
121
+ if should_log_errors && exception.is_a?(Exception)
122
+ data[:error_class] = exception.class.name
123
+ data[:error_message] = exception.message[0..5000]
124
+ data[:error_backtrace] = Array(exception.backtrace).join(NEW_LINE)[0..5000]
125
+ end
126
+
127
+ data
128
+ end
129
+
130
+ def extract_route(env)
131
+ action_params = env["action_dispatch.request.parameters"]
132
+ if action_params
133
+ "#{action_params['controller']}##{action_params['action']}"
134
+ else
135
+ env["sinatra.route"]
136
+ end
137
+ end
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,22 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module Rack
5
+ class SilenceActionDispatchLogger
6
+ class << self
7
+ attr_accessor :disabled
8
+ end
9
+
10
+ NULL_LOGGER = ::Logger.new("/dev/null")
11
+
12
+ def initialize(app)
13
+ @app = app
14
+ end
15
+
16
+ def call(env)
17
+ env["action_dispatch.logger"] = NULL_LOGGER unless self.class.disabled
18
+ @app.call(env)
19
+ end
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,21 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module Rack
5
+ class StoreRequestDetails
6
+ def initialize(app)
7
+ @app = app
8
+ end
9
+
10
+ def call(env)
11
+ request = ::Rack::Request.new(env)
12
+ RequestStore.store[:web] = true
13
+ RequestStore.store[:request_verb] = request.request_method
14
+ RequestStore.store[:request_path] = request.path
15
+
16
+ Config.instance.pre_rack_hook.call(env)
17
+ @app.call(env)
18
+ end
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+ require "action_view/log_subscriber"
5
+ require "action_controller/log_subscriber"
6
+
7
+ module Kiev
8
+ class Railtie < Rails::Railtie
9
+ initializer("kiev.insert_middleware") do |app|
10
+ app.config.middleware.insert_after(::RequestStore::Middleware, Kiev::Rack::RequestId)
11
+ app.config.middleware.insert_after(Kiev::Rack::RequestId, Kiev::Rack::StoreRequestDetails)
12
+ app.config.middleware.insert_after(ActionDispatch::ShowExceptions, Kiev::Rack::RequestLogger)
13
+ end
14
+
15
+ if Config.instance.disable_default_logger
16
+ initializer("kiev.disable_default_logger") do |app|
17
+ app.config.middleware.delete(Rails::Rack::Logger)
18
+ app.config.middleware.insert_before(ActionDispatch::DebugExceptions, Kiev::Rack::SilenceActionDispatchLogger)
19
+ Rails.logger = Config.instance.logger
20
+ app.config.after_initialize do
21
+ Kiev::Rack::SilenceActionDispatchLogger.disabled = app.config.consider_all_requests_local
22
+ remove_existing_log_subscriptions unless Kiev::Config.instance.development_mode
23
+ end
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ def remove_existing_log_subscriptions
30
+ ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber|
31
+ case subscriber
32
+ when ActionView::LogSubscriber
33
+ unsubscribe(:action_view, subscriber)
34
+ when ActionController::LogSubscriber
35
+ unsubscribe(:action_controller, subscriber)
36
+ when defined?(ActiveRecord::LogSubscriber) && ActiveRecord::LogSubscriber
37
+ unsubscribe(:active_record, subscriber)
38
+ when defined?(SequelRails::Railties::LogSubscriber) && SequelRails::Railties::LogSubscriber
39
+ unsubscribe(:sequel, subscriber)
40
+ end
41
+ end
42
+ end
43
+
44
+ def unsubscribe(component, subscriber)
45
+ events = subscriber.public_methods(false).reject { |method| method.to_s == "call" }
46
+ events.each do |event|
47
+ ActiveSupport::Notifications.notifier.listeners_for("#{event}.#{component}").each do |listener|
48
+ if listener.instance_variable_get("@delegate") == subscriber
49
+ ActiveSupport::Notifications.unsubscribe(listener)
50
+ end
51
+ end
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "request_body_filter/default"
4
+ require_relative "request_body_filter/xml"
5
+ require_relative "request_body_filter/json"
6
+ require_relative "request_body_filter/form_data"
7
+
8
+ module Kiev
9
+ module RequestBodyFilter
10
+ FILTERED = "[FILTERED]"
11
+
12
+ JSON_CONTENT_TYPE = %w(text/json application/json)
13
+ XML_CONTENT_TYPES = %w(text/xml application/xml)
14
+ FORM_DATA_CONTENT_TYPES = %w(application/x-www-form-urlencoded multipart/form-data)
15
+
16
+ def self.for_content_type(content_type)
17
+ case content_type
18
+ when *JSON_CONTENT_TYPE
19
+ Json
20
+ when *XML_CONTENT_TYPES
21
+ Xml
22
+ when *FORM_DATA_CONTENT_TYPES
23
+ FormData
24
+ else
25
+ Default
26
+ end
27
+ end
28
+
29
+ def self.filter(content_type, request_body, filtered_params, ignored_params)
30
+ body = request_body.read
31
+ request_body.rewind
32
+ body_filter = for_content_type(content_type)
33
+ body_filter.call(body, filtered_params, ignored_params)
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module RequestBodyFilter
5
+ class Default
6
+ def self.call(request_body, _filtered_params, _ignored_params)
7
+ request_body
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module RequestBodyFilter
5
+ class FormData
6
+ def self.call(request_body, filtered_params, ignored_params)
7
+ params = ::Rack::Utils.parse_nested_query(request_body)
8
+ ParamFilter.filter(params, filtered_params, ignored_params)
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module RequestBodyFilter
5
+ class Json
6
+ def self.call(request_body, filtered_params, ignored_params)
7
+ params = ::JSON.parse(request_body)
8
+ ParamFilter.filter(params, filtered_params, ignored_params)
9
+ rescue Exception
10
+ request_body
11
+ end
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "oga"
4
+
5
+ module Kiev
6
+ module RequestBodyFilter
7
+ class Xml
8
+ def self.call(request_body, filtered_params, _ignored_params)
9
+ document = Oga.parse_xml(request_body)
10
+ filtered_params.each do |param|
11
+ sensitive_param = document.at_xpath("//#{param}/text()")
12
+ sensitive_param.text = FILTERED if sensitive_param.respond_to?(:text=)
13
+ end
14
+ document.to_xml
15
+ end
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module RequestStore
5
+ def self.store
6
+ ::RequestStore.store[:kiev] ||= {}
7
+ end
8
+
9
+ module Mixin
10
+ def wrap_request_store_13
11
+ ::RequestStore.begin!
12
+ yield
13
+ ensure
14
+ ::RequestStore.end!
15
+ ::RequestStore.clear!
16
+ end
17
+
18
+ def wrap_request_store_10
19
+ ::RequestStore.clear!
20
+ yield
21
+ ensure
22
+ ::RequestStore.clear!
23
+ end
24
+
25
+ if ::RequestStore::VERSION >= "1.3"
26
+ alias_method :wrap_request_store, :wrap_request_store_13
27
+ else
28
+ alias_method :wrap_request_store, :wrap_request_store_10
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Kiev
6
+ module Sidekiq
7
+ require_relative "sidekiq/client_request_id"
8
+ require_relative "sidekiq/request_store"
9
+ require_relative "sidekiq/request_logger"
10
+ require_relative "sidekiq/request_id"
11
+ require_relative "sidekiq/store_request_details"
12
+
13
+ class << self
14
+ def enable(base = nil)
15
+ base ||= ::Sidekiq
16
+ base.configure_client do |config|
17
+ enable_client_middleware(config)
18
+ end
19
+ base.configure_server do |config|
20
+ enable_client_middleware(config)
21
+ enable_server_middleware(config)
22
+ end
23
+ end
24
+
25
+ def enable_server_middleware(config)
26
+ config.server_middleware do |chain|
27
+ chain.prepend(Kiev::Sidekiq::RequestLogger)
28
+ chain.prepend(Kiev::Sidekiq::StoreRequestDetails)
29
+ chain.prepend(Kiev::Sidekiq::RequestId)
30
+ chain.prepend(Kiev::Sidekiq::RequestStore)
31
+ end
32
+ end
33
+
34
+ def enable_client_middleware(config)
35
+ config.client_middleware do |chain|
36
+ chain.prepend(Kiev::Sidekiq::ClientRequestId)
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module Sidekiq
5
+ class ClientRequestId
6
+ def call(_worker_class, job, _queue, _redis_pool)
7
+ job.merge!(SubrequestHelper.payload)
8
+ yield
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,39 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+
5
+ module Kiev
6
+ module Sidekiq
7
+ class RequestId
8
+ REQUEST_ID = "request_id"
9
+ REQUEST_DEPTH = "request_depth"
10
+ TREE_PATH = "tree_path"
11
+
12
+ def call(_worker, job, _queue)
13
+ Kiev::RequestStore.store[:request_id] = request_id(job)
14
+ Kiev::RequestStore.store[:request_depth] = request_depth(job)
15
+ Kiev::RequestStore.store[:tree_path] = tree_path(job)
16
+ yield
17
+ end
18
+
19
+ private
20
+
21
+ def request_id(job)
22
+ # cron jobs will be triggered without request_id
23
+ job[REQUEST_ID] || SecureRandom.uuid
24
+ end
25
+
26
+ def tree_root?(job)
27
+ !job[REQUEST_ID]
28
+ end
29
+
30
+ def request_depth(job)
31
+ tree_root?(job) ? 0 : (job[REQUEST_DEPTH].to_i + 1)
32
+ end
33
+
34
+ def tree_path(job)
35
+ tree_root?(job) ? SubrequestHelper.root_path(synchronous: false) : job[TREE_PATH]
36
+ end
37
+ end
38
+ end
39
+ end