kiev 2.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +2 -0
- data/.rubocop.yml +25 -0
- data/.ruby-version +1 -0
- data/.travis.yml +27 -0
- data/Gemfile +5 -0
- data/LICENSE.md +7 -0
- data/README.md +461 -0
- data/Rakefile +18 -0
- data/bin/console +8 -0
- data/config.ru +9 -0
- data/gemfiles/que_0.12.2.gemfile +14 -0
- data/gemfiles/que_0.12.3.gemfile +15 -0
- data/gemfiles/rails_4.1.gemfile +13 -0
- data/gemfiles/rails_4.2.gemfile +13 -0
- data/gemfiles/sidekiq_4.2.gemfile +14 -0
- data/gemfiles/sinatra_1.4.gemfile +15 -0
- data/gemfiles/sinatra_2.0.gemfile +15 -0
- data/kiev.gemspec +28 -0
- data/lib/ext/rack/common_logger.rb +12 -0
- data/lib/kiev.rb +9 -0
- data/lib/kiev/base.rb +51 -0
- data/lib/kiev/base52.rb +20 -0
- data/lib/kiev/config.rb +164 -0
- data/lib/kiev/her_ext/client_request_id.rb +14 -0
- data/lib/kiev/httparty.rb +11 -0
- data/lib/kiev/json.rb +118 -0
- data/lib/kiev/logger.rb +122 -0
- data/lib/kiev/param_filter.rb +30 -0
- data/lib/kiev/que/job.rb +78 -0
- data/lib/kiev/rack.rb +20 -0
- data/lib/kiev/rack/request_id.rb +68 -0
- data/lib/kiev/rack/request_logger.rb +140 -0
- data/lib/kiev/rack/silence_action_dispatch_logger.rb +22 -0
- data/lib/kiev/rack/store_request_details.rb +21 -0
- data/lib/kiev/railtie.rb +55 -0
- data/lib/kiev/request_body_filter.rb +36 -0
- data/lib/kiev/request_body_filter/default.rb +11 -0
- data/lib/kiev/request_body_filter/form_data.rb +12 -0
- data/lib/kiev/request_body_filter/json.rb +14 -0
- data/lib/kiev/request_body_filter/xml.rb +18 -0
- data/lib/kiev/request_store.rb +32 -0
- data/lib/kiev/sidekiq.rb +41 -0
- data/lib/kiev/sidekiq/client_request_id.rb +12 -0
- data/lib/kiev/sidekiq/request_id.rb +39 -0
- data/lib/kiev/sidekiq/request_logger.rb +39 -0
- data/lib/kiev/sidekiq/request_store.rb +13 -0
- data/lib/kiev/sidekiq/store_request_details.rb +27 -0
- data/lib/kiev/subrequest_helper.rb +61 -0
- data/lib/kiev/util.rb +14 -0
- data/lib/kiev/version.rb +5 -0
- metadata +208 -0
@@ -0,0 +1,140 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiev
|
4
|
+
module Rack
|
5
|
+
class RequestLogger
|
6
|
+
ERROR_STATUS = 500
|
7
|
+
ERROR_HEADERS = [].freeze
|
8
|
+
ERROR_BODY = [""].freeze
|
9
|
+
|
10
|
+
def initialize(app)
|
11
|
+
@app = app
|
12
|
+
end
|
13
|
+
|
14
|
+
def call(env)
|
15
|
+
rescued_exception = nil
|
16
|
+
began_at = Time.now.to_f
|
17
|
+
|
18
|
+
request = ::Rack::Request.new(env)
|
19
|
+
|
20
|
+
begin
|
21
|
+
status, headers, body = @app.call(env)
|
22
|
+
rescue Exception => e
|
23
|
+
rescued_exception = e
|
24
|
+
|
25
|
+
status = ERROR_STATUS
|
26
|
+
headers = ERROR_HEADERS
|
27
|
+
body = ERROR_BODY
|
28
|
+
|
29
|
+
if defined?(ActionDispatch::ExceptionWrapper)
|
30
|
+
status = ::ActionDispatch::ExceptionWrapper.status_code_for_exception(rescued_exception.class.name)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
response = ::Rack::Response.new(body, status, headers)
|
35
|
+
|
36
|
+
rack_exception = log_rack_exception?(env[SINATRA_ERROR]) ? env[SINATRA_ERROR] : nil
|
37
|
+
log_exception = RequestStore.store[:error]
|
38
|
+
exception = rescued_exception || rack_exception || log_exception
|
39
|
+
|
40
|
+
if exception || Config.instance.log_request_condition.call(request, response)
|
41
|
+
Kiev.event(
|
42
|
+
:request_finished,
|
43
|
+
form_data(
|
44
|
+
began_at: began_at,
|
45
|
+
env: env,
|
46
|
+
request: request,
|
47
|
+
response: response,
|
48
|
+
status: status,
|
49
|
+
body: body,
|
50
|
+
exception: exception
|
51
|
+
)
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
raise rescued_exception if rescued_exception
|
56
|
+
|
57
|
+
[status, headers, body]
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
HTTP_USER_AGENT = "HTTP_USER_AGENT"
|
63
|
+
ACTION_REQUEST_PARAMETERS = "action_dispatch.request.request_parameters"
|
64
|
+
ACTION_QUERY_PARAMETERS = "action_dispatch.request.query_parameters"
|
65
|
+
NEW_LINE = "\n"
|
66
|
+
HTTP_X_REQUEST_START = "HTTP_X_REQUEST_START"
|
67
|
+
SINATRA_ERROR = "sinatra.error"
|
68
|
+
|
69
|
+
def log_rack_exception?(exception)
|
70
|
+
!Config.instance.ignored_rack_exceptions.include?(exception.class.name)
|
71
|
+
end
|
72
|
+
|
73
|
+
def form_data(request:, began_at:, status:, env:, body:, response:, exception:)
|
74
|
+
config = Config.instance
|
75
|
+
|
76
|
+
params =
|
77
|
+
if env[ACTION_REQUEST_PARAMETERS] && env[ACTION_QUERY_PARAMETERS]
|
78
|
+
env[ACTION_REQUEST_PARAMETERS].merge(env[ACTION_QUERY_PARAMETERS])
|
79
|
+
elsif env[ACTION_REQUEST_PARAMETERS]
|
80
|
+
env[ACTION_REQUEST_PARAMETERS]
|
81
|
+
else
|
82
|
+
request.params
|
83
|
+
end
|
84
|
+
|
85
|
+
params = ParamFilter.filter(params, config.filtered_params, config.ignored_params)
|
86
|
+
|
87
|
+
data = {
|
88
|
+
host: request.host, # env["HTTP_HOST"] || env["HTTPS_HOST"],
|
89
|
+
params: params.empty? ? nil : params, # env[Rack::QUERY_STRING],
|
90
|
+
ip: request.ip, # split_http_x_forwarded_headers(env) || env["REMOTE_ADDR"]
|
91
|
+
user_agent: env[HTTP_USER_AGENT],
|
92
|
+
status: status,
|
93
|
+
request_duration: ((Time.now.to_f - began_at) * 1000).round(3),
|
94
|
+
route: extract_route(env)
|
95
|
+
}
|
96
|
+
|
97
|
+
if env[HTTP_X_REQUEST_START]
|
98
|
+
data[:request_latency] = ((began_at - env[HTTP_X_REQUEST_START].to_f) * 1000).round(3)
|
99
|
+
end
|
100
|
+
|
101
|
+
if config.log_request_body_condition.call(request, response)
|
102
|
+
data[:request_body] =
|
103
|
+
RequestBodyFilter.filter(
|
104
|
+
request.content_type,
|
105
|
+
request.body,
|
106
|
+
config.filtered_params,
|
107
|
+
config.ignored_params
|
108
|
+
)
|
109
|
+
end
|
110
|
+
|
111
|
+
if config.log_response_body_condition.call(request, response)
|
112
|
+
# it should always respond to each, but this code is not streaming friendly
|
113
|
+
full_body = []
|
114
|
+
body.each do |str|
|
115
|
+
full_body << str
|
116
|
+
end
|
117
|
+
data[:body] = full_body.join
|
118
|
+
end
|
119
|
+
|
120
|
+
should_log_errors = config.log_request_error_condition.call(request, response)
|
121
|
+
if should_log_errors && exception.is_a?(Exception)
|
122
|
+
data[:error_class] = exception.class.name
|
123
|
+
data[:error_message] = exception.message[0..5000]
|
124
|
+
data[:error_backtrace] = Array(exception.backtrace).join(NEW_LINE)[0..5000]
|
125
|
+
end
|
126
|
+
|
127
|
+
data
|
128
|
+
end
|
129
|
+
|
130
|
+
def extract_route(env)
|
131
|
+
action_params = env["action_dispatch.request.parameters"]
|
132
|
+
if action_params
|
133
|
+
"#{action_params['controller']}##{action_params['action']}"
|
134
|
+
else
|
135
|
+
env["sinatra.route"]
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiev
|
4
|
+
module Rack
|
5
|
+
class SilenceActionDispatchLogger
|
6
|
+
class << self
|
7
|
+
attr_accessor :disabled
|
8
|
+
end
|
9
|
+
|
10
|
+
NULL_LOGGER = ::Logger.new("/dev/null")
|
11
|
+
|
12
|
+
def initialize(app)
|
13
|
+
@app = app
|
14
|
+
end
|
15
|
+
|
16
|
+
def call(env)
|
17
|
+
env["action_dispatch.logger"] = NULL_LOGGER unless self.class.disabled
|
18
|
+
@app.call(env)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiev
|
4
|
+
module Rack
|
5
|
+
class StoreRequestDetails
|
6
|
+
def initialize(app)
|
7
|
+
@app = app
|
8
|
+
end
|
9
|
+
|
10
|
+
def call(env)
|
11
|
+
request = ::Rack::Request.new(env)
|
12
|
+
RequestStore.store[:web] = true
|
13
|
+
RequestStore.store[:request_verb] = request.request_method
|
14
|
+
RequestStore.store[:request_path] = request.path
|
15
|
+
|
16
|
+
Config.instance.pre_rack_hook.call(env)
|
17
|
+
@app.call(env)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/kiev/railtie.rb
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "base"
|
4
|
+
require "action_view/log_subscriber"
|
5
|
+
require "action_controller/log_subscriber"
|
6
|
+
|
7
|
+
module Kiev
|
8
|
+
class Railtie < Rails::Railtie
|
9
|
+
initializer("kiev.insert_middleware") do |app|
|
10
|
+
app.config.middleware.insert_after(::RequestStore::Middleware, Kiev::Rack::RequestId)
|
11
|
+
app.config.middleware.insert_after(Kiev::Rack::RequestId, Kiev::Rack::StoreRequestDetails)
|
12
|
+
app.config.middleware.insert_after(ActionDispatch::ShowExceptions, Kiev::Rack::RequestLogger)
|
13
|
+
end
|
14
|
+
|
15
|
+
if Config.instance.disable_default_logger
|
16
|
+
initializer("kiev.disable_default_logger") do |app|
|
17
|
+
app.config.middleware.delete(Rails::Rack::Logger)
|
18
|
+
app.config.middleware.insert_before(ActionDispatch::DebugExceptions, Kiev::Rack::SilenceActionDispatchLogger)
|
19
|
+
Rails.logger = Config.instance.logger
|
20
|
+
app.config.after_initialize do
|
21
|
+
Kiev::Rack::SilenceActionDispatchLogger.disabled = app.config.consider_all_requests_local
|
22
|
+
remove_existing_log_subscriptions unless Kiev::Config.instance.development_mode
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def remove_existing_log_subscriptions
|
30
|
+
ActiveSupport::LogSubscriber.log_subscribers.each do |subscriber|
|
31
|
+
case subscriber
|
32
|
+
when ActionView::LogSubscriber
|
33
|
+
unsubscribe(:action_view, subscriber)
|
34
|
+
when ActionController::LogSubscriber
|
35
|
+
unsubscribe(:action_controller, subscriber)
|
36
|
+
when defined?(ActiveRecord::LogSubscriber) && ActiveRecord::LogSubscriber
|
37
|
+
unsubscribe(:active_record, subscriber)
|
38
|
+
when defined?(SequelRails::Railties::LogSubscriber) && SequelRails::Railties::LogSubscriber
|
39
|
+
unsubscribe(:sequel, subscriber)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def unsubscribe(component, subscriber)
|
45
|
+
events = subscriber.public_methods(false).reject { |method| method.to_s == "call" }
|
46
|
+
events.each do |event|
|
47
|
+
ActiveSupport::Notifications.notifier.listeners_for("#{event}.#{component}").each do |listener|
|
48
|
+
if listener.instance_variable_get("@delegate") == subscriber
|
49
|
+
ActiveSupport::Notifications.unsubscribe(listener)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "request_body_filter/default"
|
4
|
+
require_relative "request_body_filter/xml"
|
5
|
+
require_relative "request_body_filter/json"
|
6
|
+
require_relative "request_body_filter/form_data"
|
7
|
+
|
8
|
+
module Kiev
|
9
|
+
module RequestBodyFilter
|
10
|
+
FILTERED = "[FILTERED]"
|
11
|
+
|
12
|
+
JSON_CONTENT_TYPE = %w(text/json application/json)
|
13
|
+
XML_CONTENT_TYPES = %w(text/xml application/xml)
|
14
|
+
FORM_DATA_CONTENT_TYPES = %w(application/x-www-form-urlencoded multipart/form-data)
|
15
|
+
|
16
|
+
def self.for_content_type(content_type)
|
17
|
+
case content_type
|
18
|
+
when *JSON_CONTENT_TYPE
|
19
|
+
Json
|
20
|
+
when *XML_CONTENT_TYPES
|
21
|
+
Xml
|
22
|
+
when *FORM_DATA_CONTENT_TYPES
|
23
|
+
FormData
|
24
|
+
else
|
25
|
+
Default
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.filter(content_type, request_body, filtered_params, ignored_params)
|
30
|
+
body = request_body.read
|
31
|
+
request_body.rewind
|
32
|
+
body_filter = for_content_type(content_type)
|
33
|
+
body_filter.call(body, filtered_params, ignored_params)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiev
|
4
|
+
module RequestBodyFilter
|
5
|
+
class FormData
|
6
|
+
def self.call(request_body, filtered_params, ignored_params)
|
7
|
+
params = ::Rack::Utils.parse_nested_query(request_body)
|
8
|
+
ParamFilter.filter(params, filtered_params, ignored_params)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiev
|
4
|
+
module RequestBodyFilter
|
5
|
+
class Json
|
6
|
+
def self.call(request_body, filtered_params, ignored_params)
|
7
|
+
params = ::JSON.parse(request_body)
|
8
|
+
ParamFilter.filter(params, filtered_params, ignored_params)
|
9
|
+
rescue Exception
|
10
|
+
request_body
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "oga"
|
4
|
+
|
5
|
+
module Kiev
|
6
|
+
module RequestBodyFilter
|
7
|
+
class Xml
|
8
|
+
def self.call(request_body, filtered_params, _ignored_params)
|
9
|
+
document = Oga.parse_xml(request_body)
|
10
|
+
filtered_params.each do |param|
|
11
|
+
sensitive_param = document.at_xpath("//#{param}/text()")
|
12
|
+
sensitive_param.text = FILTERED if sensitive_param.respond_to?(:text=)
|
13
|
+
end
|
14
|
+
document.to_xml
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiev
|
4
|
+
module RequestStore
|
5
|
+
def self.store
|
6
|
+
::RequestStore.store[:kiev] ||= {}
|
7
|
+
end
|
8
|
+
|
9
|
+
module Mixin
|
10
|
+
def wrap_request_store_13
|
11
|
+
::RequestStore.begin!
|
12
|
+
yield
|
13
|
+
ensure
|
14
|
+
::RequestStore.end!
|
15
|
+
::RequestStore.clear!
|
16
|
+
end
|
17
|
+
|
18
|
+
def wrap_request_store_10
|
19
|
+
::RequestStore.clear!
|
20
|
+
yield
|
21
|
+
ensure
|
22
|
+
::RequestStore.clear!
|
23
|
+
end
|
24
|
+
|
25
|
+
if ::RequestStore::VERSION >= "1.3"
|
26
|
+
alias_method :wrap_request_store, :wrap_request_store_13
|
27
|
+
else
|
28
|
+
alias_method :wrap_request_store, :wrap_request_store_10
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
data/lib/kiev/sidekiq.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "base"
|
4
|
+
|
5
|
+
module Kiev
|
6
|
+
module Sidekiq
|
7
|
+
require_relative "sidekiq/client_request_id"
|
8
|
+
require_relative "sidekiq/request_store"
|
9
|
+
require_relative "sidekiq/request_logger"
|
10
|
+
require_relative "sidekiq/request_id"
|
11
|
+
require_relative "sidekiq/store_request_details"
|
12
|
+
|
13
|
+
class << self
|
14
|
+
def enable(base = nil)
|
15
|
+
base ||= ::Sidekiq
|
16
|
+
base.configure_client do |config|
|
17
|
+
enable_client_middleware(config)
|
18
|
+
end
|
19
|
+
base.configure_server do |config|
|
20
|
+
enable_client_middleware(config)
|
21
|
+
enable_server_middleware(config)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def enable_server_middleware(config)
|
26
|
+
config.server_middleware do |chain|
|
27
|
+
chain.prepend(Kiev::Sidekiq::RequestLogger)
|
28
|
+
chain.prepend(Kiev::Sidekiq::StoreRequestDetails)
|
29
|
+
chain.prepend(Kiev::Sidekiq::RequestId)
|
30
|
+
chain.prepend(Kiev::Sidekiq::RequestStore)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def enable_client_middleware(config)
|
35
|
+
config.client_middleware do |chain|
|
36
|
+
chain.prepend(Kiev::Sidekiq::ClientRequestId)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
|
5
|
+
module Kiev
|
6
|
+
module Sidekiq
|
7
|
+
class RequestId
|
8
|
+
REQUEST_ID = "request_id"
|
9
|
+
REQUEST_DEPTH = "request_depth"
|
10
|
+
TREE_PATH = "tree_path"
|
11
|
+
|
12
|
+
def call(_worker, job, _queue)
|
13
|
+
Kiev::RequestStore.store[:request_id] = request_id(job)
|
14
|
+
Kiev::RequestStore.store[:request_depth] = request_depth(job)
|
15
|
+
Kiev::RequestStore.store[:tree_path] = tree_path(job)
|
16
|
+
yield
|
17
|
+
end
|
18
|
+
|
19
|
+
private
|
20
|
+
|
21
|
+
def request_id(job)
|
22
|
+
# cron jobs will be triggered without request_id
|
23
|
+
job[REQUEST_ID] || SecureRandom.uuid
|
24
|
+
end
|
25
|
+
|
26
|
+
def tree_root?(job)
|
27
|
+
!job[REQUEST_ID]
|
28
|
+
end
|
29
|
+
|
30
|
+
def request_depth(job)
|
31
|
+
tree_root?(job) ? 0 : (job[REQUEST_DEPTH].to_i + 1)
|
32
|
+
end
|
33
|
+
|
34
|
+
def tree_path(job)
|
35
|
+
tree_root?(job) ? SubrequestHelper.root_path(synchronous: false) : job[TREE_PATH]
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|