kiev 2.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +2 -0
- data/.rubocop.yml +25 -0
- data/.ruby-version +1 -0
- data/.travis.yml +27 -0
- data/Gemfile +5 -0
- data/LICENSE.md +7 -0
- data/README.md +461 -0
- data/Rakefile +18 -0
- data/bin/console +8 -0
- data/config.ru +9 -0
- data/gemfiles/que_0.12.2.gemfile +14 -0
- data/gemfiles/que_0.12.3.gemfile +15 -0
- data/gemfiles/rails_4.1.gemfile +13 -0
- data/gemfiles/rails_4.2.gemfile +13 -0
- data/gemfiles/sidekiq_4.2.gemfile +14 -0
- data/gemfiles/sinatra_1.4.gemfile +15 -0
- data/gemfiles/sinatra_2.0.gemfile +15 -0
- data/kiev.gemspec +28 -0
- data/lib/ext/rack/common_logger.rb +12 -0
- data/lib/kiev.rb +9 -0
- data/lib/kiev/base.rb +51 -0
- data/lib/kiev/base52.rb +20 -0
- data/lib/kiev/config.rb +164 -0
- data/lib/kiev/her_ext/client_request_id.rb +14 -0
- data/lib/kiev/httparty.rb +11 -0
- data/lib/kiev/json.rb +118 -0
- data/lib/kiev/logger.rb +122 -0
- data/lib/kiev/param_filter.rb +30 -0
- data/lib/kiev/que/job.rb +78 -0
- data/lib/kiev/rack.rb +20 -0
- data/lib/kiev/rack/request_id.rb +68 -0
- data/lib/kiev/rack/request_logger.rb +140 -0
- data/lib/kiev/rack/silence_action_dispatch_logger.rb +22 -0
- data/lib/kiev/rack/store_request_details.rb +21 -0
- data/lib/kiev/railtie.rb +55 -0
- data/lib/kiev/request_body_filter.rb +36 -0
- data/lib/kiev/request_body_filter/default.rb +11 -0
- data/lib/kiev/request_body_filter/form_data.rb +12 -0
- data/lib/kiev/request_body_filter/json.rb +14 -0
- data/lib/kiev/request_body_filter/xml.rb +18 -0
- data/lib/kiev/request_store.rb +32 -0
- data/lib/kiev/sidekiq.rb +41 -0
- data/lib/kiev/sidekiq/client_request_id.rb +12 -0
- data/lib/kiev/sidekiq/request_id.rb +39 -0
- data/lib/kiev/sidekiq/request_logger.rb +39 -0
- data/lib/kiev/sidekiq/request_store.rb +13 -0
- data/lib/kiev/sidekiq/store_request_details.rb +27 -0
- data/lib/kiev/subrequest_helper.rb +61 -0
- data/lib/kiev/util.rb +14 -0
- data/lib/kiev/version.rb +5 -0
- metadata +208 -0
data/lib/kiev/json.rb
ADDED
@@ -0,0 +1,118 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "time"
|
4
|
+
|
5
|
+
module Kiev
|
6
|
+
class JSON
|
7
|
+
class << self
|
8
|
+
attr_accessor :engine
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
begin
|
14
|
+
require "oj"
|
15
|
+
Kiev::JSON.engine = :oj
|
16
|
+
rescue LoadError
|
17
|
+
require "json"
|
18
|
+
|
19
|
+
if defined?(ActiveSupport::JSON)
|
20
|
+
Kiev::JSON.engine = :activesupport
|
21
|
+
elsif defined?(::JSON)
|
22
|
+
Kiev::JSON.engine = :json
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
module Kiev
|
27
|
+
class JSON
|
28
|
+
OJ_OPTIONS_3 = {
|
29
|
+
mode: :rails,
|
30
|
+
use_as_json: true,
|
31
|
+
use_to_json: true
|
32
|
+
} # do not do freeze for Oj3 and Rails 4.1
|
33
|
+
|
34
|
+
OJ_OPTIONS_2 = {
|
35
|
+
float_precision: 16,
|
36
|
+
bigdecimal_as_decimal: false,
|
37
|
+
nan: :null,
|
38
|
+
time_format: :xmlschema,
|
39
|
+
second_precision: 3,
|
40
|
+
mode: :compat,
|
41
|
+
use_as_json: true,
|
42
|
+
use_to_json: true
|
43
|
+
}.freeze
|
44
|
+
|
45
|
+
OJ_OPTIONS = (defined?(Oj::VERSION) && Oj::VERSION >= "3") ? OJ_OPTIONS_3 : OJ_OPTIONS_2
|
46
|
+
|
47
|
+
FAIL_JSON = "{\"error_json\":\"failed to generate json\"}"
|
48
|
+
NO_JSON = "{\"error_json\":\"no json backend\"}"
|
49
|
+
|
50
|
+
class << self
|
51
|
+
def generate(obj)
|
52
|
+
if engine == :oj
|
53
|
+
oj_generate(obj)
|
54
|
+
elsif engine == :activesupport
|
55
|
+
activesupport_generate(obj)
|
56
|
+
elsif engine == :json
|
57
|
+
json_generate(obj)
|
58
|
+
else
|
59
|
+
NO_JSON.dup
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def logstash(entry)
|
64
|
+
entry.each do |key, value|
|
65
|
+
entry[key] = if value.respond_to?(:iso8601)
|
66
|
+
value.iso8601(3)
|
67
|
+
elsif !scalar?(value)
|
68
|
+
generate(value)
|
69
|
+
elsif value.is_a?(String) && value.encoding != Encoding::UTF_8
|
70
|
+
value.encode(
|
71
|
+
Encoding::UTF_8,
|
72
|
+
invalid: :replace,
|
73
|
+
undef: :replace,
|
74
|
+
replace: "?"
|
75
|
+
)
|
76
|
+
elsif value.respond_to?(:infinite?) && value.infinite?
|
77
|
+
nil
|
78
|
+
else
|
79
|
+
value
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
generate(entry) << "\n"
|
84
|
+
end
|
85
|
+
|
86
|
+
private
|
87
|
+
|
88
|
+
# Arrays excluded here because Elastic indexes very picky:
|
89
|
+
# if you have array of mixed things it will complain
|
90
|
+
def scalar?(value)
|
91
|
+
value.is_a?(String) ||
|
92
|
+
value.is_a?(Numeric) ||
|
93
|
+
value.is_a?(Symbol) ||
|
94
|
+
value.is_a?(TrueClass) ||
|
95
|
+
value.is_a?(FalseClass) ||
|
96
|
+
value.is_a?(NilClass)
|
97
|
+
end
|
98
|
+
|
99
|
+
def oj_generate(obj)
|
100
|
+
Oj.dump(obj, OJ_OPTIONS)
|
101
|
+
rescue Exception
|
102
|
+
FAIL_JSON.dup
|
103
|
+
end
|
104
|
+
|
105
|
+
def activesupport_generate(obj)
|
106
|
+
ActiveSupport::JSON.encode(obj)
|
107
|
+
rescue Exception
|
108
|
+
FAIL_JSON.dup
|
109
|
+
end
|
110
|
+
|
111
|
+
def json_generate(obj)
|
112
|
+
::JSON.generate(obj, quirks_mode: true)
|
113
|
+
rescue Exception
|
114
|
+
FAIL_JSON.dup
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
data/lib/kiev/logger.rb
ADDED
@@ -0,0 +1,122 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "logger"
|
4
|
+
require "time"
|
5
|
+
require "forwardable"
|
6
|
+
|
7
|
+
# Keep this class minimal and compatible with Ruby Logger.
|
8
|
+
# If you add custom methods to this class and they will be used by developer,
|
9
|
+
# it will be hard to swap this class with any other Logger implementation.
|
10
|
+
module Kiev
|
11
|
+
class Logger
|
12
|
+
extend Forwardable
|
13
|
+
def_delegators(*([:@logger] + ::Logger.instance_methods(false)))
|
14
|
+
|
15
|
+
DEFAULT_EVENT_NAME = "log"
|
16
|
+
|
17
|
+
FORMATTER = proc do |severity, time, event_name, data|
|
18
|
+
entry =
|
19
|
+
{
|
20
|
+
application: Config.instance.app,
|
21
|
+
event: event_name || DEFAULT_EVENT_NAME,
|
22
|
+
level: severity,
|
23
|
+
timestamp: time.utc,
|
24
|
+
request_id: RequestStore.store[:request_id],
|
25
|
+
request_depth: RequestStore.store[:request_depth],
|
26
|
+
tree_path: RequestStore.store[:tree_path]
|
27
|
+
}
|
28
|
+
|
29
|
+
# data required to restore source of log entry
|
30
|
+
if RequestStore.store[:web]
|
31
|
+
entry[:verb] = RequestStore.store[:request_verb]
|
32
|
+
entry[:path] = RequestStore.store[:request_path]
|
33
|
+
end
|
34
|
+
if RequestStore.store[:background_job]
|
35
|
+
entry[:job_name] = RequestStore.store[:job_name]
|
36
|
+
entry[:jid] = RequestStore.store[:jid]
|
37
|
+
end
|
38
|
+
|
39
|
+
if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(event_name)
|
40
|
+
entry[:tree_leaf] = true
|
41
|
+
end
|
42
|
+
|
43
|
+
if RequestStore.store[:payload]
|
44
|
+
if %i(request_finished job_finished).include?(event_name)
|
45
|
+
entry.merge!(RequestStore.store[:payload])
|
46
|
+
else
|
47
|
+
Config.instance.persistent_log_fields.each do |field|
|
48
|
+
entry[field] = RequestStore.store[:payload][field]
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
if data.is_a?(Hash)
|
54
|
+
entry.merge!(data)
|
55
|
+
elsif !data.nil?
|
56
|
+
entry[:message] = data.to_s
|
57
|
+
end
|
58
|
+
|
59
|
+
# Save some disk space
|
60
|
+
entry.reject! { |_, value| value.nil? }
|
61
|
+
|
62
|
+
JSON.logstash(entry)
|
63
|
+
end
|
64
|
+
|
65
|
+
DEVELOPMENT_FORMATTER = proc do |severity, time, event_name, data|
|
66
|
+
entry = []
|
67
|
+
|
68
|
+
entry << time.iso8601
|
69
|
+
entry << (event_name || severity).upcase
|
70
|
+
|
71
|
+
if data.is_a?(String)
|
72
|
+
entry << "#{data}\n"
|
73
|
+
end
|
74
|
+
|
75
|
+
if %i(request_finished job_finished).include?(event_name)
|
76
|
+
verb = RequestStore.store[:request_verb]
|
77
|
+
path = RequestStore.store[:request_path]
|
78
|
+
entry << "#{verb} #{path}" if verb && path
|
79
|
+
|
80
|
+
job_name = RequestStore.store[:job_name]
|
81
|
+
jid = RequestStore.store[:jid]
|
82
|
+
entry << "#{job_name} #{jid}" if job_name && jid
|
83
|
+
|
84
|
+
status = data.is_a?(Hash) ? data.delete(:status) : nil
|
85
|
+
entry << "- #{status}" if status
|
86
|
+
duration = data.is_a?(Hash) ? data.delete(:request_duration) : nil
|
87
|
+
entry << "(#{duration}ms)" if duration
|
88
|
+
entry << "\n"
|
89
|
+
|
90
|
+
meta = {
|
91
|
+
request_id: RequestStore.store[:request_id],
|
92
|
+
request_depth: RequestStore.store[:request_depth]
|
93
|
+
}.merge!(Hash(RequestStore.store[:payload]))
|
94
|
+
|
95
|
+
meta.reject! { |_, value| value.nil? }
|
96
|
+
|
97
|
+
entry << " Meta: #{meta.inspect}\n"
|
98
|
+
|
99
|
+
entry << " Params: #{data[:params].inspect}\n" if data.is_a?(Hash) && data[:params]
|
100
|
+
|
101
|
+
if data.is_a?(Hash) && data[:body]
|
102
|
+
entry << " Response: #{data[:body]}\n"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
entry.join(" ")
|
107
|
+
end
|
108
|
+
|
109
|
+
def initialize(log_path)
|
110
|
+
@logger = ::Logger.new(log_path)
|
111
|
+
end
|
112
|
+
|
113
|
+
def path=(log_path)
|
114
|
+
previous_logger = @logger
|
115
|
+
@logger = ::Logger.new(log_path)
|
116
|
+
if previous_logger
|
117
|
+
@logger.level = previous_logger.level
|
118
|
+
@logger.formatter = previous_logger.formatter
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Kiev
|
4
|
+
module ParamFilter
|
5
|
+
FILTERED = "[FILTERED]"
|
6
|
+
|
7
|
+
def self.filter(params, filtered_params, ignored_params)
|
8
|
+
params.each_with_object({}) do |(key, value), acc|
|
9
|
+
next if ignored_params.include?(key)
|
10
|
+
|
11
|
+
if defined?(ActionDispatch) && value.is_a?(ActionDispatch::Http::UploadedFile)
|
12
|
+
value = {
|
13
|
+
original_filename: value.original_filename,
|
14
|
+
content_type: value.content_type,
|
15
|
+
headers: value.headers
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
acc[key] =
|
20
|
+
if filtered_params.include?(key) && !value.is_a?(Hash)
|
21
|
+
FILTERED
|
22
|
+
elsif value.is_a?(Hash)
|
23
|
+
filter(value, filtered_params, ignored_params)
|
24
|
+
else
|
25
|
+
value
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/kiev/que/job.rb
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "../base"
|
4
|
+
|
5
|
+
module Kiev
|
6
|
+
module Que
|
7
|
+
# Original implementation https://github.com/chanks/que/blob/master/lib/que/job.rb
|
8
|
+
class Job < ::Que::Job
|
9
|
+
include Kiev::RequestStore::Mixin
|
10
|
+
|
11
|
+
def self.enqueue(*args)
|
12
|
+
if ::Que.mode == :async
|
13
|
+
super(*args.unshift(SubrequestHelper.payload))
|
14
|
+
else
|
15
|
+
super
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def _run
|
20
|
+
if ::Que.mode == :async
|
21
|
+
wrap_request_store { kiev_run }
|
22
|
+
else
|
23
|
+
kiev_run
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
NEW_LINE = "\n"
|
30
|
+
|
31
|
+
def kiev_run
|
32
|
+
args = attrs[:args]
|
33
|
+
payload = {}
|
34
|
+
|
35
|
+
if args.first.is_a?(Hash)
|
36
|
+
options = args.shift
|
37
|
+
payload = Config.instance.all_jobs_propagated_fields.map do |key|
|
38
|
+
# sometimes JSON decoder is overridden and it can be instructed to symbolize keys
|
39
|
+
[key, options.delete(key.to_s) || options.delete(key)]
|
40
|
+
end.to_h
|
41
|
+
args.unshift(options) if options.any?
|
42
|
+
end
|
43
|
+
|
44
|
+
if ::Que.mode == :async
|
45
|
+
Config.instance.jobs_propagated_fields.each do |key|
|
46
|
+
Kiev[key] = payload[key]
|
47
|
+
end
|
48
|
+
request_store = Kiev::RequestStore.store
|
49
|
+
request_store[:request_id] = payload[:request_id]
|
50
|
+
request_store[:request_depth] = payload[:request_depth].to_i + 1
|
51
|
+
request_store[:tree_path] = payload[:tree_path]
|
52
|
+
|
53
|
+
request_store[:background_job] = true
|
54
|
+
request_store[:job_name] = attrs[:job_class]
|
55
|
+
end
|
56
|
+
|
57
|
+
began_at = Time.now
|
58
|
+
|
59
|
+
::Que::Job.instance_method(:_run).bind(self).call
|
60
|
+
|
61
|
+
data = {
|
62
|
+
params: attrs[:args],
|
63
|
+
request_duration: ((Time.now - began_at) * 1000).round(3)
|
64
|
+
}
|
65
|
+
|
66
|
+
error ||= _error
|
67
|
+
|
68
|
+
if error
|
69
|
+
data[:error_class] = error.class.name
|
70
|
+
data[:error_message] = error.message[0..5000]
|
71
|
+
data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000]
|
72
|
+
end
|
73
|
+
|
74
|
+
Kiev.event(:job_finished, data)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
data/lib/kiev/rack.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "base"
|
4
|
+
require_relative "rack/request_logger"
|
5
|
+
require_relative "rack/request_id"
|
6
|
+
require_relative "rack/store_request_details"
|
7
|
+
require_relative "rack/silence_action_dispatch_logger"
|
8
|
+
require_relative "../ext/rack/common_logger"
|
9
|
+
|
10
|
+
module Kiev
|
11
|
+
module Rack
|
12
|
+
def self.included(base)
|
13
|
+
# The order is important
|
14
|
+
base.use(::RequestStore::Middleware)
|
15
|
+
base.use(Kiev::Rack::RequestLogger)
|
16
|
+
base.use(Kiev::Rack::StoreRequestDetails)
|
17
|
+
base.use(Kiev::Rack::RequestId)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
|
5
|
+
module Kiev
|
6
|
+
module Rack
|
7
|
+
class RequestId
|
8
|
+
# for Rails 4
|
9
|
+
RAILS_REQUEST_ID = "action_dispatch.request_id"
|
10
|
+
|
11
|
+
def initialize(app)
|
12
|
+
@app = app
|
13
|
+
end
|
14
|
+
|
15
|
+
def call(env)
|
16
|
+
request_id_header_out = to_rack(:request_id)
|
17
|
+
request_id_header_in = to_http(:request_id)
|
18
|
+
|
19
|
+
request_id = make_request_id(env[RAILS_REQUEST_ID] || env[request_id_header_in])
|
20
|
+
RequestStore.store[:request_id] = request_id
|
21
|
+
RequestStore.store[:request_depth] = request_depth(env)
|
22
|
+
RequestStore.store[:tree_path] = tree_path(env)
|
23
|
+
|
24
|
+
@app.call(env).tap { |_status, headers, _body| headers[request_id_header_out] = request_id }
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
# TODO: in Rails 5 they set `headers[X_REQUEST_ID]`, so this will not work
|
30
|
+
# https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/request_id.rb
|
31
|
+
# https://github.com/interagent/pliny/blob/master/lib/pliny/middleware/request_id.rb
|
32
|
+
def tree_root?(env)
|
33
|
+
request_id_header_in = to_http(:request_id)
|
34
|
+
!env[request_id_header_in]
|
35
|
+
end
|
36
|
+
|
37
|
+
def request_depth(env)
|
38
|
+
request_depth_header = to_http(:request_depth)
|
39
|
+
tree_root?(env) ? 0 : (env[request_depth_header].to_i + 1)
|
40
|
+
end
|
41
|
+
|
42
|
+
def tree_path(env)
|
43
|
+
tree_path_header = to_http(:tree_path)
|
44
|
+
tree_root?(env) ? SubrequestHelper.root_path(synchronous: true) : Util.sanitize(env[tree_path_header])
|
45
|
+
end
|
46
|
+
|
47
|
+
def to_http(value)
|
48
|
+
Util.to_http(to_rack(value))
|
49
|
+
end
|
50
|
+
|
51
|
+
def to_rack(value)
|
52
|
+
Config.instance.all_http_propagated_fields[value]
|
53
|
+
end
|
54
|
+
|
55
|
+
def make_request_id(request_id)
|
56
|
+
if request_id.nil? || request_id.empty?
|
57
|
+
internal_request_id
|
58
|
+
else
|
59
|
+
Util.sanitize(request_id)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def internal_request_id
|
64
|
+
SecureRandom.uuid
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|