kiev 2.7.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.rspec +2 -0
  4. data/.rubocop.yml +25 -0
  5. data/.ruby-version +1 -0
  6. data/.travis.yml +27 -0
  7. data/Gemfile +5 -0
  8. data/LICENSE.md +7 -0
  9. data/README.md +461 -0
  10. data/Rakefile +18 -0
  11. data/bin/console +8 -0
  12. data/config.ru +9 -0
  13. data/gemfiles/que_0.12.2.gemfile +14 -0
  14. data/gemfiles/que_0.12.3.gemfile +15 -0
  15. data/gemfiles/rails_4.1.gemfile +13 -0
  16. data/gemfiles/rails_4.2.gemfile +13 -0
  17. data/gemfiles/sidekiq_4.2.gemfile +14 -0
  18. data/gemfiles/sinatra_1.4.gemfile +15 -0
  19. data/gemfiles/sinatra_2.0.gemfile +15 -0
  20. data/kiev.gemspec +28 -0
  21. data/lib/ext/rack/common_logger.rb +12 -0
  22. data/lib/kiev.rb +9 -0
  23. data/lib/kiev/base.rb +51 -0
  24. data/lib/kiev/base52.rb +20 -0
  25. data/lib/kiev/config.rb +164 -0
  26. data/lib/kiev/her_ext/client_request_id.rb +14 -0
  27. data/lib/kiev/httparty.rb +11 -0
  28. data/lib/kiev/json.rb +118 -0
  29. data/lib/kiev/logger.rb +122 -0
  30. data/lib/kiev/param_filter.rb +30 -0
  31. data/lib/kiev/que/job.rb +78 -0
  32. data/lib/kiev/rack.rb +20 -0
  33. data/lib/kiev/rack/request_id.rb +68 -0
  34. data/lib/kiev/rack/request_logger.rb +140 -0
  35. data/lib/kiev/rack/silence_action_dispatch_logger.rb +22 -0
  36. data/lib/kiev/rack/store_request_details.rb +21 -0
  37. data/lib/kiev/railtie.rb +55 -0
  38. data/lib/kiev/request_body_filter.rb +36 -0
  39. data/lib/kiev/request_body_filter/default.rb +11 -0
  40. data/lib/kiev/request_body_filter/form_data.rb +12 -0
  41. data/lib/kiev/request_body_filter/json.rb +14 -0
  42. data/lib/kiev/request_body_filter/xml.rb +18 -0
  43. data/lib/kiev/request_store.rb +32 -0
  44. data/lib/kiev/sidekiq.rb +41 -0
  45. data/lib/kiev/sidekiq/client_request_id.rb +12 -0
  46. data/lib/kiev/sidekiq/request_id.rb +39 -0
  47. data/lib/kiev/sidekiq/request_logger.rb +39 -0
  48. data/lib/kiev/sidekiq/request_store.rb +13 -0
  49. data/lib/kiev/sidekiq/store_request_details.rb +27 -0
  50. data/lib/kiev/subrequest_helper.rb +61 -0
  51. data/lib/kiev/util.rb +14 -0
  52. data/lib/kiev/version.rb +5 -0
  53. metadata +208 -0
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+
5
+ module Kiev
6
+ module HTTParty
7
+ def self.headers
8
+ SubrequestHelper.headers
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,118 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "time"
4
+
5
+ module Kiev
6
+ class JSON
7
+ class << self
8
+ attr_accessor :engine
9
+ end
10
+ end
11
+ end
12
+
13
+ begin
14
+ require "oj"
15
+ Kiev::JSON.engine = :oj
16
+ rescue LoadError
17
+ require "json"
18
+
19
+ if defined?(ActiveSupport::JSON)
20
+ Kiev::JSON.engine = :activesupport
21
+ elsif defined?(::JSON)
22
+ Kiev::JSON.engine = :json
23
+ end
24
+ end
25
+
26
+ module Kiev
27
+ class JSON
28
+ OJ_OPTIONS_3 = {
29
+ mode: :rails,
30
+ use_as_json: true,
31
+ use_to_json: true
32
+ } # do not do freeze for Oj3 and Rails 4.1
33
+
34
+ OJ_OPTIONS_2 = {
35
+ float_precision: 16,
36
+ bigdecimal_as_decimal: false,
37
+ nan: :null,
38
+ time_format: :xmlschema,
39
+ second_precision: 3,
40
+ mode: :compat,
41
+ use_as_json: true,
42
+ use_to_json: true
43
+ }.freeze
44
+
45
+ OJ_OPTIONS = (defined?(Oj::VERSION) && Oj::VERSION >= "3") ? OJ_OPTIONS_3 : OJ_OPTIONS_2
46
+
47
+ FAIL_JSON = "{\"error_json\":\"failed to generate json\"}"
48
+ NO_JSON = "{\"error_json\":\"no json backend\"}"
49
+
50
+ class << self
51
+ def generate(obj)
52
+ if engine == :oj
53
+ oj_generate(obj)
54
+ elsif engine == :activesupport
55
+ activesupport_generate(obj)
56
+ elsif engine == :json
57
+ json_generate(obj)
58
+ else
59
+ NO_JSON.dup
60
+ end
61
+ end
62
+
63
+ def logstash(entry)
64
+ entry.each do |key, value|
65
+ entry[key] = if value.respond_to?(:iso8601)
66
+ value.iso8601(3)
67
+ elsif !scalar?(value)
68
+ generate(value)
69
+ elsif value.is_a?(String) && value.encoding != Encoding::UTF_8
70
+ value.encode(
71
+ Encoding::UTF_8,
72
+ invalid: :replace,
73
+ undef: :replace,
74
+ replace: "?"
75
+ )
76
+ elsif value.respond_to?(:infinite?) && value.infinite?
77
+ nil
78
+ else
79
+ value
80
+ end
81
+ end
82
+
83
+ generate(entry) << "\n"
84
+ end
85
+
86
+ private
87
+
88
+ # Arrays excluded here because Elastic indexes very picky:
89
+ # if you have array of mixed things it will complain
90
+ def scalar?(value)
91
+ value.is_a?(String) ||
92
+ value.is_a?(Numeric) ||
93
+ value.is_a?(Symbol) ||
94
+ value.is_a?(TrueClass) ||
95
+ value.is_a?(FalseClass) ||
96
+ value.is_a?(NilClass)
97
+ end
98
+
99
+ def oj_generate(obj)
100
+ Oj.dump(obj, OJ_OPTIONS)
101
+ rescue Exception
102
+ FAIL_JSON.dup
103
+ end
104
+
105
+ def activesupport_generate(obj)
106
+ ActiveSupport::JSON.encode(obj)
107
+ rescue Exception
108
+ FAIL_JSON.dup
109
+ end
110
+
111
+ def json_generate(obj)
112
+ ::JSON.generate(obj, quirks_mode: true)
113
+ rescue Exception
114
+ FAIL_JSON.dup
115
+ end
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "logger"
4
+ require "time"
5
+ require "forwardable"
6
+
7
+ # Keep this class minimal and compatible with Ruby Logger.
8
+ # If you add custom methods to this class and they will be used by developer,
9
+ # it will be hard to swap this class with any other Logger implementation.
10
+ module Kiev
11
+ class Logger
12
+ extend Forwardable
13
+ def_delegators(*([:@logger] + ::Logger.instance_methods(false)))
14
+
15
+ DEFAULT_EVENT_NAME = "log"
16
+
17
+ FORMATTER = proc do |severity, time, event_name, data|
18
+ entry =
19
+ {
20
+ application: Config.instance.app,
21
+ event: event_name || DEFAULT_EVENT_NAME,
22
+ level: severity,
23
+ timestamp: time.utc,
24
+ request_id: RequestStore.store[:request_id],
25
+ request_depth: RequestStore.store[:request_depth],
26
+ tree_path: RequestStore.store[:tree_path]
27
+ }
28
+
29
+ # data required to restore source of log entry
30
+ if RequestStore.store[:web]
31
+ entry[:verb] = RequestStore.store[:request_verb]
32
+ entry[:path] = RequestStore.store[:request_path]
33
+ end
34
+ if RequestStore.store[:background_job]
35
+ entry[:job_name] = RequestStore.store[:job_name]
36
+ entry[:jid] = RequestStore.store[:jid]
37
+ end
38
+
39
+ if !RequestStore.store[:subrequest_count] && %i(request_finished job_finished).include?(event_name)
40
+ entry[:tree_leaf] = true
41
+ end
42
+
43
+ if RequestStore.store[:payload]
44
+ if %i(request_finished job_finished).include?(event_name)
45
+ entry.merge!(RequestStore.store[:payload])
46
+ else
47
+ Config.instance.persistent_log_fields.each do |field|
48
+ entry[field] = RequestStore.store[:payload][field]
49
+ end
50
+ end
51
+ end
52
+
53
+ if data.is_a?(Hash)
54
+ entry.merge!(data)
55
+ elsif !data.nil?
56
+ entry[:message] = data.to_s
57
+ end
58
+
59
+ # Save some disk space
60
+ entry.reject! { |_, value| value.nil? }
61
+
62
+ JSON.logstash(entry)
63
+ end
64
+
65
+ DEVELOPMENT_FORMATTER = proc do |severity, time, event_name, data|
66
+ entry = []
67
+
68
+ entry << time.iso8601
69
+ entry << (event_name || severity).upcase
70
+
71
+ if data.is_a?(String)
72
+ entry << "#{data}\n"
73
+ end
74
+
75
+ if %i(request_finished job_finished).include?(event_name)
76
+ verb = RequestStore.store[:request_verb]
77
+ path = RequestStore.store[:request_path]
78
+ entry << "#{verb} #{path}" if verb && path
79
+
80
+ job_name = RequestStore.store[:job_name]
81
+ jid = RequestStore.store[:jid]
82
+ entry << "#{job_name} #{jid}" if job_name && jid
83
+
84
+ status = data.is_a?(Hash) ? data.delete(:status) : nil
85
+ entry << "- #{status}" if status
86
+ duration = data.is_a?(Hash) ? data.delete(:request_duration) : nil
87
+ entry << "(#{duration}ms)" if duration
88
+ entry << "\n"
89
+
90
+ meta = {
91
+ request_id: RequestStore.store[:request_id],
92
+ request_depth: RequestStore.store[:request_depth]
93
+ }.merge!(Hash(RequestStore.store[:payload]))
94
+
95
+ meta.reject! { |_, value| value.nil? }
96
+
97
+ entry << " Meta: #{meta.inspect}\n"
98
+
99
+ entry << " Params: #{data[:params].inspect}\n" if data.is_a?(Hash) && data[:params]
100
+
101
+ if data.is_a?(Hash) && data[:body]
102
+ entry << " Response: #{data[:body]}\n"
103
+ end
104
+ end
105
+
106
+ entry.join(" ")
107
+ end
108
+
109
+ def initialize(log_path)
110
+ @logger = ::Logger.new(log_path)
111
+ end
112
+
113
+ def path=(log_path)
114
+ previous_logger = @logger
115
+ @logger = ::Logger.new(log_path)
116
+ if previous_logger
117
+ @logger.level = previous_logger.level
118
+ @logger.formatter = previous_logger.formatter
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kiev
4
+ module ParamFilter
5
+ FILTERED = "[FILTERED]"
6
+
7
+ def self.filter(params, filtered_params, ignored_params)
8
+ params.each_with_object({}) do |(key, value), acc|
9
+ next if ignored_params.include?(key)
10
+
11
+ if defined?(ActionDispatch) && value.is_a?(ActionDispatch::Http::UploadedFile)
12
+ value = {
13
+ original_filename: value.original_filename,
14
+ content_type: value.content_type,
15
+ headers: value.headers
16
+ }
17
+ end
18
+
19
+ acc[key] =
20
+ if filtered_params.include?(key) && !value.is_a?(Hash)
21
+ FILTERED
22
+ elsif value.is_a?(Hash)
23
+ filter(value, filtered_params, ignored_params)
24
+ else
25
+ value
26
+ end
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,78 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../base"
4
+
5
+ module Kiev
6
+ module Que
7
+ # Original implementation https://github.com/chanks/que/blob/master/lib/que/job.rb
8
+ class Job < ::Que::Job
9
+ include Kiev::RequestStore::Mixin
10
+
11
+ def self.enqueue(*args)
12
+ if ::Que.mode == :async
13
+ super(*args.unshift(SubrequestHelper.payload))
14
+ else
15
+ super
16
+ end
17
+ end
18
+
19
+ def _run
20
+ if ::Que.mode == :async
21
+ wrap_request_store { kiev_run }
22
+ else
23
+ kiev_run
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ NEW_LINE = "\n"
30
+
31
+ def kiev_run
32
+ args = attrs[:args]
33
+ payload = {}
34
+
35
+ if args.first.is_a?(Hash)
36
+ options = args.shift
37
+ payload = Config.instance.all_jobs_propagated_fields.map do |key|
38
+ # sometimes JSON decoder is overridden and it can be instructed to symbolize keys
39
+ [key, options.delete(key.to_s) || options.delete(key)]
40
+ end.to_h
41
+ args.unshift(options) if options.any?
42
+ end
43
+
44
+ if ::Que.mode == :async
45
+ Config.instance.jobs_propagated_fields.each do |key|
46
+ Kiev[key] = payload[key]
47
+ end
48
+ request_store = Kiev::RequestStore.store
49
+ request_store[:request_id] = payload[:request_id]
50
+ request_store[:request_depth] = payload[:request_depth].to_i + 1
51
+ request_store[:tree_path] = payload[:tree_path]
52
+
53
+ request_store[:background_job] = true
54
+ request_store[:job_name] = attrs[:job_class]
55
+ end
56
+
57
+ began_at = Time.now
58
+
59
+ ::Que::Job.instance_method(:_run).bind(self).call
60
+
61
+ data = {
62
+ params: attrs[:args],
63
+ request_duration: ((Time.now - began_at) * 1000).round(3)
64
+ }
65
+
66
+ error ||= _error
67
+
68
+ if error
69
+ data[:error_class] = error.class.name
70
+ data[:error_message] = error.message[0..5000]
71
+ data[:error_backtrace] = Array(error.backtrace).join(NEW_LINE)[0..5000]
72
+ end
73
+
74
+ Kiev.event(:job_finished, data)
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "base"
4
+ require_relative "rack/request_logger"
5
+ require_relative "rack/request_id"
6
+ require_relative "rack/store_request_details"
7
+ require_relative "rack/silence_action_dispatch_logger"
8
+ require_relative "../ext/rack/common_logger"
9
+
10
+ module Kiev
11
+ module Rack
12
+ def self.included(base)
13
+ # The order is important
14
+ base.use(::RequestStore::Middleware)
15
+ base.use(Kiev::Rack::RequestLogger)
16
+ base.use(Kiev::Rack::StoreRequestDetails)
17
+ base.use(Kiev::Rack::RequestId)
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "securerandom"
4
+
5
+ module Kiev
6
+ module Rack
7
+ class RequestId
8
+ # for Rails 4
9
+ RAILS_REQUEST_ID = "action_dispatch.request_id"
10
+
11
+ def initialize(app)
12
+ @app = app
13
+ end
14
+
15
+ def call(env)
16
+ request_id_header_out = to_rack(:request_id)
17
+ request_id_header_in = to_http(:request_id)
18
+
19
+ request_id = make_request_id(env[RAILS_REQUEST_ID] || env[request_id_header_in])
20
+ RequestStore.store[:request_id] = request_id
21
+ RequestStore.store[:request_depth] = request_depth(env)
22
+ RequestStore.store[:tree_path] = tree_path(env)
23
+
24
+ @app.call(env).tap { |_status, headers, _body| headers[request_id_header_out] = request_id }
25
+ end
26
+
27
+ private
28
+
29
+ # TODO: in Rails 5 they set `headers[X_REQUEST_ID]`, so this will not work
30
+ # https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/request_id.rb
31
+ # https://github.com/interagent/pliny/blob/master/lib/pliny/middleware/request_id.rb
32
+ def tree_root?(env)
33
+ request_id_header_in = to_http(:request_id)
34
+ !env[request_id_header_in]
35
+ end
36
+
37
+ def request_depth(env)
38
+ request_depth_header = to_http(:request_depth)
39
+ tree_root?(env) ? 0 : (env[request_depth_header].to_i + 1)
40
+ end
41
+
42
+ def tree_path(env)
43
+ tree_path_header = to_http(:tree_path)
44
+ tree_root?(env) ? SubrequestHelper.root_path(synchronous: true) : Util.sanitize(env[tree_path_header])
45
+ end
46
+
47
+ def to_http(value)
48
+ Util.to_http(to_rack(value))
49
+ end
50
+
51
+ def to_rack(value)
52
+ Config.instance.all_http_propagated_fields[value]
53
+ end
54
+
55
+ def make_request_id(request_id)
56
+ if request_id.nil? || request_id.empty?
57
+ internal_request_id
58
+ else
59
+ Util.sanitize(request_id)
60
+ end
61
+ end
62
+
63
+ def internal_request_id
64
+ SecureRandom.uuid
65
+ end
66
+ end
67
+ end
68
+ end