sidekiq-logstash 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +6 -0
- data/Gemfile +1 -0
- data/README.md +1 -0
- data/gemfiles/sidekiq4.gemfile +5 -0
- data/gemfiles/sidekiq5.gemfile +5 -0
- data/lib/sidekiq/logging/argument_filter.rb +8 -8
- data/lib/sidekiq/logging/logstash_formatter.rb +1 -1
- data/lib/sidekiq/logging/shared.rb +62 -0
- data/lib/sidekiq/logstash.rb +10 -4
- data/lib/sidekiq/logstash/version.rb +1 -1
- data/lib/sidekiq/logstash_job_logger.rb +23 -0
- data/lib/sidekiq/middleware/server/logstah_logging.rb +4 -57
- data/sidekiq-logstash.gemspec +1 -1
- metadata +17 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 25dbb440e6e8695797cc7857512139580ea799dc
|
4
|
+
data.tar.gz: adc96a196faa34ee2b45822bca2cb6328fe1061c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f10f38029bd0168fe3306fba03d3d7b6cce126a78333e7ce8e791194332988979a37d07d4bf5ea00c65e606f233e153cfc91af2702b26d2810adbd679e57dd4e
|
7
|
+
data.tar.gz: 52fcd3c3e046e03b7e1c464d1eff487a916f06ebfd51fb435f9539650a2d690241ff8548136385bbac93f1e5f345ff0a260d4ba1072fc04d7515011538bb700c
|
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
sidekiq-logstash
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.3.0
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
[](https://badge.fury.io/rb/sidekiq-logstash) [<img src="https://travis-ci.org/iMacTia/sidekiq-logstash.svg?branch=master" alt="version" />](https://travis-ci.org/iMacTia/sidekiq-logstash)
|
4
4
|
|
5
|
+
|
5
6
|
Sidekiq::Logstash turns your [Sidekiq](https://github.com/mperham/sidekiq) log into an organised, aggregated, JSON-syntax log ready to be sent to a logstash server.
|
6
7
|
|
7
8
|
```json
|
@@ -26,12 +26,12 @@ module Sidekiq
|
|
26
26
|
strings, regexps, blocks = [], [], []
|
27
27
|
filters.each do |item|
|
28
28
|
case item
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
29
|
+
when Proc
|
30
|
+
blocks << item
|
31
|
+
when Regexp
|
32
|
+
regexps << item
|
33
|
+
else
|
34
|
+
strings << Regexp.escape(item.to_s)
|
35
35
|
end
|
36
36
|
end
|
37
37
|
deep_regexps, regexps = regexps.partition { |r| r.to_s.include?("\\.".freeze) }
|
@@ -44,9 +44,9 @@ module Sidekiq
|
|
44
44
|
attr_reader :regexps, :deep_regexps, :blocks
|
45
45
|
|
46
46
|
def initialize(regexps, deep_regexps, blocks)
|
47
|
-
@regexps
|
47
|
+
@regexps = regexps
|
48
48
|
@deep_regexps = deep_regexps.any? ? deep_regexps : nil
|
49
|
-
@blocks
|
49
|
+
@blocks = blocks
|
50
50
|
end
|
51
51
|
|
52
52
|
def call(original_args, parents = [])
|
@@ -0,0 +1,62 @@
|
|
1
|
+
module Sidekiq
|
2
|
+
module Logging
|
3
|
+
module Shared
|
4
|
+
def log_job(payload, started_at, exc = nil)
|
5
|
+
# Create a copy of the payload using JSON
|
6
|
+
# This should always be possible since Sidekiq store it in Redis
|
7
|
+
payload = JSON.parse(JSON.unparse(payload))
|
8
|
+
|
9
|
+
# Convert timestamps into Time instances
|
10
|
+
%w( created_at enqueued_at retried_at failed_at completed_at ).each do |key|
|
11
|
+
payload[key] = parse_time(payload[key]) if payload[key]
|
12
|
+
end
|
13
|
+
|
14
|
+
# Add process id params
|
15
|
+
payload['pid'] = ::Process.pid
|
16
|
+
payload['duration'] = elapsed(started_at)
|
17
|
+
|
18
|
+
message = "#{payload['class']} JID-#{payload['jid']}"
|
19
|
+
|
20
|
+
if exc
|
21
|
+
payload['message'] = "#{message}: fail: #{payload['duration']} sec"
|
22
|
+
payload['job_status'] = 'fail'
|
23
|
+
payload['error_message'] = exc.message
|
24
|
+
payload['error'] = exc.class
|
25
|
+
payload['error_backtrace'] = %('#{exc.backtrace.join("\n")}')
|
26
|
+
else
|
27
|
+
payload['message'] = "#{message}: done: #{payload['duration']} sec"
|
28
|
+
payload['job_status'] = 'done'
|
29
|
+
payload['completed_at'] = Time.now.utc
|
30
|
+
end
|
31
|
+
|
32
|
+
# Filter sensitive parameters
|
33
|
+
unless filter_args.empty?
|
34
|
+
args_filter = Sidekiq::Logging::ArgumentFilter.new(filter_args)
|
35
|
+
payload['args'] = args_filter.filter({ args: payload['args'] })[:args]
|
36
|
+
end
|
37
|
+
|
38
|
+
# Needs to map all args to strings for ElasticSearch compatibility
|
39
|
+
payload['args'].map!(&:to_s)
|
40
|
+
|
41
|
+
payload
|
42
|
+
end
|
43
|
+
|
44
|
+
def elapsed(start)
|
45
|
+
(Time.now.utc - start).round(3)
|
46
|
+
end
|
47
|
+
|
48
|
+
def parse_time(timestamp)
|
49
|
+
return timestamp if timestamp.is_a? Time
|
50
|
+
timestamp.is_a?(Float) ?
|
51
|
+
Time.at(timestamp).utc :
|
52
|
+
Time.parse(timestamp)
|
53
|
+
rescue
|
54
|
+
timestamp
|
55
|
+
end
|
56
|
+
|
57
|
+
def filter_args
|
58
|
+
Sidekiq::Logstash.configuration.filter_args
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
data/lib/sidekiq/logstash.rb
CHANGED
@@ -3,6 +3,7 @@ require 'sidekiq/logstash/version'
|
|
3
3
|
require 'sidekiq/middleware/server/logstah_logging'
|
4
4
|
require 'sidekiq/logging/logstash_formatter'
|
5
5
|
require 'sidekiq/logging/argument_filter'
|
6
|
+
require 'sidekiq/logstash_job_logger'
|
6
7
|
|
7
8
|
module Sidekiq
|
8
9
|
module Logstash
|
@@ -18,12 +19,17 @@ module Sidekiq
|
|
18
19
|
# Calls Sidekiq.configure_server to inject logics
|
19
20
|
Sidekiq.configure_server do |config|
|
20
21
|
# Remove default Sidekiq error_handler that logs errors
|
21
|
-
config.error_handlers.delete_if {|h| h.is_a?(Sidekiq::ExceptionHandler::Logger) }
|
22
|
+
config.error_handlers.delete_if { |h| h.is_a?(Sidekiq::ExceptionHandler::Logger) }
|
22
23
|
|
23
24
|
# Add logstash support
|
24
|
-
|
25
|
-
|
26
|
-
|
25
|
+
# The logging server middleware was removed in Sidekiq 5.0.0, see: https://github.com/mperham/sidekiq/blob/master/Changes.md
|
26
|
+
if Sidekiq::Middleware::Server.const_defined?(:Logging)
|
27
|
+
config.server_middleware do |chain|
|
28
|
+
chain.add Sidekiq::Middleware::Server::LogstashLogging
|
29
|
+
chain.remove Sidekiq::Middleware::Server::Logging
|
30
|
+
end
|
31
|
+
else
|
32
|
+
Sidekiq.options[:job_logger] = Sidekiq::LogstashJobLogger
|
27
33
|
end
|
28
34
|
|
29
35
|
# Set custom formatter for Sidekiq logger
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'sidekiq/logging/shared'
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class LogstashJobLogger
|
5
|
+
include Sidekiq::Logging::Shared
|
6
|
+
|
7
|
+
def call(job, _queue)
|
8
|
+
started_at = Time.now.utc
|
9
|
+
yield
|
10
|
+
Sidekiq.logger.info log_job(job, started_at)
|
11
|
+
rescue => exc
|
12
|
+
begin
|
13
|
+
Sidekiq.logger.warn log_job(job, started_at, exc)
|
14
|
+
rescue => ex
|
15
|
+
Sidekiq.logger.error 'Error logging the job execution!'
|
16
|
+
Sidekiq.logger.error "Job: #{job}"
|
17
|
+
Sidekiq.logger.error "Job Exception: #{exc}"
|
18
|
+
Sidekiq.logger.error "Log Exception: #{ex}"
|
19
|
+
end
|
20
|
+
raise
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,7 +1,11 @@
|
|
1
|
+
require 'sidekiq/logging/shared'
|
2
|
+
|
1
3
|
module Sidekiq
|
2
4
|
module Middleware
|
3
5
|
module Server
|
4
6
|
class LogstashLogging
|
7
|
+
include Sidekiq::Logging::Shared
|
8
|
+
|
5
9
|
def call(_, job, _)
|
6
10
|
started_at = Time.now.utc
|
7
11
|
yield
|
@@ -17,63 +21,6 @@ module Sidekiq
|
|
17
21
|
end
|
18
22
|
raise
|
19
23
|
end
|
20
|
-
|
21
|
-
def log_job(payload, started_at, exc = nil)
|
22
|
-
# Create a copy of the payload using JSON
|
23
|
-
# This should always be possible since Sidekiq store it in Redis
|
24
|
-
payload = JSON.parse(JSON.unparse(payload))
|
25
|
-
|
26
|
-
# Convert timestamps into Time instances
|
27
|
-
%w( created_at enqueued_at retried_at failed_at completed_at ).each do |key|
|
28
|
-
payload[key] = parse_time(payload[key]) if payload[key]
|
29
|
-
end
|
30
|
-
|
31
|
-
# Add process id params
|
32
|
-
payload['pid'] = ::Process.pid
|
33
|
-
payload['duration'] = elapsed(started_at)
|
34
|
-
|
35
|
-
message = "#{payload['class']} JID-#{payload['jid']}"
|
36
|
-
|
37
|
-
if exc
|
38
|
-
payload['message'] = "#{message}: fail: #{payload['duration']} sec"
|
39
|
-
payload['job_status'] = 'fail'
|
40
|
-
payload['error_message'] = exc.message
|
41
|
-
payload['error'] = exc.class
|
42
|
-
payload['error_backtrace'] = %('#{exc.backtrace.join("\n")}')
|
43
|
-
else
|
44
|
-
payload['message'] = "#{message}: done: #{payload['duration']} sec"
|
45
|
-
payload['job_status'] = 'done'
|
46
|
-
payload['completed_at'] = Time.now.utc
|
47
|
-
end
|
48
|
-
|
49
|
-
# Filter sensitive parameters
|
50
|
-
unless filter_args.empty?
|
51
|
-
args_filter = Sidekiq::Logging::ArgumentFilter.new(filter_args)
|
52
|
-
payload['args'] = args_filter.filter({ args: payload['args'] })[:args]
|
53
|
-
end
|
54
|
-
|
55
|
-
# Needs to map all args to strings for ElasticSearch compatibility
|
56
|
-
payload['args'].map!(&:to_s)
|
57
|
-
|
58
|
-
payload
|
59
|
-
end
|
60
|
-
|
61
|
-
def elapsed(start)
|
62
|
-
(Time.now.utc - start).round(3)
|
63
|
-
end
|
64
|
-
|
65
|
-
def parse_time(timestamp)
|
66
|
-
return timestamp if timestamp.is_a? Time
|
67
|
-
timestamp.is_a?(Float) ?
|
68
|
-
Time.at(timestamp).utc :
|
69
|
-
Time.parse(timestamp)
|
70
|
-
rescue
|
71
|
-
timestamp
|
72
|
-
end
|
73
|
-
|
74
|
-
def filter_args
|
75
|
-
Sidekiq::Logstash.configuration.filter_args
|
76
|
-
end
|
77
24
|
end
|
78
25
|
end
|
79
26
|
end
|
data/sidekiq-logstash.gemspec
CHANGED
@@ -22,7 +22,7 @@ DESC
|
|
22
22
|
spec.require_paths = ['lib']
|
23
23
|
|
24
24
|
spec.add_dependency 'logstash-event', '~> 1.2'
|
25
|
-
spec.add_runtime_dependency 'sidekiq', '
|
25
|
+
spec.add_runtime_dependency 'sidekiq', '>= 4.0' , '<6'
|
26
26
|
|
27
27
|
spec.add_development_dependency 'bundler', '~> 1.11'
|
28
28
|
spec.add_development_dependency 'rake', '~> 10.0'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-logstash
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mattia Giuffrida
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-07-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-event
|
@@ -28,16 +28,22 @@ dependencies:
|
|
28
28
|
name: sidekiq
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - "
|
31
|
+
- - ">="
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '4.0'
|
34
|
+
- - "<"
|
35
|
+
- !ruby/object:Gem::Version
|
36
|
+
version: '6'
|
34
37
|
type: :runtime
|
35
38
|
prerelease: false
|
36
39
|
version_requirements: !ruby/object:Gem::Requirement
|
37
40
|
requirements:
|
38
|
-
- - "
|
41
|
+
- - ">="
|
39
42
|
- !ruby/object:Gem::Version
|
40
43
|
version: '4.0'
|
44
|
+
- - "<"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '6'
|
41
47
|
- !ruby/object:Gem::Dependency
|
42
48
|
name: bundler
|
43
49
|
requirement: !ruby/object:Gem::Requirement
|
@@ -106,6 +112,8 @@ extra_rdoc_files: []
|
|
106
112
|
files:
|
107
113
|
- ".gitignore"
|
108
114
|
- ".rspec"
|
115
|
+
- ".ruby-gemset"
|
116
|
+
- ".ruby-version"
|
109
117
|
- ".travis.yml"
|
110
118
|
- Gemfile
|
111
119
|
- README.md
|
@@ -113,11 +121,15 @@ files:
|
|
113
121
|
- bin/console
|
114
122
|
- bin/setup
|
115
123
|
- bin/test_console
|
124
|
+
- gemfiles/sidekiq4.gemfile
|
125
|
+
- gemfiles/sidekiq5.gemfile
|
116
126
|
- lib/sidekiq/logging/argument_filter.rb
|
117
127
|
- lib/sidekiq/logging/logstash_formatter.rb
|
128
|
+
- lib/sidekiq/logging/shared.rb
|
118
129
|
- lib/sidekiq/logstash.rb
|
119
130
|
- lib/sidekiq/logstash/configuration.rb
|
120
131
|
- lib/sidekiq/logstash/version.rb
|
132
|
+
- lib/sidekiq/logstash_job_logger.rb
|
121
133
|
- lib/sidekiq/middleware/server/logstah_logging.rb
|
122
134
|
- sidekiq-logstash.gemspec
|
123
135
|
homepage: https://github.com/iMacTia/sidekiq-logstash
|
@@ -140,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
140
152
|
version: '0'
|
141
153
|
requirements: []
|
142
154
|
rubyforge_project:
|
143
|
-
rubygems_version: 2.6.
|
155
|
+
rubygems_version: 2.6.12
|
144
156
|
signing_key:
|
145
157
|
specification_version: 4
|
146
158
|
summary: Logstash plugin for Sidekiq
|