sidekiq-logstash 0.3.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/ci.yml +41 -0
- data/.github/workflows/publish.yml +29 -0
- data/.rubocop.yml +23 -0
- data/.rubocop_todo.yml +37 -0
- data/.ruby-version +1 -1
- data/Gemfile +2 -0
- data/README.md +10 -4
- data/Rakefile +5 -3
- data/bin/console +1 -0
- data/bin/test_console +2 -1
- data/lib/sidekiq/logging/argument_filter.rb +27 -11
- data/lib/sidekiq/logging/logstash_formatter.rb +15 -7
- data/lib/sidekiq/logging/shared.rb +103 -25
- data/lib/sidekiq/logstash.rb +7 -11
- data/lib/sidekiq/logstash/configuration.rb +7 -2
- data/lib/sidekiq/logstash/version.rb +3 -1
- data/lib/sidekiq/logstash_job_logger.rb +8 -16
- data/sidekiq-logstash.gemspec +22 -19
- metadata +36 -23
- data/.travis.yml +0 -10
- data/gemfiles/sidekiq4.gemfile +0 -5
- data/gemfiles/sidekiq5.gemfile +0 -5
- data/lib/sidekiq/middleware/server/logstah_logging.rb +0 -27
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: f1986944d83eef688f7d35ab06cf9dfdb613c51a41cb8f33143ef85fc972be25
|
|
4
|
+
data.tar.gz: d6480ae4134cb68185ac9c6334c36e6ed9d92e3c0684b760a7064be6f6cdf909
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b9f8ade62f6abdb3fda0e2b33d617545b127f8b3f7ffa24501669ba76d05c8c9b3ccf461fd33360896421b9f2becebc914b82bc4681f9382ec2a52b373533844
|
|
7
|
+
data.tar.gz: 9ecfa6afc056cb0b0281eb971bd3e74ceefc71efc9d493b91fa0f901b1f5d2490429d47155301377e1b0d2fece87765821949f27b617f28805383f1b1111aab8
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
pull_request:
|
|
5
|
+
|
|
6
|
+
push:
|
|
7
|
+
branches: [master]
|
|
8
|
+
|
|
9
|
+
env:
|
|
10
|
+
GIT_COMMIT_SHA: ${{ github.sha }}
|
|
11
|
+
GIT_BRANCH: ${{ github.ref }}
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
linting:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@v2
|
|
19
|
+
- uses: actions/setup-ruby@v1
|
|
20
|
+
with:
|
|
21
|
+
ruby-version: 2.7.x
|
|
22
|
+
- name: Install dependencies
|
|
23
|
+
run: bundle install
|
|
24
|
+
- name: Run rubocop
|
|
25
|
+
run: bundle exec rubocop --format progress
|
|
26
|
+
build:
|
|
27
|
+
needs: [linting]
|
|
28
|
+
runs-on: ubuntu-latest
|
|
29
|
+
strategy:
|
|
30
|
+
matrix:
|
|
31
|
+
ruby: ['2.5.x', '2.6.x', '2.7.x']
|
|
32
|
+
|
|
33
|
+
steps:
|
|
34
|
+
- uses: actions/checkout@v2
|
|
35
|
+
- uses: actions/setup-ruby@v1
|
|
36
|
+
with:
|
|
37
|
+
ruby-version: ${{ matrix.ruby }}
|
|
38
|
+
- name: Install dependencies
|
|
39
|
+
run: bundle install
|
|
40
|
+
- name: Run tests
|
|
41
|
+
run: bundle exec rake
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
name: Publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
release:
|
|
5
|
+
types: [published]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
name: Publish to Rubygems
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
|
|
12
|
+
steps:
|
|
13
|
+
- uses: actions/checkout@master
|
|
14
|
+
|
|
15
|
+
- name: Set up Ruby 2.7
|
|
16
|
+
uses: actions/setup-ruby@v1
|
|
17
|
+
with:
|
|
18
|
+
ruby-version: 2.7.x
|
|
19
|
+
|
|
20
|
+
- name: Publish to RubyGems
|
|
21
|
+
run: |
|
|
22
|
+
mkdir -p $HOME/.gem
|
|
23
|
+
touch $HOME/.gem/credentials
|
|
24
|
+
chmod 0600 $HOME/.gem/credentials
|
|
25
|
+
printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
|
|
26
|
+
gem build sidekiq-logstash.gemspec
|
|
27
|
+
gem push sidekiq-logstash-*.gem
|
|
28
|
+
env:
|
|
29
|
+
GEM_HOST_API_KEY: ${{ secrets.RUBYGEMS_AUTH_TOKEN }}
|
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
|
2
|
+
|
|
3
|
+
require:
|
|
4
|
+
- rubocop-performance
|
|
5
|
+
|
|
6
|
+
AllCops:
|
|
7
|
+
DisplayCopNames: true
|
|
8
|
+
DisplayStyleGuide: true
|
|
9
|
+
NewCops: enable
|
|
10
|
+
TargetRubyVersion: 2.5
|
|
11
|
+
|
|
12
|
+
Metrics/BlockLength:
|
|
13
|
+
Exclude:
|
|
14
|
+
- spec/**/*.rb
|
|
15
|
+
|
|
16
|
+
Layout/LineLength:
|
|
17
|
+
Max: 120
|
|
18
|
+
Exclude:
|
|
19
|
+
- spec/**/*.rb
|
|
20
|
+
|
|
21
|
+
Style/Documentation:
|
|
22
|
+
Exclude:
|
|
23
|
+
- 'spec/**/*'
|
data/.rubocop_todo.yml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# This configuration was generated by
|
|
2
|
+
# `rubocop --auto-gen-config`
|
|
3
|
+
# on 2020-08-16 16:36:05 UTC using RuboCop version 0.89.1.
|
|
4
|
+
# The point is for the user to remove these configuration records
|
|
5
|
+
# one by one as the offenses are removed from the code base.
|
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
|
8
|
+
|
|
9
|
+
# Offense count: 1
|
|
10
|
+
Lint/DuplicateMethods:
|
|
11
|
+
Exclude:
|
|
12
|
+
- 'lib/sidekiq/logstash/configuration.rb'
|
|
13
|
+
|
|
14
|
+
# Offense count: 3
|
|
15
|
+
# Configuration parameters: IgnoredMethods.
|
|
16
|
+
Metrics/AbcSize:
|
|
17
|
+
Max: 36
|
|
18
|
+
|
|
19
|
+
# Offense count: 2
|
|
20
|
+
# Configuration parameters: IgnoredMethods.
|
|
21
|
+
Metrics/CyclomaticComplexity:
|
|
22
|
+
Max: 18
|
|
23
|
+
|
|
24
|
+
# Offense count: 6
|
|
25
|
+
# Configuration parameters: CountComments, CountAsOne, ExcludedMethods.
|
|
26
|
+
Metrics/MethodLength:
|
|
27
|
+
Max: 28
|
|
28
|
+
|
|
29
|
+
# Offense count: 1
|
|
30
|
+
# Configuration parameters: IgnoredMethods.
|
|
31
|
+
Metrics/PerceivedComplexity:
|
|
32
|
+
Max: 19
|
|
33
|
+
|
|
34
|
+
# Offense count: 1
|
|
35
|
+
Style/OptionalBooleanParameter:
|
|
36
|
+
Exclude:
|
|
37
|
+
- 'spec/workers/spec_worker.rb'
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
2.
|
|
1
|
+
2.7.1
|
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Sidekiq::Logstash
|
|
2
2
|
|
|
3
|
-
[](https://badge.fury.io/rb/sidekiq-logstash)
|
|
4
|
-
|
|
3
|
+
[](https://badge.fury.io/rb/sidekiq-logstash)
|
|
4
|
+
[](https://github.com/iMacTia/sidekiq-logstash/actions?query=workflow%3ACI)
|
|
5
5
|
|
|
6
6
|
Sidekiq::Logstash turns your [Sidekiq](https://github.com/mperham/sidekiq) log into an organised, aggregated, JSON-syntax log ready to be sent to a logstash server.
|
|
7
7
|
|
|
@@ -30,10 +30,11 @@ Sidekiq::Logstash turns your [Sidekiq](https://github.com/mperham/sidekiq) log i
|
|
|
30
30
|
|
|
31
31
|
## Installation
|
|
32
32
|
|
|
33
|
-
Add
|
|
33
|
+
Add one of the following lines to your application's Gemfile:
|
|
34
34
|
|
|
35
35
|
```ruby
|
|
36
|
-
gem 'sidekiq-logstash'
|
|
36
|
+
gem 'sidekiq-logstash', '~> 2.0' # Sidekiq 6
|
|
37
|
+
gem 'sidekiq-logstash', '< 2' # Sidekiq 5 or older
|
|
37
38
|
```
|
|
38
39
|
|
|
39
40
|
And then execute:
|
|
@@ -68,6 +69,11 @@ Sidekiq::Logstash.configure do |config|
|
|
|
68
69
|
# it works just like rails params filtering (http://guides.rubyonrails.org/action_controller_overview.html#parameters-filtering)
|
|
69
70
|
config.filter_args << 'foo'
|
|
70
71
|
|
|
72
|
+
# by default, the "job started" logs are omitted from the logs
|
|
73
|
+
# to have one-line logs for each log (following Lograge), but you can
|
|
74
|
+
# enable job start logs by setting the following flag:
|
|
75
|
+
config.job_start_log = true
|
|
76
|
+
|
|
71
77
|
# custom_option is a Proc that will be called before logging the payload, allowing you to add fields to it
|
|
72
78
|
config.custom_options = lambda do |payload|
|
|
73
79
|
payload['my_custom_field'] = 'my_custom_value'
|
data/Rakefile
CHANGED
data/bin/console
CHANGED
data/bin/test_console
CHANGED
|
@@ -1,15 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
# This implementation is taken directly from https://github.com/rails/rails/blob/52ce6ece8c8f74064bb64e0a0b1ddd83092718e1/actionpack/lib/action_dispatch/http/parameter_filter.rb
|
|
2
4
|
# Adding actionpack to the gem dependencies would have been too heavy, so here is just what we need.
|
|
3
|
-
|
|
4
5
|
module Sidekiq
|
|
5
6
|
module Logging
|
|
7
|
+
# Class that allows to filter-out sensible arguments.
|
|
6
8
|
class ArgumentFilter
|
|
7
|
-
|
|
9
|
+
# String used to replace sensible arguments.
|
|
10
|
+
FILTERED = '[FILTERED]'
|
|
8
11
|
|
|
9
12
|
def initialize(filters = [])
|
|
10
13
|
@filters = filters
|
|
11
14
|
end
|
|
12
15
|
|
|
16
|
+
# Filters argument by using the filters provided upon initialization.
|
|
17
|
+
# @param args [Hash] the list of arguments in a hash with key :args.
|
|
13
18
|
def filter(args)
|
|
14
19
|
compiled_filter.call(args)
|
|
15
20
|
end
|
|
@@ -22,8 +27,11 @@ module Sidekiq
|
|
|
22
27
|
|
|
23
28
|
class CompiledFilter # :nodoc:
|
|
24
29
|
def self.compile(filters)
|
|
25
|
-
return
|
|
26
|
-
|
|
30
|
+
return ->(args) { args.dup } if filters.empty?
|
|
31
|
+
|
|
32
|
+
strings = []
|
|
33
|
+
regexps = []
|
|
34
|
+
blocks = []
|
|
27
35
|
filters.each do |item|
|
|
28
36
|
case item
|
|
29
37
|
when Proc
|
|
@@ -34,10 +42,10 @@ module Sidekiq
|
|
|
34
42
|
strings << Regexp.escape(item.to_s)
|
|
35
43
|
end
|
|
36
44
|
end
|
|
37
|
-
deep_regexps, regexps = regexps.partition { |r| r.to_s.include?(
|
|
38
|
-
deep_strings, strings = strings.partition { |s| s.include?(
|
|
39
|
-
regexps << Regexp.new(strings.join('|'
|
|
40
|
-
deep_regexps << Regexp.new(deep_strings.join('|'
|
|
45
|
+
deep_regexps, regexps = regexps.partition { |r| r.to_s.include?('\\.') }
|
|
46
|
+
deep_strings, strings = strings.partition { |s| s.include?('\\.') }
|
|
47
|
+
regexps << Regexp.new(strings.join('|'), true) unless strings.empty?
|
|
48
|
+
deep_regexps << Regexp.new(deep_strings.join('|'), true) unless deep_strings.empty?
|
|
41
49
|
new regexps, deep_regexps, blocks
|
|
42
50
|
end
|
|
43
51
|
|
|
@@ -62,8 +70,16 @@ module Sidekiq
|
|
|
62
70
|
elsif value.is_a?(Array)
|
|
63
71
|
value = value.map { |v| v.is_a?(Hash) ? call(v, parents) : v }
|
|
64
72
|
elsif blocks.any?
|
|
65
|
-
key =
|
|
66
|
-
|
|
73
|
+
key = begin
|
|
74
|
+
key.dup
|
|
75
|
+
rescue StandardError
|
|
76
|
+
key
|
|
77
|
+
end
|
|
78
|
+
value = begin
|
|
79
|
+
value.dup
|
|
80
|
+
rescue StandardError
|
|
81
|
+
value
|
|
82
|
+
end
|
|
67
83
|
blocks.each { |b| b.call(key, value) }
|
|
68
84
|
end
|
|
69
85
|
parents.pop if deep_regexps
|
|
@@ -74,4 +90,4 @@ module Sidekiq
|
|
|
74
90
|
end
|
|
75
91
|
end
|
|
76
92
|
end
|
|
77
|
-
end
|
|
93
|
+
end
|
|
@@ -1,25 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'logstash-event'
|
|
2
4
|
|
|
3
5
|
module Sidekiq
|
|
4
6
|
module Logging
|
|
7
|
+
# Class that takes a log payload and format it to be Logstash-compatible.
|
|
5
8
|
class LogstashFormatter
|
|
6
|
-
def call(severity,
|
|
9
|
+
def call(severity, _time, _progname, data)
|
|
10
|
+
json_data = { severity: severity }
|
|
11
|
+
|
|
7
12
|
if data.is_a? Hash
|
|
8
|
-
json_data
|
|
13
|
+
json_data.merge!(data)
|
|
9
14
|
else
|
|
10
|
-
json_data =
|
|
11
|
-
severity: severity,
|
|
12
|
-
message: data
|
|
13
|
-
}
|
|
15
|
+
json_data[:message] = data
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
# Merge custom_options to provide customization
|
|
17
|
-
|
|
19
|
+
begin
|
|
20
|
+
custom_options&.call(json_data)
|
|
21
|
+
rescue StandardError
|
|
22
|
+
nil
|
|
23
|
+
end
|
|
18
24
|
event = LogStash::Event.new(json_data)
|
|
19
25
|
|
|
20
26
|
"#{event.to_json}\n"
|
|
21
27
|
end
|
|
22
28
|
|
|
29
|
+
private
|
|
30
|
+
|
|
23
31
|
def custom_options
|
|
24
32
|
Sidekiq::Logstash.configuration.custom_options
|
|
25
33
|
end
|
|
@@ -1,62 +1,140 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Sidekiq
|
|
2
4
|
module Logging
|
|
5
|
+
# Shared module with all the logics used by job loggers.
|
|
3
6
|
module Shared
|
|
4
|
-
|
|
7
|
+
ENCRYPTED = '[ENCRYPTED]'
|
|
8
|
+
|
|
9
|
+
def log_job(job)
|
|
10
|
+
started_at = Time.now.utc
|
|
11
|
+
log_start = log_job_start(job)
|
|
12
|
+
Sidekiq.logger.info log_start if log_start
|
|
13
|
+
yield if block_given?
|
|
14
|
+
Sidekiq.logger.info log_job_exec(job, started_at)
|
|
15
|
+
rescue StandardError => e
|
|
16
|
+
begin
|
|
17
|
+
Sidekiq.logger.warn log_job_exception(job, started_at, e)
|
|
18
|
+
rescue StandardError => e
|
|
19
|
+
log_standard_error(job, e, e)
|
|
20
|
+
end
|
|
21
|
+
raise
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def log_job_start(job)
|
|
25
|
+
return unless Sidekiq::Logstash.configuration.job_start_log
|
|
26
|
+
# Skips start logs for retrying jobs
|
|
27
|
+
return if job['failed_at']
|
|
28
|
+
|
|
29
|
+
payload = setup_payload(job)
|
|
30
|
+
payload['job_status'] = 'started'
|
|
31
|
+
payload['message'] += ': started'
|
|
32
|
+
process_payload(payload)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def log_job_exec(job, started_at)
|
|
36
|
+
payload = setup_payload(job)
|
|
37
|
+
payload['duration'] = elapsed(started_at)
|
|
38
|
+
|
|
39
|
+
payload['message'] += ": done: #{payload['duration']} sec"
|
|
40
|
+
payload['job_status'] = 'done'
|
|
41
|
+
payload['completed_at'] = Time.now.utc
|
|
42
|
+
|
|
43
|
+
process_payload(payload)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def log_job_exception(job, started_at, exc)
|
|
47
|
+
payload = setup_payload(job)
|
|
48
|
+
payload['duration'] = elapsed(started_at)
|
|
49
|
+
|
|
50
|
+
payload['message'] += ": fail: #{payload['duration']} sec"
|
|
51
|
+
payload['job_status'] = 'fail'
|
|
52
|
+
payload['error_message'] = exc.message
|
|
53
|
+
payload['error'] = exc.class
|
|
54
|
+
payload['error_backtrace'] = %('#{exc.backtrace.join("\n")}')
|
|
55
|
+
|
|
56
|
+
process_payload(payload)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
private
|
|
60
|
+
|
|
61
|
+
def setup_payload(job)
|
|
5
62
|
# Create a copy of the payload using JSON
|
|
6
63
|
# This should always be possible since Sidekiq store it in Redis
|
|
7
|
-
payload = JSON.parse(JSON.unparse(
|
|
64
|
+
payload = JSON.parse(JSON.unparse(job))
|
|
8
65
|
|
|
9
66
|
# Convert timestamps into Time instances
|
|
10
|
-
%w
|
|
67
|
+
%w[created_at enqueued_at retried_at failed_at completed_at].each do |key|
|
|
11
68
|
payload[key] = parse_time(payload[key]) if payload[key]
|
|
12
69
|
end
|
|
13
70
|
|
|
14
|
-
#
|
|
15
|
-
payload['
|
|
16
|
-
payload['duration'] = elapsed(started_at)
|
|
71
|
+
# Sets the initial message
|
|
72
|
+
payload['message'] = "#{payload['class']} JID-#{payload['jid']}"
|
|
17
73
|
|
|
18
|
-
|
|
74
|
+
# Add process id params
|
|
75
|
+
payload['pid'] = ::Process.pid
|
|
19
76
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
payload['job_status'] = 'fail'
|
|
23
|
-
payload['error_message'] = exc.message
|
|
24
|
-
payload['error'] = exc.class
|
|
25
|
-
payload['error_backtrace'] = %('#{exc.backtrace.join("\n")}')
|
|
26
|
-
else
|
|
27
|
-
payload['message'] = "#{message}: done: #{payload['duration']} sec"
|
|
28
|
-
payload['job_status'] = 'done'
|
|
29
|
-
payload['completed_at'] = Time.now.utc
|
|
30
|
-
end
|
|
77
|
+
payload
|
|
78
|
+
end
|
|
31
79
|
|
|
80
|
+
def process_payload(payload)
|
|
32
81
|
# Filter sensitive parameters
|
|
33
82
|
unless filter_args.empty?
|
|
34
|
-
args_filter
|
|
35
|
-
payload['args'] = args_filter.filter(
|
|
83
|
+
args_filter = Sidekiq::Logging::ArgumentFilter.new(filter_args)
|
|
84
|
+
payload['args'] = args_filter.filter(args: payload['args'])[:args]
|
|
36
85
|
end
|
|
37
86
|
|
|
87
|
+
# If encrypt is true, the last arg is encrypted so hide it
|
|
88
|
+
payload['args'][-1] = ENCRYPTED if payload['encrypt']
|
|
89
|
+
|
|
38
90
|
# Needs to map all args to strings for ElasticSearch compatibility
|
|
39
|
-
payload['args']
|
|
91
|
+
deep_stringify!(payload['args'])
|
|
92
|
+
|
|
93
|
+
# Needs to map all unique_args to strings for ElasticSearch
|
|
94
|
+
# compatibility in case sidekiq-unique-jobs is used
|
|
95
|
+
deep_stringify!(payload['unique_args'])
|
|
96
|
+
|
|
97
|
+
if payload['retry'].is_a?(Integer)
|
|
98
|
+
payload['max_retries'] = payload['retry']
|
|
99
|
+
payload['retry'] = true
|
|
100
|
+
end
|
|
40
101
|
|
|
41
102
|
payload
|
|
42
103
|
end
|
|
43
104
|
|
|
105
|
+
def log_standard_error(job, job_exc, log_exc)
|
|
106
|
+
Sidekiq.logger.error 'Error logging the job execution!'
|
|
107
|
+
Sidekiq.logger.error "Job: #{job}"
|
|
108
|
+
Sidekiq.logger.error "Job Exception: #{job_exc}"
|
|
109
|
+
Sidekiq.logger.error "Log Exception: #{log_exc}"
|
|
110
|
+
end
|
|
111
|
+
|
|
44
112
|
def elapsed(start)
|
|
45
113
|
(Time.now.utc - start).round(3)
|
|
46
114
|
end
|
|
47
115
|
|
|
48
116
|
def parse_time(timestamp)
|
|
49
117
|
return timestamp if timestamp.is_a? Time
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
rescue
|
|
118
|
+
|
|
119
|
+
timestamp.is_a?(Float) ? Time.at(timestamp).utc : Time.parse(timestamp)
|
|
120
|
+
rescue StandardError
|
|
54
121
|
timestamp
|
|
55
122
|
end
|
|
56
123
|
|
|
57
124
|
def filter_args
|
|
58
125
|
Sidekiq::Logstash.configuration.filter_args
|
|
59
126
|
end
|
|
127
|
+
|
|
128
|
+
def deep_stringify!(args)
|
|
129
|
+
case args
|
|
130
|
+
when Hash
|
|
131
|
+
Hash[args.map { |key, value| [deep_stringify!(key), deep_stringify!(value)] }]
|
|
132
|
+
when Array
|
|
133
|
+
args.map! { |val| deep_stringify!(val) }
|
|
134
|
+
else
|
|
135
|
+
args.to_s
|
|
136
|
+
end
|
|
137
|
+
end
|
|
60
138
|
end
|
|
61
139
|
end
|
|
62
140
|
end
|
data/lib/sidekiq/logstash.rb
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
require 'sidekiq/logstash/configuration'
|
|
2
4
|
require 'sidekiq/logstash/version'
|
|
3
|
-
require 'sidekiq/middleware/server/logstah_logging'
|
|
4
5
|
require 'sidekiq/logging/logstash_formatter'
|
|
5
6
|
require 'sidekiq/logging/argument_filter'
|
|
6
7
|
require 'sidekiq/logstash_job_logger'
|
|
7
8
|
|
|
8
9
|
module Sidekiq
|
|
10
|
+
# Main level module for Sidekiq::Logstash.
|
|
11
|
+
# Provides integration between Sidekiq and Logstash by changing the way
|
|
12
|
+
# Sidekiq jobs are logged.
|
|
9
13
|
module Logstash
|
|
10
14
|
def self.configuration
|
|
11
15
|
@configuration ||= Configuration.new
|
|
@@ -15,22 +19,14 @@ module Sidekiq
|
|
|
15
19
|
yield(configuration)
|
|
16
20
|
end
|
|
17
21
|
|
|
18
|
-
def self.setup(
|
|
22
|
+
def self.setup(_opts = {})
|
|
19
23
|
# Calls Sidekiq.configure_server to inject logics
|
|
20
24
|
Sidekiq.configure_server do |config|
|
|
21
25
|
# Remove default Sidekiq error_handler that logs errors
|
|
22
26
|
config.error_handlers.delete_if { |h| h.is_a?(Sidekiq::ExceptionHandler::Logger) }
|
|
23
27
|
|
|
24
28
|
# Add logstash support
|
|
25
|
-
|
|
26
|
-
if Sidekiq::Middleware::Server.const_defined?(:Logging)
|
|
27
|
-
config.server_middleware do |chain|
|
|
28
|
-
chain.add Sidekiq::Middleware::Server::LogstashLogging
|
|
29
|
-
chain.remove Sidekiq::Middleware::Server::Logging
|
|
30
|
-
end
|
|
31
|
-
else
|
|
32
|
-
Sidekiq.options[:job_logger] = Sidekiq::LogstashJobLogger
|
|
33
|
-
end
|
|
29
|
+
config.options[:job_logger] = Sidekiq::LogstashJobLogger
|
|
34
30
|
|
|
35
31
|
# Set custom formatter for Sidekiq logger
|
|
36
32
|
config.logger.formatter = Sidekiq::Logging::LogstashFormatter.new
|
|
@@ -1,17 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
1
3
|
module Sidekiq
|
|
2
4
|
module Logstash
|
|
5
|
+
# Class that allows to configure the gem behaviour.
|
|
3
6
|
class Configuration
|
|
4
|
-
attr_accessor :custom_options, :filter_args
|
|
7
|
+
attr_accessor :custom_options, :filter_args, :job_start_log
|
|
5
8
|
|
|
6
9
|
def initialize
|
|
7
10
|
@filter_args = []
|
|
11
|
+
@job_start_log = false
|
|
8
12
|
end
|
|
9
13
|
|
|
10
14
|
# Added to ensure custom_options is a Proc
|
|
11
15
|
def custom_options=(proc)
|
|
12
16
|
raise ArgumentError, 'Argument must be a Proc.' unless proc.is_a?(Proc)
|
|
17
|
+
|
|
13
18
|
@custom_options = proc
|
|
14
19
|
end
|
|
15
20
|
end
|
|
16
21
|
end
|
|
17
|
-
end
|
|
22
|
+
end
|
|
@@ -1,23 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'sidekiq/job_logger'
|
|
1
4
|
require 'sidekiq/logging/shared'
|
|
2
5
|
|
|
3
6
|
module Sidekiq
|
|
4
|
-
|
|
7
|
+
# Class used to replace Sidekiq default job logger.
|
|
8
|
+
class LogstashJobLogger < ::Sidekiq::JobLogger
|
|
5
9
|
include Sidekiq::Logging::Shared
|
|
6
10
|
|
|
7
|
-
def call(job, _queue)
|
|
8
|
-
|
|
9
|
-
yield
|
|
10
|
-
Sidekiq.logger.info log_job(job, started_at)
|
|
11
|
-
rescue => exc
|
|
12
|
-
begin
|
|
13
|
-
Sidekiq.logger.warn log_job(job, started_at, exc)
|
|
14
|
-
rescue => ex
|
|
15
|
-
Sidekiq.logger.error 'Error logging the job execution!'
|
|
16
|
-
Sidekiq.logger.error "Job: #{job}"
|
|
17
|
-
Sidekiq.logger.error "Job Exception: #{exc}"
|
|
18
|
-
Sidekiq.logger.error "Log Exception: #{ex}"
|
|
19
|
-
end
|
|
20
|
-
raise
|
|
11
|
+
def call(job, _queue, &block)
|
|
12
|
+
log_job(job, &block)
|
|
21
13
|
end
|
|
22
14
|
end
|
|
23
|
-
end
|
|
15
|
+
end
|
data/sidekiq-logstash.gemspec
CHANGED
|
@@ -1,31 +1,34 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
require 'sidekiq/logstash/version'
|
|
5
6
|
|
|
6
7
|
Gem::Specification.new do |spec|
|
|
7
|
-
spec.name
|
|
8
|
-
spec.version
|
|
9
|
-
spec.authors
|
|
10
|
-
spec.email
|
|
8
|
+
spec.name = 'sidekiq-logstash'
|
|
9
|
+
spec.version = Sidekiq::Logstash::VERSION
|
|
10
|
+
spec.authors = ['Mattia Giuffrida']
|
|
11
|
+
spec.email = ['giuffrida.mattia@gmail.com']
|
|
11
12
|
|
|
12
|
-
spec.summary
|
|
13
|
-
spec.description
|
|
14
|
-
Sidekiq::Logstash turns your Sidekiq log into an organised, aggregated, JSON-syntax log ready to be sent to a logstash server.
|
|
15
|
-
DESC
|
|
16
|
-
spec.homepage
|
|
17
|
-
spec.license
|
|
13
|
+
spec.summary = 'Logstash plugin for Sidekiq'
|
|
14
|
+
spec.description = <<~DESC
|
|
15
|
+
Sidekiq::Logstash turns your Sidekiq log into an organised, aggregated, JSON-syntax log ready to be sent to a logstash server.
|
|
16
|
+
DESC
|
|
17
|
+
spec.homepage = 'https://github.com/iMacTia/sidekiq-logstash'
|
|
18
|
+
spec.license = 'MIT'
|
|
18
19
|
|
|
19
|
-
spec.files
|
|
20
|
-
spec.bindir
|
|
21
|
-
spec.executables
|
|
20
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
|
21
|
+
spec.bindir = 'exe'
|
|
22
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
22
23
|
spec.require_paths = ['lib']
|
|
24
|
+
spec.required_ruby_version = '>= 2.5.0'
|
|
23
25
|
|
|
24
26
|
spec.add_dependency 'logstash-event', '~> 1.2'
|
|
25
|
-
spec.add_runtime_dependency 'sidekiq', '>=
|
|
27
|
+
spec.add_runtime_dependency 'sidekiq', '>= 6.0', '<7'
|
|
26
28
|
|
|
27
|
-
spec.add_development_dependency '
|
|
28
|
-
spec.add_development_dependency 'rake', '
|
|
29
|
+
spec.add_development_dependency 'factory_bot', '~> 6.1'
|
|
30
|
+
spec.add_development_dependency 'rake', '>= 10'
|
|
29
31
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
|
30
|
-
spec.add_development_dependency '
|
|
32
|
+
spec.add_development_dependency 'rspec-json_expectations', '~> 2.1.0'
|
|
33
|
+
spec.add_development_dependency 'rubocop-performance', '~> 1.5'
|
|
31
34
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: sidekiq-logstash
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 2.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Mattia Giuffrida
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2020-08-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: logstash-event
|
|
@@ -30,48 +30,48 @@ dependencies:
|
|
|
30
30
|
requirements:
|
|
31
31
|
- - ">="
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
33
|
+
version: '6.0'
|
|
34
34
|
- - "<"
|
|
35
35
|
- !ruby/object:Gem::Version
|
|
36
|
-
version: '
|
|
36
|
+
version: '7'
|
|
37
37
|
type: :runtime
|
|
38
38
|
prerelease: false
|
|
39
39
|
version_requirements: !ruby/object:Gem::Requirement
|
|
40
40
|
requirements:
|
|
41
41
|
- - ">="
|
|
42
42
|
- !ruby/object:Gem::Version
|
|
43
|
-
version: '
|
|
43
|
+
version: '6.0'
|
|
44
44
|
- - "<"
|
|
45
45
|
- !ruby/object:Gem::Version
|
|
46
|
-
version: '
|
|
46
|
+
version: '7'
|
|
47
47
|
- !ruby/object:Gem::Dependency
|
|
48
|
-
name:
|
|
48
|
+
name: factory_bot
|
|
49
49
|
requirement: !ruby/object:Gem::Requirement
|
|
50
50
|
requirements:
|
|
51
51
|
- - "~>"
|
|
52
52
|
- !ruby/object:Gem::Version
|
|
53
|
-
version: '1
|
|
53
|
+
version: '6.1'
|
|
54
54
|
type: :development
|
|
55
55
|
prerelease: false
|
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
|
57
57
|
requirements:
|
|
58
58
|
- - "~>"
|
|
59
59
|
- !ruby/object:Gem::Version
|
|
60
|
-
version: '1
|
|
60
|
+
version: '6.1'
|
|
61
61
|
- !ruby/object:Gem::Dependency
|
|
62
62
|
name: rake
|
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
|
64
64
|
requirements:
|
|
65
|
-
- - "
|
|
65
|
+
- - ">="
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
|
-
version: '10
|
|
67
|
+
version: '10'
|
|
68
68
|
type: :development
|
|
69
69
|
prerelease: false
|
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
|
71
71
|
requirements:
|
|
72
|
-
- - "
|
|
72
|
+
- - ">="
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
|
-
version: '10
|
|
74
|
+
version: '10'
|
|
75
75
|
- !ruby/object:Gem::Dependency
|
|
76
76
|
name: rspec
|
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -87,19 +87,33 @@ dependencies:
|
|
|
87
87
|
- !ruby/object:Gem::Version
|
|
88
88
|
version: '3.0'
|
|
89
89
|
- !ruby/object:Gem::Dependency
|
|
90
|
-
name:
|
|
90
|
+
name: rspec-json_expectations
|
|
91
|
+
requirement: !ruby/object:Gem::Requirement
|
|
92
|
+
requirements:
|
|
93
|
+
- - "~>"
|
|
94
|
+
- !ruby/object:Gem::Version
|
|
95
|
+
version: 2.1.0
|
|
96
|
+
type: :development
|
|
97
|
+
prerelease: false
|
|
98
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
99
|
+
requirements:
|
|
100
|
+
- - "~>"
|
|
101
|
+
- !ruby/object:Gem::Version
|
|
102
|
+
version: 2.1.0
|
|
103
|
+
- !ruby/object:Gem::Dependency
|
|
104
|
+
name: rubocop-performance
|
|
91
105
|
requirement: !ruby/object:Gem::Requirement
|
|
92
106
|
requirements:
|
|
93
107
|
- - "~>"
|
|
94
108
|
- !ruby/object:Gem::Version
|
|
95
|
-
version: '
|
|
109
|
+
version: '1.5'
|
|
96
110
|
type: :development
|
|
97
111
|
prerelease: false
|
|
98
112
|
version_requirements: !ruby/object:Gem::Requirement
|
|
99
113
|
requirements:
|
|
100
114
|
- - "~>"
|
|
101
115
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: '
|
|
116
|
+
version: '1.5'
|
|
103
117
|
description: 'Sidekiq::Logstash turns your Sidekiq log into an organised, aggregated,
|
|
104
118
|
JSON-syntax log ready to be sent to a logstash server.
|
|
105
119
|
|
|
@@ -110,19 +124,20 @@ executables: []
|
|
|
110
124
|
extensions: []
|
|
111
125
|
extra_rdoc_files: []
|
|
112
126
|
files:
|
|
127
|
+
- ".github/workflows/ci.yml"
|
|
128
|
+
- ".github/workflows/publish.yml"
|
|
113
129
|
- ".gitignore"
|
|
114
130
|
- ".rspec"
|
|
131
|
+
- ".rubocop.yml"
|
|
132
|
+
- ".rubocop_todo.yml"
|
|
115
133
|
- ".ruby-gemset"
|
|
116
134
|
- ".ruby-version"
|
|
117
|
-
- ".travis.yml"
|
|
118
135
|
- Gemfile
|
|
119
136
|
- README.md
|
|
120
137
|
- Rakefile
|
|
121
138
|
- bin/console
|
|
122
139
|
- bin/setup
|
|
123
140
|
- bin/test_console
|
|
124
|
-
- gemfiles/sidekiq4.gemfile
|
|
125
|
-
- gemfiles/sidekiq5.gemfile
|
|
126
141
|
- lib/sidekiq/logging/argument_filter.rb
|
|
127
142
|
- lib/sidekiq/logging/logstash_formatter.rb
|
|
128
143
|
- lib/sidekiq/logging/shared.rb
|
|
@@ -130,7 +145,6 @@ files:
|
|
|
130
145
|
- lib/sidekiq/logstash/configuration.rb
|
|
131
146
|
- lib/sidekiq/logstash/version.rb
|
|
132
147
|
- lib/sidekiq/logstash_job_logger.rb
|
|
133
|
-
- lib/sidekiq/middleware/server/logstah_logging.rb
|
|
134
148
|
- sidekiq-logstash.gemspec
|
|
135
149
|
homepage: https://github.com/iMacTia/sidekiq-logstash
|
|
136
150
|
licenses:
|
|
@@ -144,15 +158,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
144
158
|
requirements:
|
|
145
159
|
- - ">="
|
|
146
160
|
- !ruby/object:Gem::Version
|
|
147
|
-
version:
|
|
161
|
+
version: 2.5.0
|
|
148
162
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
163
|
requirements:
|
|
150
164
|
- - ">="
|
|
151
165
|
- !ruby/object:Gem::Version
|
|
152
166
|
version: '0'
|
|
153
167
|
requirements: []
|
|
154
|
-
|
|
155
|
-
rubygems_version: 2.6.12
|
|
168
|
+
rubygems_version: 3.1.2
|
|
156
169
|
signing_key:
|
|
157
170
|
specification_version: 4
|
|
158
171
|
summary: Logstash plugin for Sidekiq
|
data/.travis.yml
DELETED
data/gemfiles/sidekiq4.gemfile
DELETED
data/gemfiles/sidekiq5.gemfile
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
require 'sidekiq/logging/shared'
|
|
2
|
-
|
|
3
|
-
module Sidekiq
|
|
4
|
-
module Middleware
|
|
5
|
-
module Server
|
|
6
|
-
class LogstashLogging
|
|
7
|
-
include Sidekiq::Logging::Shared
|
|
8
|
-
|
|
9
|
-
def call(_, job, _)
|
|
10
|
-
started_at = Time.now.utc
|
|
11
|
-
yield
|
|
12
|
-
Sidekiq.logger.info log_job(job, started_at)
|
|
13
|
-
rescue => exc
|
|
14
|
-
begin
|
|
15
|
-
Sidekiq.logger.warn log_job(job, started_at, exc)
|
|
16
|
-
rescue => ex
|
|
17
|
-
Sidekiq.logger.error 'Error logging the job execution!'
|
|
18
|
-
Sidekiq.logger.error "Job: #{job}"
|
|
19
|
-
Sidekiq.logger.error "Job Exception: #{exc}"
|
|
20
|
-
Sidekiq.logger.error "Log Exception: #{ex}"
|
|
21
|
-
end
|
|
22
|
-
raise
|
|
23
|
-
end
|
|
24
|
-
end
|
|
25
|
-
end
|
|
26
|
-
end
|
|
27
|
-
end
|