sidekiq_log_analyser 0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: e3a036b6772fc546a4a801308218311359568ebe
4
+ data.tar.gz: 45a43301a978ce126ab49a0f1cf8bfba8ab2efa5
5
+ SHA512:
6
+ metadata.gz: b42b0cb77d124462e3401e52c5f466056904cb25a1c46c6c0f2d4493948436d7bf677e131f9fc9a7be4793a1a78826c024c7279c32f090f64328e61b10f3b4ee
7
+ data.tar.gz: 0b981740d105bd1ea54afe6687e47263612d940c0f635e3f80ce7f73c3a8c3e6653f8bed7d496803eb93172d00598ddb1c21fa4da4cf956bfba6eafa0dbfcbb8
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in sidekiq_log_analyser.gemspec
4
+ gemspec
@@ -0,0 +1,62 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ sidekiq_log_analyser (0.1)
5
+ colorize
6
+ groupdate
7
+
8
+ GEM
9
+ remote: https://rubygems.org/
10
+ specs:
11
+ activesupport (5.2.1)
12
+ concurrent-ruby (~> 1.0, >= 1.0.2)
13
+ i18n (>= 0.7, < 2)
14
+ minitest (~> 5.1)
15
+ tzinfo (~> 1.1)
16
+ coderay (1.1.2)
17
+ colorize (0.8.1)
18
+ concurrent-ruby (1.0.5)
19
+ diff-lcs (1.3)
20
+ groupdate (4.0.1)
21
+ activesupport (>= 4.2)
22
+ i18n (1.1.0)
23
+ concurrent-ruby (~> 1.0)
24
+ method_source (0.9.0)
25
+ minitest (5.11.3)
26
+ pry (0.11.3)
27
+ coderay (~> 1.1.0)
28
+ method_source (~> 0.9.0)
29
+ rake (10.5.0)
30
+ rspec (3.8.0)
31
+ rspec-core (~> 3.8.0)
32
+ rspec-expectations (~> 3.8.0)
33
+ rspec-mocks (~> 3.8.0)
34
+ rspec-core (3.8.0)
35
+ rspec-support (~> 3.8.0)
36
+ rspec-expectations (3.8.1)
37
+ diff-lcs (>= 1.2.0, < 2.0)
38
+ rspec-support (~> 3.8.0)
39
+ rspec-mocks (3.8.0)
40
+ diff-lcs (>= 1.2.0, < 2.0)
41
+ rspec-support (~> 3.8.0)
42
+ rspec-support (3.8.0)
43
+ terminal-table (1.8.0)
44
+ unicode-display_width (~> 1.1, >= 1.1.1)
45
+ thread_safe (0.3.6)
46
+ tzinfo (1.2.5)
47
+ thread_safe (~> 0.1)
48
+ unicode-display_width (1.4.0)
49
+
50
+ PLATFORMS
51
+ ruby
52
+
53
+ DEPENDENCIES
54
+ bundler (~> 1.13)
55
+ pry
56
+ rake (~> 10.0)
57
+ rspec (~> 3.0)
58
+ sidekiq_log_analyser!
59
+ terminal-table
60
+
61
+ BUNDLED WITH
62
+ 1.16.1
@@ -0,0 +1,41 @@
1
+ # SidekiqLogAnalyser
2
+
3
+ Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/sidekiq_log_analyser`. To experiment with that code, run `bin/console` for an interactive prompt.
4
+
5
+ TODO: Delete this and the text above, and describe your gem
6
+
7
+ ## Installation
8
+
9
+ Add this line to your application's Gemfile:
10
+
11
+ ```ruby
12
+ gem 'sidekiq_log_analyser'
13
+ ```
14
+
15
+ And then execute:
16
+
17
+ $ bundle
18
+
19
+ Or install it yourself as:
20
+
21
+ $ gem install sidekiq_log_analyser
22
+
23
+ ## Usage
24
+
25
+ TODO: Write usage instructions here
26
+
27
+ ## Development
28
+
29
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
30
+
31
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
32
+
33
+ ## Contributing
34
+
35
+ Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/sidekiq_log_analyser.
36
+
37
+
38
+ ## License
39
+
40
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
41
+
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,16 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "pry"
5
+ require "groupdate"
6
+ require "sidekiq_log_analyser"
7
+
8
+ # You can add fixtures and/or initialization code here to make experimenting
9
+ # with your gem easier. You can also use a different console, if you like.
10
+
11
+ # (If you use this, don't forget to add pry to your Gemfile!)
12
+ # require "pry"
13
+ # Pry.start
14
+
15
+ require "irb"
16
+ IRB.start
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative "../lib/sidekiq_log_analyser.rb"
4
+
5
+ filename = ARGV[0] || 'log/sidekiq.log'
6
+
7
+ begin
8
+ SidekiqLogAnalyser::App.run(filename)
9
+ rescue Errno::ENOENT
10
+ puts "File not found: #{ARGV[0] || 'log/sidekiq.log'}".colorize(:red)
11
+ puts
12
+ puts "Sample:".colorize(:green)
13
+ puts "sidekiq_log_analyser ./log/sidekiq.log".colorize(:green)
14
+ end
@@ -0,0 +1,15 @@
1
+ require 'time'
2
+ require 'groupdate'
3
+ require 'colorize'
4
+ require 'terminal-table'
5
+ require "sidekiq_log_analyser/version"
6
+ require "sidekiq_log_analyser/row"
7
+ require "sidekiq_log_analyser/pretty_print"
8
+ require "sidekiq_log_analyser/collection"
9
+ require "sidekiq_log_analyser/log_file"
10
+ require "sidekiq_log_analyser/simple_stat"
11
+ require "sidekiq_log_analyser/timeline"
12
+ require "sidekiq_log_analyser/app"
13
+
14
+ module SidekiqLogAnalyser
15
+ end
@@ -0,0 +1,20 @@
1
+ module SidekiqLogAnalyser
2
+ class App
3
+ def App.run(filename)
4
+ log_file = LogFile.new(filename)
5
+ collection = Collection.new(log_file.lines)
6
+ statistics = SimpleStat.new(collection)
7
+
8
+ PrettyPrint.print_summary_report(statistics.total)
9
+ PrettyPrint.print_avg_report(statistics.avg)
10
+ PrettyPrint.print_timeline_report(statistics.timeline)
11
+
12
+ nil
13
+ end
14
+ end
15
+ end
16
+
17
+
18
+ def ttt
19
+ SidekiqLogAnalyser::App.run("sample/small.log")
20
+ end
@@ -0,0 +1,29 @@
1
+ module SidekiqLogAnalyser
2
+ class Collection
3
+ attr_reader :collection
4
+ attr_reader :start_collection, :end_collection
5
+
6
+ def initialize(lines)
7
+ populate(lines)
8
+ end
9
+
10
+ def populate(lines)
11
+ @collection = []
12
+ lines.each do |line|
13
+ row = Row.new(line)
14
+ if row.valid?
15
+ @collection << row.metadata
16
+ end
17
+ end
18
+ @start_collection ||= collection(:start).group_by{|metadata| metadata[:worker]}
19
+ @end_collection ||= collection(:end).group_by{|metadata| metadata[:worker]}
20
+ end
21
+
22
+ private
23
+
24
+ def collection(type)
25
+ @collection.select{|metadata| metadata[:type] == type}
26
+ end
27
+
28
+ end
29
+ end
@@ -0,0 +1,9 @@
1
+ module SidekiqLogAnalyser
2
+ class LogFile
3
+ attr_reader :lines
4
+
5
+ def initialize(filename)
6
+ @lines = IO.readlines(filename)
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,39 @@
1
+ module SidekiqLogAnalyser
2
+ class PrettyPrint
3
+ SUMMARY_REPORT_TITLE = 'Summary'.colorize(:green)
4
+ AVG_REPORT_TITLE = 'Avg Performance / Time'.colorize(:green)
5
+
6
+ def PrettyPrint.print_summary_report(report)
7
+ table = Terminal::Table.new title: SUMMARY_REPORT_TITLE, headings: ['Worker', 'Count'] do |t|
8
+ report.each do |k, v|
9
+ t.add_row [k, v]
10
+ end
11
+ end
12
+ puts(table)
13
+ puts
14
+ end
15
+
16
+ def PrettyPrint.print_avg_report(report)
17
+ table = Terminal::Table.new title: AVG_REPORT_TITLE, headings: ['Worker', 'Avg (sec)'] do |t|
18
+ report.each do |k, v|
19
+ t.add_row [k, v.round(2)]
20
+ end
21
+ end
22
+ puts(table)
23
+ puts
24
+ end
25
+
26
+ def PrettyPrint.print_timeline_report(report)
27
+ report.each do |worker, data|
28
+ table = Terminal::Table.new title: worker.colorize(:green), headings: ['Worker', 'Count', 'Avg (sec)'] do |t|
29
+ data.each do |datetime, info|
30
+ t.add_row [datetime.strftime('%d-%m-%Y %H:%M'), info[:count], info[:avg].round(2)]
31
+ end
32
+ end
33
+ puts(table)
34
+ puts
35
+ end
36
+ end
37
+
38
+ end
39
+ end
@@ -0,0 +1,45 @@
1
+ module SidekiqLogAnalyser
2
+ class Row
3
+ attr_reader :row, :start_match, :end_match
4
+
5
+ def initialize(row)
6
+ @row = "#{row}"
7
+ @start_match = @row.match(/(.*Z) (.*) (T.*) (J.*) INFO: start/)
8
+ @end_match = @row.match(/(.*Z) (.*) (T.*) (J.*) INFO: done: (.*) sec/)
9
+ end
10
+
11
+ def start?
12
+ start_match != nil && start_match.length > 0
13
+ end
14
+
15
+ def end?
16
+ end_match != nil && end_match.length > 0
17
+ end
18
+
19
+ def valid?
20
+ start? || end?
21
+ end
22
+
23
+ def metadata
24
+ if start?
25
+ {
26
+ datetime: Time.parse(start_match[1]),
27
+ worker: start_match[3].split(' ').last,
28
+ job_id: start_match[4],
29
+ type: :start
30
+ }
31
+ elsif end?
32
+ {
33
+ datetime: Time.parse(end_match[1]),
34
+ worker: end_match[3].split(' ').last,
35
+ job_id: end_match[4],
36
+ duration: end_match[5].to_f,
37
+ type: :end
38
+ }
39
+ else
40
+ {}
41
+ end
42
+ end
43
+
44
+ end
45
+ end
@@ -0,0 +1,41 @@
1
+ module SidekiqLogAnalyser
2
+ class SimpleStat
3
+ extend Forwardable
4
+
5
+ attr_reader :collection
6
+
7
+ def_delegator :collection, :start_collection
8
+ def_delegator :collection, :end_collection
9
+
10
+ def initialize(collection)
11
+ @collection = collection
12
+ end
13
+
14
+ def total
15
+ result = {}
16
+ raw = end_collection
17
+ raw.each do |worker, data|
18
+ result[worker] = data.count
19
+ end
20
+ result
21
+ end
22
+
23
+ def avg
24
+ result = {}
25
+ raw = end_collection
26
+ raw.each do |worker, data|
27
+ result[worker] = data.sum{|metadata| metadata[:duration]}.to_f / data.length
28
+ end
29
+ result
30
+ end
31
+
32
+ def timeline
33
+ result = {}
34
+ raw = end_collection
35
+ raw.each do |worker, data|
36
+ result[worker] = Timeline.timeline(data)
37
+ end
38
+ result
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,14 @@
1
+ module SidekiqLogAnalyser
2
+ class Timeline
3
+ def Timeline.timeline(collection)
4
+ result = {}
5
+ collection.group_by_hour{|e| e[:datetime]}.each do |datetime, data|
6
+ result[datetime] = {
7
+ count: data.count,
8
+ avg: data.collect{|ee| ee[:duration]}.sum / data.count
9
+ }
10
+ end
11
+ result
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,3 @@
1
+ module SidekiqLogAnalyser
2
+ VERSION = "0.1"
3
+ end
@@ -0,0 +1,50 @@
1
+ require "spec_helper"
2
+
3
+ describe SidekiqLogAnalyser::Row do
4
+
5
+ let!(:line1) { "row2017-05-31T14:53:37.657Z 7947 TID-98ndc INFO: Booting Sidekiq 4.2.10 with redis options {:url=>nil}" }
6
+ let!(:line2) { "2017-05-31T15:00:05.753Z 7947 TID-sfsls UpdateCountersWorker JID-16a22bc4cb44c2d69ad536a9 INFO: start" }
7
+ let!(:line3) { "2017-05-31T15:00:06.590Z 7947 TID-sfsls UpdateCountersWorker JID-16a22bc4cb44c2d69ad536a9 INFO: done: 0.837 sec" }
8
+ let!(:line4) { "" }
9
+ let!(:line5) { nil }
10
+
11
+ it "matches start and end" do
12
+ row1 = SidekiqLogAnalyser::Row.new(line1)
13
+ expect(row1.start?).to eq(false)
14
+ expect(row1.end?).to eq(false)
15
+
16
+ row2 = SidekiqLogAnalyser::Row.new(line2)
17
+ expect(row2.start?).to eq(true)
18
+ expect(row2.end?).to eq(false)
19
+
20
+ row3 = SidekiqLogAnalyser::Row.new(line3)
21
+ expect(row3.start?).to eq(false)
22
+ expect(row3.end?).to eq(true)
23
+
24
+ row4 = SidekiqLogAnalyser::Row.new(line4)
25
+ expect(row4.start?).to eq(false)
26
+ expect(row4.end?).to eq(false)
27
+
28
+ row5 = SidekiqLogAnalyser::Row.new(line5)
29
+ expect(row5.start?).to eq(false)
30
+ expect(row5.end?).to eq(false)
31
+ end
32
+
33
+ it 'returns start_match' do
34
+ row = SidekiqLogAnalyser::Row.new(line2)
35
+ expect(row.metadata).to eq({:datetime=>Time.parse('2017-05-31T15:00:05.753Z'), :job_id=>"JID-16a22bc4cb44c2d69ad536a9", :worker=>"UpdateCountersWorker"})
36
+ end
37
+
38
+ it 'returns end_match' do
39
+ row = SidekiqLogAnalyser::Row.new(line3)
40
+ expect(row.metadata).to eq({:datetime=>Time.parse('2017-05-31T15:00:06.590Z'), :job_id=>"JID-16a22bc4cb44c2d69ad536a9", :worker=>"UpdateCountersWorker", :duration=>0.837})
41
+ end
42
+
43
+ it 'not fails metadata for bad data' do
44
+ row = SidekiqLogAnalyser::Row.new(line4)
45
+ expect(row.metadata).to eq({})
46
+
47
+ row = SidekiqLogAnalyser::Row.new(line5)
48
+ expect(row.metadata).to eq({})
49
+ end
50
+ end
@@ -0,0 +1,7 @@
1
+ require "spec_helper"
2
+
3
+ describe SidekiqLogAnalyser do
4
+ it "has a version number" do
5
+ expect(SidekiqLogAnalyser::VERSION).not_to be nil
6
+ end
7
+ end
@@ -0,0 +1,3 @@
1
+ $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__)
2
+ require "sidekiq_log_analyser"
3
+ require "pry"
metadata ADDED
@@ -0,0 +1,163 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sidekiq_log_analyser
3
+ version: !ruby/object:Gem::Version
4
+ version: '0.1'
5
+ platform: ruby
6
+ authors:
7
+ - Igor Kasyanchuk
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2018-08-08 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: groupdate
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: colorize
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: bundler
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.13'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.13'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '10.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '10.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rspec
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '3.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '3.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: pry
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ - !ruby/object:Gem::Dependency
98
+ name: terminal-table
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ description: Simple sidekiq log analyzer.
112
+ email:
113
+ - igorkasyanchuk@gmail.com
114
+ executables:
115
+ - sidekiq_log_analyser
116
+ extensions: []
117
+ extra_rdoc_files: []
118
+ files:
119
+ - Gemfile
120
+ - Gemfile.lock
121
+ - README.md
122
+ - Rakefile
123
+ - bin/console
124
+ - bin/setup
125
+ - bin/sidekiq_log_analyser
126
+ - lib/sidekiq_log_analyser.rb
127
+ - lib/sidekiq_log_analyser/app.rb
128
+ - lib/sidekiq_log_analyser/collection.rb
129
+ - lib/sidekiq_log_analyser/log_file.rb
130
+ - lib/sidekiq_log_analyser/pretty_print.rb
131
+ - lib/sidekiq_log_analyser/row.rb
132
+ - lib/sidekiq_log_analyser/simple_stat.rb
133
+ - lib/sidekiq_log_analyser/timeline.rb
134
+ - lib/sidekiq_log_analyser/version.rb
135
+ - spec/row_spec.rb
136
+ - spec/sidekiq_log_analyser_spec.rb
137
+ - spec/spec_helper.rb
138
+ homepage: https://github.com/igorkasyanchuk@gmail
139
+ licenses:
140
+ - MIT
141
+ metadata: {}
142
+ post_install_message:
143
+ rdoc_options: []
144
+ require_paths:
145
+ - lib
146
+ - bin
147
+ required_ruby_version: !ruby/object:Gem::Requirement
148
+ requirements:
149
+ - - ">="
150
+ - !ruby/object:Gem::Version
151
+ version: '0'
152
+ required_rubygems_version: !ruby/object:Gem::Requirement
153
+ requirements:
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: '0'
157
+ requirements: []
158
+ rubyforge_project:
159
+ rubygems_version: 2.6.14
160
+ signing_key:
161
+ specification_version: 4
162
+ summary: Simple sidekiq log analyzer.
163
+ test_files: []