sidekiq-pool 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 69b664da7b6e3f336f809887dc3f99a997c40faf
4
+ data.tar.gz: 17673028467d9cbdb6d1b4ce99813d42b8ab6470
5
+ SHA512:
6
+ metadata.gz: 1de8801d8ea2cc4493dca8b8e2649bfc03c87d9535f5fff17c6c8b689d9e86f1d24f8be202e883d39a79fc819f01a722072402791c93c7792ae3a1e31d69ba00
7
+ data.tar.gz: 4745c45574e730406e280c14c2d7273a87eef5c245364144ca77d889e19807c65250998949cca5354f716d0234cfc52278a04220ca97171e1fb9b7d5d58bf0b0
data/.gitignore ADDED
@@ -0,0 +1,10 @@
1
+ /.ruby-version
2
+ /.bundle/
3
+ /.yardoc
4
+ /Gemfile.lock
5
+ /_yardoc/
6
+ /coverage/
7
+ /doc/
8
+ /pkg/
9
+ /spec/reports/
10
+ /tmp/
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --require spec_helper
2
+ --format documentation
3
+ --color
data/.travis.yml ADDED
@@ -0,0 +1,4 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.1.3
4
+ before_install: gem install bundler -v 1.11.2
data/Gemfile ADDED
@@ -0,0 +1,8 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in sidekiq-pool.gemspec
4
+ gemspec
5
+
6
+ group :development, :test do
7
+ gem 'pry'
8
+ end
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2016 Laurynas Butkus
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,42 @@
1
+ # Sidekiq::Pool
2
+
3
+ Allows using more CPU cores with Sidekiq on Ruby MRI by forking multiple processes.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'sidekiq-pool'
11
+ ```
12
+
13
+ ## Usage
14
+
15
+ Help
16
+
17
+ $ bundle exec sidekiq-pool -h
18
+
19
+ Start pool with 3 instances
20
+
21
+ $ bundle exec sidekiq-pool --pool-size 3
22
+
23
+ ## Signals
24
+
25
+ Signals `USR1`, `USR2` are forwarded to the children.
26
+
27
+ `TTIN` forks new child
28
+
29
+ $ kill -TTIN masterpid
30
+
31
+ `TTOU` kills one child
32
+
33
+ $ kill -TTOU masterpid
34
+
35
+ ## Contributing
36
+
37
+ Bug reports and pull requests are welcome on GitHub at https://github.com/laurynas/sidekiq-pool.
38
+
39
+
40
+ ## License
41
+
42
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require "rspec/core/rake_task"
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/sidekiq-pool ADDED
@@ -0,0 +1,18 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Quiet some warnings we see when running in warning mode:
4
+ # RUBYOPT=-w bundle exec sidekiq-pool
5
+ $TESTING = false
6
+
7
+ require_relative '../lib/sidekiq/pool/cli'
8
+
9
+ begin
10
+ cli = Sidekiq::Pool::CLI.instance
11
+ cli.parse
12
+ cli.run
13
+ rescue => e
14
+ raise e if $DEBUG
15
+ STDERR.puts e.message
16
+ STDERR.puts e.backtrace.join("\n")
17
+ exit 1
18
+ end
@@ -0,0 +1,50 @@
1
+ # Contributor Code of Conduct
2
+
3
+ As contributors and maintainers of this project, and in the interest of
4
+ fostering an open and welcoming community, we pledge to respect all people who
5
+ contribute through reporting issues, posting feature requests, updating
6
+ documentation, submitting pull requests or patches, and other activities.
7
+
8
+ We are committed to making participation in this project a harassment-free
9
+ experience for everyone, regardless of level of experience, gender, gender
10
+ identity and expression, sexual orientation, disability, personal appearance,
11
+ body size, race, ethnicity, age, religion, or nationality.
12
+
13
+ Examples of unacceptable behavior by participants include:
14
+
15
+ * The use of sexualized language or imagery
16
+ * Personal attacks
17
+ * Trolling or insulting/derogatory comments
18
+ * Public or private harassment
19
+ * Publishing other's private information, such as physical or electronic
20
+ addresses, without explicit permission
21
+ * Other unethical or unprofessional conduct
22
+
23
+ Project maintainers have the right and responsibility to remove, edit, or
24
+ reject comments, commits, code, wiki edits, issues, and other contributions
25
+ that are not aligned to this Code of Conduct, or to ban temporarily or
26
+ permanently any contributor for other behaviors that they deem inappropriate,
27
+ threatening, offensive, or harmful.
28
+
29
+ By adopting this Code of Conduct, project maintainers commit themselves to
30
+ fairly and consistently applying these principles to every aspect of managing
31
+ this project. Project maintainers who do not follow or enforce the Code of
32
+ Conduct may be permanently removed from the project team.
33
+
34
+ This Code of Conduct applies both within project spaces and in public spaces
35
+ when an individual is representing the project or its community.
36
+
37
+ Instances of abusive, harassing, or otherwise unacceptable behavior may be
38
+ reported by contacting the project maintainer at mperham AT gmail.com. All
39
+ complaints will be reviewed and investigated and will result in a response that
40
+ is deemed necessary and appropriate to the circumstances. Maintainers are
41
+ obligated to maintain confidentiality with regard to the reporter of an
42
+ incident.
43
+
44
+
45
+ This Code of Conduct is adapted from the [Contributor Covenant][homepage],
46
+ version 1.3.0, available at
47
+ [http://contributor-covenant.org/version/1/3/0/][version]
48
+
49
+ [homepage]: http://contributor-covenant.org
50
+ [version]: http://contributor-covenant.org/version/1/3/0/
@@ -0,0 +1,7 @@
1
+ require 'sidekiq/pool/version'
2
+ require 'sidekiq/pool/cli'
3
+
4
+ module Sidekiq
5
+ module Pool
6
+ end
7
+ end
@@ -0,0 +1,185 @@
1
+ require 'sidekiq/cli'
2
+
3
+ module Sidekiq
4
+ module Pool
5
+ class CLI < Sidekiq::CLI
6
+ def initialize
7
+ @child_index = 0
8
+ @pool = []
9
+ super
10
+ end
11
+
12
+ alias_method :run_child, :run
13
+
14
+ def run
15
+ raise ArgumentError, 'Please specify pool size' unless @pool_size
16
+
17
+ self_read, self_write = IO.pipe
18
+
19
+ %w(INT TERM USR1 USR2 TTIN TTOU CLD).each do |sig|
20
+ begin
21
+ trap sig do
22
+ self_write.puts(sig)
23
+ end
24
+ rescue ArgumentError
25
+ puts "Signal #{sig} not supported"
26
+ end
27
+ end
28
+
29
+ logger.info "Starting pool with #{@pool_size} instances"
30
+
31
+ @pool_size.times { fork_child }
32
+
33
+ while readable_io = IO.select([self_read])
34
+ signal = readable_io.first[0].gets.strip
35
+ handle_master_signal(signal)
36
+ end
37
+ end
38
+
39
+ private
40
+
41
+ def parse_options(argv)
42
+ opts = {}
43
+
44
+ @parser = OptionParser.new do |o|
45
+ o.on '--pool-size INT', "pool size" do |arg|
46
+ @pool_size = Integer(arg)
47
+ end
48
+
49
+ o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
50
+ opts[:concurrency] = Integer(arg)
51
+ end
52
+
53
+ o.on '-d', '--daemon', "Daemonize process" do |arg|
54
+ opts[:daemon] = arg
55
+ end
56
+
57
+ o.on '-e', '--environment ENV', "Application environment" do |arg|
58
+ opts[:environment] = arg
59
+ end
60
+
61
+ o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
62
+ opts[:tag] = arg
63
+ end
64
+
65
+ o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
66
+ queue, weight = arg.split(",")
67
+ parse_queue opts, queue, weight
68
+ end
69
+
70
+ o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
71
+ opts[:require] = arg
72
+ end
73
+
74
+ o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
75
+ opts[:timeout] = Integer(arg)
76
+ end
77
+
78
+ o.on "-v", "--verbose", "Print more verbose output" do |arg|
79
+ opts[:verbose] = arg
80
+ end
81
+
82
+ o.on '-C', '--config PATH', "path to YAML config file" do |arg|
83
+ opts[:config_file] = arg
84
+ end
85
+
86
+ o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
87
+ opts[:logfile] = arg
88
+ end
89
+
90
+ o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
91
+ opts[:pidfile] = arg
92
+ end
93
+
94
+ o.on '-V', '--version', "Print version and exit" do |arg|
95
+ puts "Sidekiq #{Sidekiq::VERSION}"
96
+ die(0)
97
+ end
98
+ end
99
+
100
+ @parser.banner = 'sidekiq-pool [options]'
101
+ @parser.on_tail '-h', '--help', 'Show help' do
102
+ logger.info @parser
103
+ die 1
104
+ end
105
+ @parser.parse!(argv)
106
+
107
+ %w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
108
+ opts[:config_file] ||= filename if File.exist?(filename)
109
+ end
110
+
111
+ opts
112
+ end
113
+
114
+ def handle_master_signal(sig)
115
+ case sig
116
+ when 'INT', 'TERM'
117
+ signal_to_pool(sig)
118
+ wait_and_exit
119
+ when 'TTIN'
120
+ logger.info 'Adding child'
121
+ fork_child
122
+ when 'TTOU'
123
+ remove_child
124
+ when 'CLD'
125
+ check_pool
126
+ else
127
+ signal_to_pool(sig)
128
+ end
129
+ end
130
+
131
+ def fork_child
132
+ @pool << fork do
133
+ options[:index] = @child_index++
134
+ run_child
135
+ end
136
+ end
137
+
138
+ def remove_child
139
+ return if @pool.empty?
140
+
141
+ if @pool.size == 1
142
+ logger.info 'Cowardly refusing to kill the last child'
143
+ return
144
+ end
145
+
146
+ logger.info 'Removing child'
147
+ ::Process.kill('TERM', @pool.pop)
148
+ end
149
+
150
+ def signal_to_pool(sig)
151
+ logger.info "Sending #{sig} signal to the pool"
152
+ @pool.each { |pid| ::Process.kill(sig, pid) }
153
+ end
154
+
155
+ def check_pool
156
+ @pool.each do |pid|
157
+ next if alive?(pid)
158
+ handle_dead_child(pid)
159
+ end
160
+ end
161
+
162
+ def handle_dead_child(pid)
163
+ logger.info "Child #{pid} died"
164
+ @pool.delete(pid)
165
+ end
166
+
167
+ def alive?(pid)
168
+ ::Process.getpgid(pid)
169
+ true
170
+ rescue Errno::ESRCH
171
+ false
172
+ end
173
+
174
+ def wait_and_exit
175
+ loop do
176
+ break if @pool.none? { |pid| alive?(pid) }
177
+ sleep(0.1)
178
+ end
179
+
180
+ logger.info 'Bye!'
181
+ exit(0)
182
+ end
183
+ end
184
+ end
185
+ end
@@ -0,0 +1,5 @@
1
+ module Sidekiq
2
+ module Pool
3
+ VERSION = '0.1.0'
4
+ end
5
+ end
@@ -0,0 +1,26 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'sidekiq/pool/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'sidekiq-pool'
8
+ spec.version = Sidekiq::Pool::VERSION
9
+ spec.authors = ['Laurynas Butkus']
10
+ spec.email = ['laurynas.butkus@gmail.com']
11
+
12
+ spec.summary = %q{Forks and manages multiple Sidekiq processes}
13
+ spec.description = %q{Allows using more CPU cores with Sidekiq on Ruby MRI by forking multiple processes.}
14
+ spec.homepage = 'https://github.com/laurynas/sidekiq-pool'
15
+ spec.license = 'MIT'
16
+
17
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
18
+ spec.executables = ['sidekiq-pool']
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_dependency 'sidekiq', '~> 4.0'
22
+
23
+ spec.add_development_dependency 'bundler', '~> 1.11'
24
+ spec.add_development_dependency 'rake', '~> 10.0'
25
+ spec.add_development_dependency 'rspec', '~> 3.0'
26
+ end
metadata ADDED
@@ -0,0 +1,115 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sidekiq-pool
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Laurynas Butkus
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2016-02-04 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: sidekiq
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '4.0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '4.0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.11'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.11'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '10.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '10.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '3.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '3.0'
69
+ description: Allows using more CPU cores with Sidekiq on Ruby MRI by forking multiple
70
+ processes.
71
+ email:
72
+ - laurynas.butkus@gmail.com
73
+ executables:
74
+ - sidekiq-pool
75
+ extensions: []
76
+ extra_rdoc_files: []
77
+ files:
78
+ - ".gitignore"
79
+ - ".rspec"
80
+ - ".travis.yml"
81
+ - Gemfile
82
+ - LICENSE.txt
83
+ - README.md
84
+ - Rakefile
85
+ - bin/sidekiq-pool
86
+ - code_of_conduct.md
87
+ - lib/sidekiq/pool.rb
88
+ - lib/sidekiq/pool/cli.rb
89
+ - lib/sidekiq/pool/version.rb
90
+ - sidekiq-pool.gemspec
91
+ homepage: https://github.com/laurynas/sidekiq-pool
92
+ licenses:
93
+ - MIT
94
+ metadata: {}
95
+ post_install_message:
96
+ rdoc_options: []
97
+ require_paths:
98
+ - lib
99
+ required_ruby_version: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ required_rubygems_version: !ruby/object:Gem::Requirement
105
+ requirements:
106
+ - - ">="
107
+ - !ruby/object:Gem::Version
108
+ version: '0'
109
+ requirements: []
110
+ rubyforge_project:
111
+ rubygems_version: 2.4.5
112
+ signing_key:
113
+ specification_version: 4
114
+ summary: Forks and manages multiple Sidekiq processes
115
+ test_files: []