roundhouse-x 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.travis.yml +16 -0
- data/3.0-Upgrade.md +70 -0
- data/Changes.md +1127 -0
- data/Gemfile +27 -0
- data/LICENSE +7 -0
- data/README.md +52 -0
- data/Rakefile +9 -0
- data/bin/roundhouse +19 -0
- data/bin/roundhousectl +93 -0
- data/lib/generators/roundhouse/templates/worker.rb.erb +9 -0
- data/lib/generators/roundhouse/templates/worker_spec.rb.erb +6 -0
- data/lib/generators/roundhouse/templates/worker_test.rb.erb +8 -0
- data/lib/generators/roundhouse/worker_generator.rb +49 -0
- data/lib/roundhouse/actor.rb +39 -0
- data/lib/roundhouse/api.rb +859 -0
- data/lib/roundhouse/cli.rb +396 -0
- data/lib/roundhouse/client.rb +210 -0
- data/lib/roundhouse/core_ext.rb +105 -0
- data/lib/roundhouse/exception_handler.rb +30 -0
- data/lib/roundhouse/fetch.rb +154 -0
- data/lib/roundhouse/launcher.rb +98 -0
- data/lib/roundhouse/logging.rb +104 -0
- data/lib/roundhouse/manager.rb +236 -0
- data/lib/roundhouse/middleware/chain.rb +149 -0
- data/lib/roundhouse/middleware/i18n.rb +41 -0
- data/lib/roundhouse/middleware/server/active_record.rb +13 -0
- data/lib/roundhouse/middleware/server/logging.rb +40 -0
- data/lib/roundhouse/middleware/server/retry_jobs.rb +206 -0
- data/lib/roundhouse/monitor.rb +124 -0
- data/lib/roundhouse/paginator.rb +42 -0
- data/lib/roundhouse/processor.rb +159 -0
- data/lib/roundhouse/rails.rb +24 -0
- data/lib/roundhouse/redis_connection.rb +77 -0
- data/lib/roundhouse/scheduled.rb +115 -0
- data/lib/roundhouse/testing/inline.rb +28 -0
- data/lib/roundhouse/testing.rb +193 -0
- data/lib/roundhouse/util.rb +68 -0
- data/lib/roundhouse/version.rb +3 -0
- data/lib/roundhouse/web.rb +264 -0
- data/lib/roundhouse/web_helpers.rb +249 -0
- data/lib/roundhouse/worker.rb +90 -0
- data/lib/roundhouse.rb +177 -0
- data/roundhouse.gemspec +27 -0
- data/test/config.yml +9 -0
- data/test/env_based_config.yml +11 -0
- data/test/fake_env.rb +0 -0
- data/test/fixtures/en.yml +2 -0
- data/test/helper.rb +49 -0
- data/test/test_api.rb +521 -0
- data/test/test_cli.rb +389 -0
- data/test/test_client.rb +294 -0
- data/test/test_exception_handler.rb +55 -0
- data/test/test_fetch.rb +206 -0
- data/test/test_logging.rb +34 -0
- data/test/test_manager.rb +169 -0
- data/test/test_middleware.rb +160 -0
- data/test/test_monitor.rb +258 -0
- data/test/test_processor.rb +176 -0
- data/test/test_rails.rb +23 -0
- data/test/test_redis_connection.rb +127 -0
- data/test/test_retry.rb +390 -0
- data/test/test_roundhouse.rb +87 -0
- data/test/test_scheduled.rb +120 -0
- data/test/test_scheduling.rb +75 -0
- data/test/test_testing.rb +78 -0
- data/test/test_testing_fake.rb +240 -0
- data/test/test_testing_inline.rb +65 -0
- data/test/test_util.rb +18 -0
- data/test/test_web.rb +605 -0
- data/test/test_web_helpers.rb +52 -0
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status/active.png +0 -0
- data/web/assets/images/status/idle.png +0 -0
- data/web/assets/images/status-sd8051fd480.png +0 -0
- data/web/assets/javascripts/application.js +83 -0
- data/web/assets/javascripts/dashboard.js +300 -0
- data/web/assets/javascripts/locales/README.md +27 -0
- data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
- data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
- data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
- data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
- data/web/assets/stylesheets/application.css +746 -0
- data/web/assets/stylesheets/bootstrap.css +9 -0
- data/web/locales/cs.yml +68 -0
- data/web/locales/da.yml +68 -0
- data/web/locales/de.yml +69 -0
- data/web/locales/el.yml +68 -0
- data/web/locales/en.yml +77 -0
- data/web/locales/es.yml +69 -0
- data/web/locales/fr.yml +69 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +69 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/no.yml +69 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +68 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +75 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/zh-cn.yml +68 -0
- data/web/locales/zh-tw.yml +68 -0
- data/web/views/_footer.erb +22 -0
- data/web/views/_job_info.erb +84 -0
- data/web/views/_nav.erb +66 -0
- data/web/views/_paging.erb +23 -0
- data/web/views/_poll_js.erb +5 -0
- data/web/views/_poll_link.erb +7 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +90 -0
- data/web/views/dashboard.erb +75 -0
- data/web/views/dead.erb +34 -0
- data/web/views/layout.erb +31 -0
- data/web/views/morgue.erb +71 -0
- data/web/views/queue.erb +45 -0
- data/web/views/queues.erb +27 -0
- data/web/views/retries.erb +74 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +54 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +404 -0
data/Gemfile
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
gemspec
|
3
|
+
|
4
|
+
gem 'rails', '~> 4.2'
|
5
|
+
gem 'simplecov'
|
6
|
+
gem 'minitest'
|
7
|
+
gem 'awesome_print', require: 'ap'
|
8
|
+
|
9
|
+
platforms :rbx do
|
10
|
+
gem 'rubysl', '~> 2.0' # if using anything in the ruby standard library
|
11
|
+
gem 'psych' # if using yaml
|
12
|
+
gem 'minitest' # if using minitest
|
13
|
+
gem 'rubinius-developer_tools' # if using any of coverage, debugger, profiler
|
14
|
+
end
|
15
|
+
|
16
|
+
platforms :ruby do
|
17
|
+
gem 'sqlite3'
|
18
|
+
end
|
19
|
+
|
20
|
+
platforms :mri do
|
21
|
+
gem 'pry-byebug'
|
22
|
+
end
|
23
|
+
|
24
|
+
platforms :jruby do
|
25
|
+
gem 'jruby-openssl'
|
26
|
+
gem 'activerecord-jdbcsqlite3-adapter'
|
27
|
+
end
|
data/LICENSE
ADDED
data/README.md
ADDED
@@ -0,0 +1,52 @@
|
|
1
|
+
Roundhouse
|
2
|
+
==============
|
3
|
+
|
4
|
+
Roundhouse is based on Sidekiq 3.5.0 (HEAD: f8ee0896076671b73739099e3c1eb7d8814e407d)
|
5
|
+
|
6
|
+
It uses the basic framework of a multi-threaded workers and managers to solve a different
|
7
|
+
kind of problem. This problem resembles that of a load balancing problem: only one worker
|
8
|
+
should be working on a work for a given user. This requires:
|
9
|
+
|
10
|
+
1. A queue for each user
|
11
|
+
2. A circular buffer to cycle through all user queues
|
12
|
+
3. A way to track how many pieces of jobs to process per user, before moving to the next user's queue.
|
13
|
+
|
14
|
+
Other things Roundhouse should be able to do:
|
15
|
+
|
16
|
+
1. It should be able to run side-by side with Sidekiq client
|
17
|
+
2. Be able to suspend, resume, and defer work
|
18
|
+
3. Metrics to measure lead times and give estimates on when work might be completed
|
19
|
+
4. Coordinate across a number of worker processes, across machines
|
20
|
+
|
21
|
+
Install
|
22
|
+
----------------
|
23
|
+
|
24
|
+
```
|
25
|
+
gem 'roundhouse-x', require: 'roundhouse'
|
26
|
+
```
|
27
|
+
|
28
|
+
Workers
|
29
|
+
----------------
|
30
|
+
|
31
|
+
```
|
32
|
+
class APIWorker
|
33
|
+
include Roundhouse::Worker
|
34
|
+
|
35
|
+
def perform(id)
|
36
|
+
# something
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
APIWorker.perform_async(api_token.id, item.id)
|
41
|
+
```
|
42
|
+
|
43
|
+
Sidekiq License
|
44
|
+
-----------------
|
45
|
+
|
46
|
+
Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
|
47
|
+
|
48
|
+
|
49
|
+
Sidekiq Original Author
|
50
|
+
-----------------
|
51
|
+
|
52
|
+
Mike Perham, [@mperham](https://twitter.com/mperham) / [@sidekiq](https://twitter.com/sidekiq), [http://www.mikeperham.com](http://www.mikeperham.com) / [http://www.contribsys.com](http://www.contribsys.com)
|
data/Rakefile
ADDED
data/bin/roundhouse
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
# Quiet some warnings we see when running in warning mode:
|
4
|
+
# RUBYOPT=-w bundle exec sidekiq
|
5
|
+
$TESTING = false
|
6
|
+
$CELLULOID_DEBUG = false
|
7
|
+
|
8
|
+
require_relative '../lib/roundhouse/cli'
|
9
|
+
|
10
|
+
begin
|
11
|
+
cli = Roundhouse::CLI.instance
|
12
|
+
cli.parse
|
13
|
+
cli.run
|
14
|
+
rescue => e
|
15
|
+
raise e if $DEBUG
|
16
|
+
STDERR.puts e.message
|
17
|
+
STDERR.puts e.backtrace.join("\n")
|
18
|
+
exit 1
|
19
|
+
end
|
data/bin/roundhousectl
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'fileutils'
|
4
|
+
|
5
|
+
class Roundhousectl
|
6
|
+
DEFAULT_KILL_TIMEOUT = 10
|
7
|
+
|
8
|
+
attr_reader :stage, :pidfile, :kill_timeout
|
9
|
+
|
10
|
+
def self.print_usage
|
11
|
+
puts "#{File.basename($0)} - stop a Roundhouse process from the command line."
|
12
|
+
puts
|
13
|
+
puts "Usage: #{File.basename($0)} <command> <pidfile> <kill_timeout>"
|
14
|
+
puts " where <command> is either 'quiet' or 'stop'"
|
15
|
+
puts " <pidfile> is path to a pidfile"
|
16
|
+
puts " <kill_timeout> is number of seconds to wait until Roundhouse exits"
|
17
|
+
puts " (default: #{Roundhousectl::DEFAULT_KILL_TIMEOUT}), after which Roundhouse will be KILL'd"
|
18
|
+
puts
|
19
|
+
puts "Be sure to set the kill_timeout LONGER than Roundhouse's -t timeout. If you want"
|
20
|
+
puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
|
21
|
+
puts " path_to_pidfile 61`"
|
22
|
+
puts
|
23
|
+
end
|
24
|
+
|
25
|
+
def initialize(stage, pidfile, timeout)
|
26
|
+
@stage = stage
|
27
|
+
@pidfile = pidfile
|
28
|
+
@kill_timeout = timeout
|
29
|
+
|
30
|
+
done('No pidfile given', :error) if !pidfile
|
31
|
+
done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
|
32
|
+
done('Invalid pidfile content', :error) if pid == 0
|
33
|
+
|
34
|
+
fetch_process
|
35
|
+
|
36
|
+
begin
|
37
|
+
send(stage)
|
38
|
+
rescue NoMethodError
|
39
|
+
done "Invalid command: #{stage}", :error
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def fetch_process
|
44
|
+
Process.getpgid(pid)
|
45
|
+
rescue Errno::ESRCH
|
46
|
+
done "Process doesn't exist", :error
|
47
|
+
end
|
48
|
+
|
49
|
+
def done(msg, error = nil)
|
50
|
+
puts msg
|
51
|
+
exit(exit_signal(error))
|
52
|
+
end
|
53
|
+
|
54
|
+
def exit_signal(error)
|
55
|
+
(error == :error) ? 1 : 0
|
56
|
+
end
|
57
|
+
|
58
|
+
def pid
|
59
|
+
@pid ||= File.read(pidfile).to_i
|
60
|
+
end
|
61
|
+
|
62
|
+
def quiet
|
63
|
+
Process.kill(:USR1, pid)
|
64
|
+
end
|
65
|
+
|
66
|
+
def stop
|
67
|
+
Process.kill(:TERM, pid)
|
68
|
+
kill_timeout.times do
|
69
|
+
begin
|
70
|
+
Process.getpgid(pid)
|
71
|
+
rescue Errno::ESRCH
|
72
|
+
FileUtils.rm_f pidfile
|
73
|
+
done 'Roundhouse shut down gracefully.'
|
74
|
+
end
|
75
|
+
sleep 1
|
76
|
+
end
|
77
|
+
Process.kill(:KILL, pid)
|
78
|
+
FileUtils.rm_f pidfile
|
79
|
+
done 'Roundhouse shut down forcefully.'
|
80
|
+
end
|
81
|
+
alias_method :shutdown, :stop
|
82
|
+
end
|
83
|
+
|
84
|
+
if ARGV.length < 2
|
85
|
+
Roundhousectl.print_usage
|
86
|
+
else
|
87
|
+
stage = ARGV[0]
|
88
|
+
pidfile = ARGV[1]
|
89
|
+
timeout = ARGV[2].to_i
|
90
|
+
timeout = Roundhousectl::DEFAULT_KILL_TIMEOUT if timeout == 0
|
91
|
+
|
92
|
+
Roundhousectl.new(stage, pidfile, timeout)
|
93
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'rails/generators/named_base'
|
2
|
+
|
3
|
+
module Roundhouse
|
4
|
+
module Generators # :nodoc:
|
5
|
+
class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
+
desc 'This generator creates a Roundhouse Worker in app/workers and a corresponding test'
|
7
|
+
|
8
|
+
check_class_collision suffix: 'Worker'
|
9
|
+
|
10
|
+
def self.default_generator_root
|
11
|
+
File.dirname(__FILE__)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_worker_file
|
15
|
+
template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_test_file
|
19
|
+
if defined?(RSpec)
|
20
|
+
create_worker_spec
|
21
|
+
else
|
22
|
+
create_worker_test
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
def create_worker_spec
|
29
|
+
template_file = File.join(
|
30
|
+
'spec/workers',
|
31
|
+
class_path,
|
32
|
+
"#{file_name}_worker_spec.rb"
|
33
|
+
)
|
34
|
+
template 'worker_spec.rb.erb', template_file
|
35
|
+
end
|
36
|
+
|
37
|
+
def create_worker_test
|
38
|
+
template_file = File.join(
|
39
|
+
'test/workers',
|
40
|
+
class_path,
|
41
|
+
"#{file_name}_worker_test.rb"
|
42
|
+
)
|
43
|
+
template 'worker_test.rb.erb', template_file
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
module Roundhouse
|
2
|
+
module Actor
|
3
|
+
|
4
|
+
module ClassMethods
|
5
|
+
def trap_exit(*args)
|
6
|
+
end
|
7
|
+
def new_link(*args)
|
8
|
+
new(*args)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
module InstanceMethods
|
13
|
+
def current_actor
|
14
|
+
self
|
15
|
+
end
|
16
|
+
def after(interval)
|
17
|
+
end
|
18
|
+
def alive?
|
19
|
+
@dead = false unless defined?(@dead)
|
20
|
+
!@dead
|
21
|
+
end
|
22
|
+
def terminate
|
23
|
+
@dead = true
|
24
|
+
end
|
25
|
+
def defer
|
26
|
+
yield
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.included(klass)
|
31
|
+
if $TESTING
|
32
|
+
klass.__send__(:include, InstanceMethods)
|
33
|
+
klass.__send__(:extend, ClassMethods)
|
34
|
+
else
|
35
|
+
klass.__send__(:include, Celluloid)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|