roundhouse-x 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.travis.yml +16 -0
- data/3.0-Upgrade.md +70 -0
- data/Changes.md +1127 -0
- data/Gemfile +27 -0
- data/LICENSE +7 -0
- data/README.md +52 -0
- data/Rakefile +9 -0
- data/bin/roundhouse +19 -0
- data/bin/roundhousectl +93 -0
- data/lib/generators/roundhouse/templates/worker.rb.erb +9 -0
- data/lib/generators/roundhouse/templates/worker_spec.rb.erb +6 -0
- data/lib/generators/roundhouse/templates/worker_test.rb.erb +8 -0
- data/lib/generators/roundhouse/worker_generator.rb +49 -0
- data/lib/roundhouse/actor.rb +39 -0
- data/lib/roundhouse/api.rb +859 -0
- data/lib/roundhouse/cli.rb +396 -0
- data/lib/roundhouse/client.rb +210 -0
- data/lib/roundhouse/core_ext.rb +105 -0
- data/lib/roundhouse/exception_handler.rb +30 -0
- data/lib/roundhouse/fetch.rb +154 -0
- data/lib/roundhouse/launcher.rb +98 -0
- data/lib/roundhouse/logging.rb +104 -0
- data/lib/roundhouse/manager.rb +236 -0
- data/lib/roundhouse/middleware/chain.rb +149 -0
- data/lib/roundhouse/middleware/i18n.rb +41 -0
- data/lib/roundhouse/middleware/server/active_record.rb +13 -0
- data/lib/roundhouse/middleware/server/logging.rb +40 -0
- data/lib/roundhouse/middleware/server/retry_jobs.rb +206 -0
- data/lib/roundhouse/monitor.rb +124 -0
- data/lib/roundhouse/paginator.rb +42 -0
- data/lib/roundhouse/processor.rb +159 -0
- data/lib/roundhouse/rails.rb +24 -0
- data/lib/roundhouse/redis_connection.rb +77 -0
- data/lib/roundhouse/scheduled.rb +115 -0
- data/lib/roundhouse/testing/inline.rb +28 -0
- data/lib/roundhouse/testing.rb +193 -0
- data/lib/roundhouse/util.rb +68 -0
- data/lib/roundhouse/version.rb +3 -0
- data/lib/roundhouse/web.rb +264 -0
- data/lib/roundhouse/web_helpers.rb +249 -0
- data/lib/roundhouse/worker.rb +90 -0
- data/lib/roundhouse.rb +177 -0
- data/roundhouse.gemspec +27 -0
- data/test/config.yml +9 -0
- data/test/env_based_config.yml +11 -0
- data/test/fake_env.rb +0 -0
- data/test/fixtures/en.yml +2 -0
- data/test/helper.rb +49 -0
- data/test/test_api.rb +521 -0
- data/test/test_cli.rb +389 -0
- data/test/test_client.rb +294 -0
- data/test/test_exception_handler.rb +55 -0
- data/test/test_fetch.rb +206 -0
- data/test/test_logging.rb +34 -0
- data/test/test_manager.rb +169 -0
- data/test/test_middleware.rb +160 -0
- data/test/test_monitor.rb +258 -0
- data/test/test_processor.rb +176 -0
- data/test/test_rails.rb +23 -0
- data/test/test_redis_connection.rb +127 -0
- data/test/test_retry.rb +390 -0
- data/test/test_roundhouse.rb +87 -0
- data/test/test_scheduled.rb +120 -0
- data/test/test_scheduling.rb +75 -0
- data/test/test_testing.rb +78 -0
- data/test/test_testing_fake.rb +240 -0
- data/test/test_testing_inline.rb +65 -0
- data/test/test_util.rb +18 -0
- data/test/test_web.rb +605 -0
- data/test/test_web_helpers.rb +52 -0
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status/active.png +0 -0
- data/web/assets/images/status/idle.png +0 -0
- data/web/assets/images/status-sd8051fd480.png +0 -0
- data/web/assets/javascripts/application.js +83 -0
- data/web/assets/javascripts/dashboard.js +300 -0
- data/web/assets/javascripts/locales/README.md +27 -0
- data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
- data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
- data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
- data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
- data/web/assets/stylesheets/application.css +746 -0
- data/web/assets/stylesheets/bootstrap.css +9 -0
- data/web/locales/cs.yml +68 -0
- data/web/locales/da.yml +68 -0
- data/web/locales/de.yml +69 -0
- data/web/locales/el.yml +68 -0
- data/web/locales/en.yml +77 -0
- data/web/locales/es.yml +69 -0
- data/web/locales/fr.yml +69 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +69 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/no.yml +69 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +68 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +75 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/zh-cn.yml +68 -0
- data/web/locales/zh-tw.yml +68 -0
- data/web/views/_footer.erb +22 -0
- data/web/views/_job_info.erb +84 -0
- data/web/views/_nav.erb +66 -0
- data/web/views/_paging.erb +23 -0
- data/web/views/_poll_js.erb +5 -0
- data/web/views/_poll_link.erb +7 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +90 -0
- data/web/views/dashboard.erb +75 -0
- data/web/views/dead.erb +34 -0
- data/web/views/layout.erb +31 -0
- data/web/views/morgue.erb +71 -0
- data/web/views/queue.erb +45 -0
- data/web/views/queues.erb +27 -0
- data/web/views/retries.erb +74 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +54 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +404 -0
data/Gemfile
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
source 'https://rubygems.org'
|
|
2
|
+
gemspec
|
|
3
|
+
|
|
4
|
+
gem 'rails', '~> 4.2'
|
|
5
|
+
gem 'simplecov'
|
|
6
|
+
gem 'minitest'
|
|
7
|
+
gem 'awesome_print', require: 'ap'
|
|
8
|
+
|
|
9
|
+
platforms :rbx do
|
|
10
|
+
gem 'rubysl', '~> 2.0' # if using anything in the ruby standard library
|
|
11
|
+
gem 'psych' # if using yaml
|
|
12
|
+
gem 'minitest' # if using minitest
|
|
13
|
+
gem 'rubinius-developer_tools' # if using any of coverage, debugger, profiler
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
platforms :ruby do
|
|
17
|
+
gem 'sqlite3'
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
platforms :mri do
|
|
21
|
+
gem 'pry-byebug'
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
platforms :jruby do
|
|
25
|
+
gem 'jruby-openssl'
|
|
26
|
+
gem 'activerecord-jdbcsqlite3-adapter'
|
|
27
|
+
end
|
data/LICENSE
ADDED
data/README.md
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
Roundhouse
|
|
2
|
+
==============
|
|
3
|
+
|
|
4
|
+
Roundhouse is based on Sidekiq 3.5.0 (HEAD: f8ee0896076671b73739099e3c1eb7d8814e407d)
|
|
5
|
+
|
|
6
|
+
It uses the basic framework of a multi-threaded workers and managers to solve a different
|
|
7
|
+
kind of problem. This problem resembles that of a load balancing problem: only one worker
|
|
8
|
+
should be working on a work for a given user. This requires:
|
|
9
|
+
|
|
10
|
+
1. A queue for each user
|
|
11
|
+
2. A circular buffer to cycle through all user queues
|
|
12
|
+
3. A way to track how many pieces of jobs to process per user, before moving to the next user's queue.
|
|
13
|
+
|
|
14
|
+
Other things Roundhouse should be able to do:
|
|
15
|
+
|
|
16
|
+
1. It should be able to run side-by side with Sidekiq client
|
|
17
|
+
2. Be able to suspend, resume, and defer work
|
|
18
|
+
3. Metrics to measure lead times and give estimates on when work might be completed
|
|
19
|
+
4. Coordinate across a number of worker processes, across machines
|
|
20
|
+
|
|
21
|
+
Install
|
|
22
|
+
----------------
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
gem 'roundhouse-x', require: 'roundhouse'
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Workers
|
|
29
|
+
----------------
|
|
30
|
+
|
|
31
|
+
```
|
|
32
|
+
class APIWorker
|
|
33
|
+
include Roundhouse::Worker
|
|
34
|
+
|
|
35
|
+
def perform(id)
|
|
36
|
+
# something
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
APIWorker.perform_async(api_token.id, item.id)
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Sidekiq License
|
|
44
|
+
-----------------
|
|
45
|
+
|
|
46
|
+
Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
Sidekiq Original Author
|
|
50
|
+
-----------------
|
|
51
|
+
|
|
52
|
+
Mike Perham, [@mperham](https://twitter.com/mperham) / [@sidekiq](https://twitter.com/sidekiq), [http://www.mikeperham.com](http://www.mikeperham.com) / [http://www.contribsys.com](http://www.contribsys.com)
|
data/Rakefile
ADDED
data/bin/roundhouse
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
# Quiet some warnings we see when running in warning mode:
|
|
4
|
+
# RUBYOPT=-w bundle exec sidekiq
|
|
5
|
+
$TESTING = false
|
|
6
|
+
$CELLULOID_DEBUG = false
|
|
7
|
+
|
|
8
|
+
require_relative '../lib/roundhouse/cli'
|
|
9
|
+
|
|
10
|
+
begin
|
|
11
|
+
cli = Roundhouse::CLI.instance
|
|
12
|
+
cli.parse
|
|
13
|
+
cli.run
|
|
14
|
+
rescue => e
|
|
15
|
+
raise e if $DEBUG
|
|
16
|
+
STDERR.puts e.message
|
|
17
|
+
STDERR.puts e.backtrace.join("\n")
|
|
18
|
+
exit 1
|
|
19
|
+
end
|
data/bin/roundhousectl
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
|
|
5
|
+
class Roundhousectl
|
|
6
|
+
DEFAULT_KILL_TIMEOUT = 10
|
|
7
|
+
|
|
8
|
+
attr_reader :stage, :pidfile, :kill_timeout
|
|
9
|
+
|
|
10
|
+
def self.print_usage
|
|
11
|
+
puts "#{File.basename($0)} - stop a Roundhouse process from the command line."
|
|
12
|
+
puts
|
|
13
|
+
puts "Usage: #{File.basename($0)} <command> <pidfile> <kill_timeout>"
|
|
14
|
+
puts " where <command> is either 'quiet' or 'stop'"
|
|
15
|
+
puts " <pidfile> is path to a pidfile"
|
|
16
|
+
puts " <kill_timeout> is number of seconds to wait until Roundhouse exits"
|
|
17
|
+
puts " (default: #{Roundhousectl::DEFAULT_KILL_TIMEOUT}), after which Roundhouse will be KILL'd"
|
|
18
|
+
puts
|
|
19
|
+
puts "Be sure to set the kill_timeout LONGER than Roundhouse's -t timeout. If you want"
|
|
20
|
+
puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
|
|
21
|
+
puts " path_to_pidfile 61`"
|
|
22
|
+
puts
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def initialize(stage, pidfile, timeout)
|
|
26
|
+
@stage = stage
|
|
27
|
+
@pidfile = pidfile
|
|
28
|
+
@kill_timeout = timeout
|
|
29
|
+
|
|
30
|
+
done('No pidfile given', :error) if !pidfile
|
|
31
|
+
done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
|
|
32
|
+
done('Invalid pidfile content', :error) if pid == 0
|
|
33
|
+
|
|
34
|
+
fetch_process
|
|
35
|
+
|
|
36
|
+
begin
|
|
37
|
+
send(stage)
|
|
38
|
+
rescue NoMethodError
|
|
39
|
+
done "Invalid command: #{stage}", :error
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def fetch_process
|
|
44
|
+
Process.getpgid(pid)
|
|
45
|
+
rescue Errno::ESRCH
|
|
46
|
+
done "Process doesn't exist", :error
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def done(msg, error = nil)
|
|
50
|
+
puts msg
|
|
51
|
+
exit(exit_signal(error))
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def exit_signal(error)
|
|
55
|
+
(error == :error) ? 1 : 0
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def pid
|
|
59
|
+
@pid ||= File.read(pidfile).to_i
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def quiet
|
|
63
|
+
Process.kill(:USR1, pid)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def stop
|
|
67
|
+
Process.kill(:TERM, pid)
|
|
68
|
+
kill_timeout.times do
|
|
69
|
+
begin
|
|
70
|
+
Process.getpgid(pid)
|
|
71
|
+
rescue Errno::ESRCH
|
|
72
|
+
FileUtils.rm_f pidfile
|
|
73
|
+
done 'Roundhouse shut down gracefully.'
|
|
74
|
+
end
|
|
75
|
+
sleep 1
|
|
76
|
+
end
|
|
77
|
+
Process.kill(:KILL, pid)
|
|
78
|
+
FileUtils.rm_f pidfile
|
|
79
|
+
done 'Roundhouse shut down forcefully.'
|
|
80
|
+
end
|
|
81
|
+
alias_method :shutdown, :stop
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
if ARGV.length < 2
|
|
85
|
+
Roundhousectl.print_usage
|
|
86
|
+
else
|
|
87
|
+
stage = ARGV[0]
|
|
88
|
+
pidfile = ARGV[1]
|
|
89
|
+
timeout = ARGV[2].to_i
|
|
90
|
+
timeout = Roundhousectl::DEFAULT_KILL_TIMEOUT if timeout == 0
|
|
91
|
+
|
|
92
|
+
Roundhousectl.new(stage, pidfile, timeout)
|
|
93
|
+
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
require 'rails/generators/named_base'
|
|
2
|
+
|
|
3
|
+
module Roundhouse
|
|
4
|
+
module Generators # :nodoc:
|
|
5
|
+
class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
|
6
|
+
desc 'This generator creates a Roundhouse Worker in app/workers and a corresponding test'
|
|
7
|
+
|
|
8
|
+
check_class_collision suffix: 'Worker'
|
|
9
|
+
|
|
10
|
+
def self.default_generator_root
|
|
11
|
+
File.dirname(__FILE__)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def create_worker_file
|
|
15
|
+
template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def create_test_file
|
|
19
|
+
if defined?(RSpec)
|
|
20
|
+
create_worker_spec
|
|
21
|
+
else
|
|
22
|
+
create_worker_test
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def create_worker_spec
|
|
29
|
+
template_file = File.join(
|
|
30
|
+
'spec/workers',
|
|
31
|
+
class_path,
|
|
32
|
+
"#{file_name}_worker_spec.rb"
|
|
33
|
+
)
|
|
34
|
+
template 'worker_spec.rb.erb', template_file
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def create_worker_test
|
|
38
|
+
template_file = File.join(
|
|
39
|
+
'test/workers',
|
|
40
|
+
class_path,
|
|
41
|
+
"#{file_name}_worker_test.rb"
|
|
42
|
+
)
|
|
43
|
+
template 'worker_test.rb.erb', template_file
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
module Roundhouse
|
|
2
|
+
module Actor
|
|
3
|
+
|
|
4
|
+
module ClassMethods
|
|
5
|
+
def trap_exit(*args)
|
|
6
|
+
end
|
|
7
|
+
def new_link(*args)
|
|
8
|
+
new(*args)
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
module InstanceMethods
|
|
13
|
+
def current_actor
|
|
14
|
+
self
|
|
15
|
+
end
|
|
16
|
+
def after(interval)
|
|
17
|
+
end
|
|
18
|
+
def alive?
|
|
19
|
+
@dead = false unless defined?(@dead)
|
|
20
|
+
!@dead
|
|
21
|
+
end
|
|
22
|
+
def terminate
|
|
23
|
+
@dead = true
|
|
24
|
+
end
|
|
25
|
+
def defer
|
|
26
|
+
yield
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.included(klass)
|
|
31
|
+
if $TESTING
|
|
32
|
+
klass.__send__(:include, InstanceMethods)
|
|
33
|
+
klass.__send__(:extend, ClassMethods)
|
|
34
|
+
else
|
|
35
|
+
klass.__send__(:include, Celluloid)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|