roundhouse-x 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (168) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/.travis.yml +16 -0
  4. data/3.0-Upgrade.md +70 -0
  5. data/Changes.md +1127 -0
  6. data/Gemfile +27 -0
  7. data/LICENSE +7 -0
  8. data/README.md +52 -0
  9. data/Rakefile +9 -0
  10. data/bin/roundhouse +19 -0
  11. data/bin/roundhousectl +93 -0
  12. data/lib/generators/roundhouse/templates/worker.rb.erb +9 -0
  13. data/lib/generators/roundhouse/templates/worker_spec.rb.erb +6 -0
  14. data/lib/generators/roundhouse/templates/worker_test.rb.erb +8 -0
  15. data/lib/generators/roundhouse/worker_generator.rb +49 -0
  16. data/lib/roundhouse/actor.rb +39 -0
  17. data/lib/roundhouse/api.rb +859 -0
  18. data/lib/roundhouse/cli.rb +396 -0
  19. data/lib/roundhouse/client.rb +210 -0
  20. data/lib/roundhouse/core_ext.rb +105 -0
  21. data/lib/roundhouse/exception_handler.rb +30 -0
  22. data/lib/roundhouse/fetch.rb +154 -0
  23. data/lib/roundhouse/launcher.rb +98 -0
  24. data/lib/roundhouse/logging.rb +104 -0
  25. data/lib/roundhouse/manager.rb +236 -0
  26. data/lib/roundhouse/middleware/chain.rb +149 -0
  27. data/lib/roundhouse/middleware/i18n.rb +41 -0
  28. data/lib/roundhouse/middleware/server/active_record.rb +13 -0
  29. data/lib/roundhouse/middleware/server/logging.rb +40 -0
  30. data/lib/roundhouse/middleware/server/retry_jobs.rb +206 -0
  31. data/lib/roundhouse/monitor.rb +124 -0
  32. data/lib/roundhouse/paginator.rb +42 -0
  33. data/lib/roundhouse/processor.rb +159 -0
  34. data/lib/roundhouse/rails.rb +24 -0
  35. data/lib/roundhouse/redis_connection.rb +77 -0
  36. data/lib/roundhouse/scheduled.rb +115 -0
  37. data/lib/roundhouse/testing/inline.rb +28 -0
  38. data/lib/roundhouse/testing.rb +193 -0
  39. data/lib/roundhouse/util.rb +68 -0
  40. data/lib/roundhouse/version.rb +3 -0
  41. data/lib/roundhouse/web.rb +264 -0
  42. data/lib/roundhouse/web_helpers.rb +249 -0
  43. data/lib/roundhouse/worker.rb +90 -0
  44. data/lib/roundhouse.rb +177 -0
  45. data/roundhouse.gemspec +27 -0
  46. data/test/config.yml +9 -0
  47. data/test/env_based_config.yml +11 -0
  48. data/test/fake_env.rb +0 -0
  49. data/test/fixtures/en.yml +2 -0
  50. data/test/helper.rb +49 -0
  51. data/test/test_api.rb +521 -0
  52. data/test/test_cli.rb +389 -0
  53. data/test/test_client.rb +294 -0
  54. data/test/test_exception_handler.rb +55 -0
  55. data/test/test_fetch.rb +206 -0
  56. data/test/test_logging.rb +34 -0
  57. data/test/test_manager.rb +169 -0
  58. data/test/test_middleware.rb +160 -0
  59. data/test/test_monitor.rb +258 -0
  60. data/test/test_processor.rb +176 -0
  61. data/test/test_rails.rb +23 -0
  62. data/test/test_redis_connection.rb +127 -0
  63. data/test/test_retry.rb +390 -0
  64. data/test/test_roundhouse.rb +87 -0
  65. data/test/test_scheduled.rb +120 -0
  66. data/test/test_scheduling.rb +75 -0
  67. data/test/test_testing.rb +78 -0
  68. data/test/test_testing_fake.rb +240 -0
  69. data/test/test_testing_inline.rb +65 -0
  70. data/test/test_util.rb +18 -0
  71. data/test/test_web.rb +605 -0
  72. data/test/test_web_helpers.rb +52 -0
  73. data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
  74. data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
  75. data/web/assets/images/logo.png +0 -0
  76. data/web/assets/images/status/active.png +0 -0
  77. data/web/assets/images/status/idle.png +0 -0
  78. data/web/assets/images/status-sd8051fd480.png +0 -0
  79. data/web/assets/javascripts/application.js +83 -0
  80. data/web/assets/javascripts/dashboard.js +300 -0
  81. data/web/assets/javascripts/locales/README.md +27 -0
  82. data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
  83. data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
  84. data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
  85. data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
  86. data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
  87. data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
  88. data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
  89. data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
  90. data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
  91. data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
  92. data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
  93. data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
  94. data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
  95. data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
  96. data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
  97. data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
  98. data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
  99. data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
  100. data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
  101. data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
  102. data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
  103. data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
  104. data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
  105. data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
  106. data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
  107. data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
  108. data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
  109. data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
  110. data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
  111. data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
  112. data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
  113. data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
  114. data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
  115. data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
  116. data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
  117. data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
  118. data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
  119. data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
  120. data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
  121. data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
  122. data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
  123. data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
  124. data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
  125. data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
  126. data/web/assets/stylesheets/application.css +746 -0
  127. data/web/assets/stylesheets/bootstrap.css +9 -0
  128. data/web/locales/cs.yml +68 -0
  129. data/web/locales/da.yml +68 -0
  130. data/web/locales/de.yml +69 -0
  131. data/web/locales/el.yml +68 -0
  132. data/web/locales/en.yml +77 -0
  133. data/web/locales/es.yml +69 -0
  134. data/web/locales/fr.yml +69 -0
  135. data/web/locales/hi.yml +75 -0
  136. data/web/locales/it.yml +69 -0
  137. data/web/locales/ja.yml +69 -0
  138. data/web/locales/ko.yml +68 -0
  139. data/web/locales/nl.yml +68 -0
  140. data/web/locales/no.yml +69 -0
  141. data/web/locales/pl.yml +59 -0
  142. data/web/locales/pt-br.yml +68 -0
  143. data/web/locales/pt.yml +67 -0
  144. data/web/locales/ru.yml +75 -0
  145. data/web/locales/sv.yml +68 -0
  146. data/web/locales/ta.yml +75 -0
  147. data/web/locales/zh-cn.yml +68 -0
  148. data/web/locales/zh-tw.yml +68 -0
  149. data/web/views/_footer.erb +22 -0
  150. data/web/views/_job_info.erb +84 -0
  151. data/web/views/_nav.erb +66 -0
  152. data/web/views/_paging.erb +23 -0
  153. data/web/views/_poll_js.erb +5 -0
  154. data/web/views/_poll_link.erb +7 -0
  155. data/web/views/_status.erb +4 -0
  156. data/web/views/_summary.erb +40 -0
  157. data/web/views/busy.erb +90 -0
  158. data/web/views/dashboard.erb +75 -0
  159. data/web/views/dead.erb +34 -0
  160. data/web/views/layout.erb +31 -0
  161. data/web/views/morgue.erb +71 -0
  162. data/web/views/queue.erb +45 -0
  163. data/web/views/queues.erb +27 -0
  164. data/web/views/retries.erb +74 -0
  165. data/web/views/retry.erb +34 -0
  166. data/web/views/scheduled.erb +54 -0
  167. data/web/views/scheduled_job_info.erb +8 -0
  168. metadata +404 -0
data/Gemfile ADDED
@@ -0,0 +1,27 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+
4
+ gem 'rails', '~> 4.2'
5
+ gem 'simplecov'
6
+ gem 'minitest'
7
+ gem 'awesome_print', require: 'ap'
8
+
9
+ platforms :rbx do
10
+ gem 'rubysl', '~> 2.0' # if using anything in the ruby standard library
11
+ gem 'psych' # if using yaml
12
+ gem 'minitest' # if using minitest
13
+ gem 'rubinius-developer_tools' # if using any of coverage, debugger, profiler
14
+ end
15
+
16
+ platforms :ruby do
17
+ gem 'sqlite3'
18
+ end
19
+
20
+ platforms :mri do
21
+ gem 'pry-byebug'
22
+ end
23
+
24
+ platforms :jruby do
25
+ gem 'jruby-openssl'
26
+ gem 'activerecord-jdbcsqlite3-adapter'
27
+ end
data/LICENSE ADDED
@@ -0,0 +1,7 @@
1
+ Original Sidekiq License:
2
+
3
+ Copyright (c) Contributed Systems LLC
4
+
5
+ Sidekiq is an Open Source project licensed under the terms of
6
+ the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
7
+ for license text.
data/README.md ADDED
@@ -0,0 +1,52 @@
1
+ Roundhouse
2
+ ==============
3
+
4
+ Roundhouse is based on Sidekiq 3.5.0 (HEAD: f8ee0896076671b73739099e3c1eb7d8814e407d)
5
+
6
+ It uses the basic framework of a multi-threaded workers and managers to solve a different
7
+ kind of problem. This problem resembles that of a load balancing problem: only one worker
8
+ should be working on a work for a given user. This requires:
9
+
10
+ 1. A queue for each user
11
+ 2. A circular buffer to cycle through all user queues
12
+ 3. A way to track how many pieces of jobs to process per user, before moving to the next user's queue.
13
+
14
+ Other things Roundhouse should be able to do:
15
+
16
+ 1. It should be able to run side-by side with Sidekiq client
17
+ 2. Be able to suspend, resume, and defer work
18
+ 3. Metrics to measure lead times and give estimates on when work might be completed
19
+ 4. Coordinate across a number of worker processes, across machines
20
+
21
+ Install
22
+ ----------------
23
+
24
+ ```
25
+ gem 'roundhouse-x', require: 'roundhouse'
26
+ ```
27
+
28
+ Workers
29
+ ----------------
30
+
31
+ ```
32
+ class APIWorker
33
+ include Roundhouse::Worker
34
+
35
+ def perform(id)
36
+ # something
37
+ end
38
+ end
39
+
40
+ APIWorker.perform_async(api_token.id, item.id)
41
+ ```
42
+
43
+ Sidekiq License
44
+ -----------------
45
+
46
+ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
47
+
48
+
49
+ Sidekiq Original Author
50
+ -----------------
51
+
52
+ Mike Perham, [@mperham](https://twitter.com/mperham) / [@sidekiq](https://twitter.com/sidekiq), [http://www.mikeperham.com](http://www.mikeperham.com) / [http://www.contribsys.com](http://www.contribsys.com)
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rake/testtask'
3
+ Rake::TestTask.new(:test) do |test|
4
+ #SO MUCH NOISE
5
+ #test.warning = true
6
+ test.pattern = 'test/**/test_*.rb'
7
+ end
8
+
9
+ task :default => :test
data/bin/roundhouse ADDED
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Quiet some warnings we see when running in warning mode:
4
+ # RUBYOPT=-w bundle exec sidekiq
5
+ $TESTING = false
6
+ $CELLULOID_DEBUG = false
7
+
8
+ require_relative '../lib/roundhouse/cli'
9
+
10
+ begin
11
+ cli = Roundhouse::CLI.instance
12
+ cli.parse
13
+ cli.run
14
+ rescue => e
15
+ raise e if $DEBUG
16
+ STDERR.puts e.message
17
+ STDERR.puts e.backtrace.join("\n")
18
+ exit 1
19
+ end
data/bin/roundhousectl ADDED
@@ -0,0 +1,93 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'fileutils'
4
+
5
+ class Roundhousectl
6
+ DEFAULT_KILL_TIMEOUT = 10
7
+
8
+ attr_reader :stage, :pidfile, :kill_timeout
9
+
10
+ def self.print_usage
11
+ puts "#{File.basename($0)} - stop a Roundhouse process from the command line."
12
+ puts
13
+ puts "Usage: #{File.basename($0)} <command> <pidfile> <kill_timeout>"
14
+ puts " where <command> is either 'quiet' or 'stop'"
15
+ puts " <pidfile> is path to a pidfile"
16
+ puts " <kill_timeout> is number of seconds to wait until Roundhouse exits"
17
+ puts " (default: #{Roundhousectl::DEFAULT_KILL_TIMEOUT}), after which Roundhouse will be KILL'd"
18
+ puts
19
+ puts "Be sure to set the kill_timeout LONGER than Roundhouse's -t timeout. If you want"
20
+ puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
21
+ puts " path_to_pidfile 61`"
22
+ puts
23
+ end
24
+
25
+ def initialize(stage, pidfile, timeout)
26
+ @stage = stage
27
+ @pidfile = pidfile
28
+ @kill_timeout = timeout
29
+
30
+ done('No pidfile given', :error) if !pidfile
31
+ done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
32
+ done('Invalid pidfile content', :error) if pid == 0
33
+
34
+ fetch_process
35
+
36
+ begin
37
+ send(stage)
38
+ rescue NoMethodError
39
+ done "Invalid command: #{stage}", :error
40
+ end
41
+ end
42
+
43
+ def fetch_process
44
+ Process.getpgid(pid)
45
+ rescue Errno::ESRCH
46
+ done "Process doesn't exist", :error
47
+ end
48
+
49
+ def done(msg, error = nil)
50
+ puts msg
51
+ exit(exit_signal(error))
52
+ end
53
+
54
+ def exit_signal(error)
55
+ (error == :error) ? 1 : 0
56
+ end
57
+
58
+ def pid
59
+ @pid ||= File.read(pidfile).to_i
60
+ end
61
+
62
+ def quiet
63
+ Process.kill(:USR1, pid)
64
+ end
65
+
66
+ def stop
67
+ Process.kill(:TERM, pid)
68
+ kill_timeout.times do
69
+ begin
70
+ Process.getpgid(pid)
71
+ rescue Errno::ESRCH
72
+ FileUtils.rm_f pidfile
73
+ done 'Roundhouse shut down gracefully.'
74
+ end
75
+ sleep 1
76
+ end
77
+ Process.kill(:KILL, pid)
78
+ FileUtils.rm_f pidfile
79
+ done 'Roundhouse shut down forcefully.'
80
+ end
81
+ alias_method :shutdown, :stop
82
+ end
83
+
84
+ if ARGV.length < 2
85
+ Roundhousectl.print_usage
86
+ else
87
+ stage = ARGV[0]
88
+ pidfile = ARGV[1]
89
+ timeout = ARGV[2].to_i
90
+ timeout = Roundhousectl::DEFAULT_KILL_TIMEOUT if timeout == 0
91
+
92
+ Roundhousectl.new(stage, pidfile, timeout)
93
+ end
@@ -0,0 +1,9 @@
1
+ <% module_namespacing do -%>
2
+ class <%= class_name %>Worker
3
+ include Roundhouse::Worker
4
+
5
+ def perform(*args)
6
+ # Do something
7
+ end
8
+ end
9
+ <% end -%>
@@ -0,0 +1,6 @@
1
+ require 'rails_helper'
2
+ <% module_namespacing do -%>
3
+ RSpec.describe <%= class_name %>Worker, :type => :worker do
4
+ pending "add some examples to (or delete) #{__FILE__}"
5
+ end
6
+ <% end -%>
@@ -0,0 +1,8 @@
1
+ require_relative 'test_helper'
2
+ <% module_namespacing do -%>
3
+ class <%= class_name %>WorkerTest < MiniTest::Unit::TestCase
4
+ def test_example
5
+ skip "add some examples to (or delete) #{__FILE__}"
6
+ end
7
+ end
8
+ <% end -%>
@@ -0,0 +1,49 @@
1
+ require 'rails/generators/named_base'
2
+
3
+ module Roundhouse
4
+ module Generators # :nodoc:
5
+ class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
+ desc 'This generator creates a Roundhouse Worker in app/workers and a corresponding test'
7
+
8
+ check_class_collision suffix: 'Worker'
9
+
10
+ def self.default_generator_root
11
+ File.dirname(__FILE__)
12
+ end
13
+
14
+ def create_worker_file
15
+ template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
16
+ end
17
+
18
+ def create_test_file
19
+ if defined?(RSpec)
20
+ create_worker_spec
21
+ else
22
+ create_worker_test
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ def create_worker_spec
29
+ template_file = File.join(
30
+ 'spec/workers',
31
+ class_path,
32
+ "#{file_name}_worker_spec.rb"
33
+ )
34
+ template 'worker_spec.rb.erb', template_file
35
+ end
36
+
37
+ def create_worker_test
38
+ template_file = File.join(
39
+ 'test/workers',
40
+ class_path,
41
+ "#{file_name}_worker_test.rb"
42
+ )
43
+ template 'worker_test.rb.erb', template_file
44
+ end
45
+
46
+
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,39 @@
1
+ module Roundhouse
2
+ module Actor
3
+
4
+ module ClassMethods
5
+ def trap_exit(*args)
6
+ end
7
+ def new_link(*args)
8
+ new(*args)
9
+ end
10
+ end
11
+
12
+ module InstanceMethods
13
+ def current_actor
14
+ self
15
+ end
16
+ def after(interval)
17
+ end
18
+ def alive?
19
+ @dead = false unless defined?(@dead)
20
+ !@dead
21
+ end
22
+ def terminate
23
+ @dead = true
24
+ end
25
+ def defer
26
+ yield
27
+ end
28
+ end
29
+
30
+ def self.included(klass)
31
+ if $TESTING
32
+ klass.__send__(:include, InstanceMethods)
33
+ klass.__send__(:extend, ClassMethods)
34
+ else
35
+ klass.__send__(:include, Celluloid)
36
+ end
37
+ end
38
+ end
39
+ end