roundhouse-x 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/.travis.yml +16 -0
  4. data/3.0-Upgrade.md +70 -0
  5. data/Changes.md +1127 -0
  6. data/Gemfile +27 -0
  7. data/LICENSE +7 -0
  8. data/README.md +52 -0
  9. data/Rakefile +9 -0
  10. data/bin/roundhouse +19 -0
  11. data/bin/roundhousectl +93 -0
  12. data/lib/generators/roundhouse/templates/worker.rb.erb +9 -0
  13. data/lib/generators/roundhouse/templates/worker_spec.rb.erb +6 -0
  14. data/lib/generators/roundhouse/templates/worker_test.rb.erb +8 -0
  15. data/lib/generators/roundhouse/worker_generator.rb +49 -0
  16. data/lib/roundhouse/actor.rb +39 -0
  17. data/lib/roundhouse/api.rb +859 -0
  18. data/lib/roundhouse/cli.rb +396 -0
  19. data/lib/roundhouse/client.rb +210 -0
  20. data/lib/roundhouse/core_ext.rb +105 -0
  21. data/lib/roundhouse/exception_handler.rb +30 -0
  22. data/lib/roundhouse/fetch.rb +154 -0
  23. data/lib/roundhouse/launcher.rb +98 -0
  24. data/lib/roundhouse/logging.rb +104 -0
  25. data/lib/roundhouse/manager.rb +236 -0
  26. data/lib/roundhouse/middleware/chain.rb +149 -0
  27. data/lib/roundhouse/middleware/i18n.rb +41 -0
  28. data/lib/roundhouse/middleware/server/active_record.rb +13 -0
  29. data/lib/roundhouse/middleware/server/logging.rb +40 -0
  30. data/lib/roundhouse/middleware/server/retry_jobs.rb +206 -0
  31. data/lib/roundhouse/monitor.rb +124 -0
  32. data/lib/roundhouse/paginator.rb +42 -0
  33. data/lib/roundhouse/processor.rb +159 -0
  34. data/lib/roundhouse/rails.rb +24 -0
  35. data/lib/roundhouse/redis_connection.rb +77 -0
  36. data/lib/roundhouse/scheduled.rb +115 -0
  37. data/lib/roundhouse/testing/inline.rb +28 -0
  38. data/lib/roundhouse/testing.rb +193 -0
  39. data/lib/roundhouse/util.rb +68 -0
  40. data/lib/roundhouse/version.rb +3 -0
  41. data/lib/roundhouse/web.rb +264 -0
  42. data/lib/roundhouse/web_helpers.rb +249 -0
  43. data/lib/roundhouse/worker.rb +90 -0
  44. data/lib/roundhouse.rb +177 -0
  45. data/roundhouse.gemspec +27 -0
  46. data/test/config.yml +9 -0
  47. data/test/env_based_config.yml +11 -0
  48. data/test/fake_env.rb +0 -0
  49. data/test/fixtures/en.yml +2 -0
  50. data/test/helper.rb +49 -0
  51. data/test/test_api.rb +521 -0
  52. data/test/test_cli.rb +389 -0
  53. data/test/test_client.rb +294 -0
  54. data/test/test_exception_handler.rb +55 -0
  55. data/test/test_fetch.rb +206 -0
  56. data/test/test_logging.rb +34 -0
  57. data/test/test_manager.rb +169 -0
  58. data/test/test_middleware.rb +160 -0
  59. data/test/test_monitor.rb +258 -0
  60. data/test/test_processor.rb +176 -0
  61. data/test/test_rails.rb +23 -0
  62. data/test/test_redis_connection.rb +127 -0
  63. data/test/test_retry.rb +390 -0
  64. data/test/test_roundhouse.rb +87 -0
  65. data/test/test_scheduled.rb +120 -0
  66. data/test/test_scheduling.rb +75 -0
  67. data/test/test_testing.rb +78 -0
  68. data/test/test_testing_fake.rb +240 -0
  69. data/test/test_testing_inline.rb +65 -0
  70. data/test/test_util.rb +18 -0
  71. data/test/test_web.rb +605 -0
  72. data/test/test_web_helpers.rb +52 -0
  73. data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
  74. data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
  75. data/web/assets/images/logo.png +0 -0
  76. data/web/assets/images/status/active.png +0 -0
  77. data/web/assets/images/status/idle.png +0 -0
  78. data/web/assets/images/status-sd8051fd480.png +0 -0
  79. data/web/assets/javascripts/application.js +83 -0
  80. data/web/assets/javascripts/dashboard.js +300 -0
  81. data/web/assets/javascripts/locales/README.md +27 -0
  82. data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
  83. data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
  84. data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
  85. data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
  86. data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
  87. data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
  88. data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
  89. data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
  90. data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
  91. data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
  92. data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
  93. data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
  94. data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
  95. data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
  96. data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
  97. data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
  98. data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
  99. data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
  100. data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
  101. data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
  102. data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
  103. data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
  104. data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
  105. data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
  106. data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
  107. data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
  108. data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
  109. data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
  110. data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
  111. data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
  112. data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
  113. data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
  114. data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
  115. data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
  116. data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
  117. data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
  118. data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
  119. data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
  120. data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
  121. data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
  122. data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
  123. data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
  124. data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
  125. data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
  126. data/web/assets/stylesheets/application.css +746 -0
  127. data/web/assets/stylesheets/bootstrap.css +9 -0
  128. data/web/locales/cs.yml +68 -0
  129. data/web/locales/da.yml +68 -0
  130. data/web/locales/de.yml +69 -0
  131. data/web/locales/el.yml +68 -0
  132. data/web/locales/en.yml +77 -0
  133. data/web/locales/es.yml +69 -0
  134. data/web/locales/fr.yml +69 -0
  135. data/web/locales/hi.yml +75 -0
  136. data/web/locales/it.yml +69 -0
  137. data/web/locales/ja.yml +69 -0
  138. data/web/locales/ko.yml +68 -0
  139. data/web/locales/nl.yml +68 -0
  140. data/web/locales/no.yml +69 -0
  141. data/web/locales/pl.yml +59 -0
  142. data/web/locales/pt-br.yml +68 -0
  143. data/web/locales/pt.yml +67 -0
  144. data/web/locales/ru.yml +75 -0
  145. data/web/locales/sv.yml +68 -0
  146. data/web/locales/ta.yml +75 -0
  147. data/web/locales/zh-cn.yml +68 -0
  148. data/web/locales/zh-tw.yml +68 -0
  149. data/web/views/_footer.erb +22 -0
  150. data/web/views/_job_info.erb +84 -0
  151. data/web/views/_nav.erb +66 -0
  152. data/web/views/_paging.erb +23 -0
  153. data/web/views/_poll_js.erb +5 -0
  154. data/web/views/_poll_link.erb +7 -0
  155. data/web/views/_status.erb +4 -0
  156. data/web/views/_summary.erb +40 -0
  157. data/web/views/busy.erb +90 -0
  158. data/web/views/dashboard.erb +75 -0
  159. data/web/views/dead.erb +34 -0
  160. data/web/views/layout.erb +31 -0
  161. data/web/views/morgue.erb +71 -0
  162. data/web/views/queue.erb +45 -0
  163. data/web/views/queues.erb +27 -0
  164. data/web/views/retries.erb +74 -0
  165. data/web/views/retry.erb +34 -0
  166. data/web/views/scheduled.erb +54 -0
  167. data/web/views/scheduled_job_info.erb +8 -0
  168. metadata +404 -0
data/Gemfile ADDED
@@ -0,0 +1,27 @@
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+
4
+ gem 'rails', '~> 4.2'
5
+ gem 'simplecov'
6
+ gem 'minitest'
7
+ gem 'awesome_print', require: 'ap'
8
+
9
+ platforms :rbx do
10
+ gem 'rubysl', '~> 2.0' # if using anything in the ruby standard library
11
+ gem 'psych' # if using yaml
12
+ gem 'minitest' # if using minitest
13
+ gem 'rubinius-developer_tools' # if using any of coverage, debugger, profiler
14
+ end
15
+
16
+ platforms :ruby do
17
+ gem 'sqlite3'
18
+ end
19
+
20
+ platforms :mri do
21
+ gem 'pry-byebug'
22
+ end
23
+
24
+ platforms :jruby do
25
+ gem 'jruby-openssl'
26
+ gem 'activerecord-jdbcsqlite3-adapter'
27
+ end
data/LICENSE ADDED
@@ -0,0 +1,7 @@
1
+ Original Sidekiq License:
2
+
3
+ Copyright (c) Contributed Systems LLC
4
+
5
+ Sidekiq is an Open Source project licensed under the terms of
6
+ the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
7
+ for license text.
data/README.md ADDED
@@ -0,0 +1,52 @@
1
+ Roundhouse
2
+ ==============
3
+
4
+ Roundhouse is based on Sidekiq 3.5.0 (HEAD: f8ee0896076671b73739099e3c1eb7d8814e407d)
5
+
6
+ It uses the basic framework of a multi-threaded workers and managers to solve a different
7
+ kind of problem. This problem resembles that of a load balancing problem: only one worker
8
+ should be working on a work for a given user. This requires:
9
+
10
+ 1. A queue for each user
11
+ 2. A circular buffer to cycle through all user queues
12
+ 3. A way to track how many pieces of jobs to process per user, before moving to the next user's queue.
13
+
14
+ Other things Roundhouse should be able to do:
15
+
16
+ 1. It should be able to run side-by side with Sidekiq client
17
+ 2. Be able to suspend, resume, and defer work
18
+ 3. Metrics to measure lead times and give estimates on when work might be completed
19
+ 4. Coordinate across a number of worker processes, across machines
20
+
21
+ Install
22
+ ----------------
23
+
24
+ ```
25
+ gem 'roundhouse-x', require: 'roundhouse'
26
+ ```
27
+
28
+ Workers
29
+ ----------------
30
+
31
+ ```
32
+ class APIWorker
33
+ include Roundhouse::Worker
34
+
35
+ def perform(id)
36
+ # something
37
+ end
38
+ end
39
+
40
+ APIWorker.perform_async(api_token.id, item.id)
41
+ ```
42
+
43
+ Sidekiq License
44
+ -----------------
45
+
46
+ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
47
+
48
+
49
+ Sidekiq Original Author
50
+ -----------------
51
+
52
+ Mike Perham, [@mperham](https://twitter.com/mperham) / [@sidekiq](https://twitter.com/sidekiq), [http://www.mikeperham.com](http://www.mikeperham.com) / [http://www.contribsys.com](http://www.contribsys.com)
data/Rakefile ADDED
@@ -0,0 +1,9 @@
1
+ require 'bundler/gem_tasks'
2
+ require 'rake/testtask'
3
+ Rake::TestTask.new(:test) do |test|
4
+ #SO MUCH NOISE
5
+ #test.warning = true
6
+ test.pattern = 'test/**/test_*.rb'
7
+ end
8
+
9
+ task :default => :test
data/bin/roundhouse ADDED
@@ -0,0 +1,19 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Quiet some warnings we see when running in warning mode:
4
+ # RUBYOPT=-w bundle exec sidekiq
5
+ $TESTING = false
6
+ $CELLULOID_DEBUG = false
7
+
8
+ require_relative '../lib/roundhouse/cli'
9
+
10
+ begin
11
+ cli = Roundhouse::CLI.instance
12
+ cli.parse
13
+ cli.run
14
+ rescue => e
15
+ raise e if $DEBUG
16
+ STDERR.puts e.message
17
+ STDERR.puts e.backtrace.join("\n")
18
+ exit 1
19
+ end
data/bin/roundhousectl ADDED
@@ -0,0 +1,93 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'fileutils'
4
+
5
+ class Roundhousectl
6
+ DEFAULT_KILL_TIMEOUT = 10
7
+
8
+ attr_reader :stage, :pidfile, :kill_timeout
9
+
10
+ def self.print_usage
11
+ puts "#{File.basename($0)} - stop a Roundhouse process from the command line."
12
+ puts
13
+ puts "Usage: #{File.basename($0)} <command> <pidfile> <kill_timeout>"
14
+ puts " where <command> is either 'quiet' or 'stop'"
15
+ puts " <pidfile> is path to a pidfile"
16
+ puts " <kill_timeout> is number of seconds to wait until Roundhouse exits"
17
+ puts " (default: #{Roundhousectl::DEFAULT_KILL_TIMEOUT}), after which Roundhouse will be KILL'd"
18
+ puts
19
+ puts "Be sure to set the kill_timeout LONGER than Roundhouse's -t timeout. If you want"
20
+ puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
21
+ puts " path_to_pidfile 61`"
22
+ puts
23
+ end
24
+
25
+ def initialize(stage, pidfile, timeout)
26
+ @stage = stage
27
+ @pidfile = pidfile
28
+ @kill_timeout = timeout
29
+
30
+ done('No pidfile given', :error) if !pidfile
31
+ done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
32
+ done('Invalid pidfile content', :error) if pid == 0
33
+
34
+ fetch_process
35
+
36
+ begin
37
+ send(stage)
38
+ rescue NoMethodError
39
+ done "Invalid command: #{stage}", :error
40
+ end
41
+ end
42
+
43
+ def fetch_process
44
+ Process.getpgid(pid)
45
+ rescue Errno::ESRCH
46
+ done "Process doesn't exist", :error
47
+ end
48
+
49
+ def done(msg, error = nil)
50
+ puts msg
51
+ exit(exit_signal(error))
52
+ end
53
+
54
+ def exit_signal(error)
55
+ (error == :error) ? 1 : 0
56
+ end
57
+
58
+ def pid
59
+ @pid ||= File.read(pidfile).to_i
60
+ end
61
+
62
+ def quiet
63
+ Process.kill(:USR1, pid)
64
+ end
65
+
66
+ def stop
67
+ Process.kill(:TERM, pid)
68
+ kill_timeout.times do
69
+ begin
70
+ Process.getpgid(pid)
71
+ rescue Errno::ESRCH
72
+ FileUtils.rm_f pidfile
73
+ done 'Roundhouse shut down gracefully.'
74
+ end
75
+ sleep 1
76
+ end
77
+ Process.kill(:KILL, pid)
78
+ FileUtils.rm_f pidfile
79
+ done 'Roundhouse shut down forcefully.'
80
+ end
81
+ alias_method :shutdown, :stop
82
+ end
83
+
84
+ if ARGV.length < 2
85
+ Roundhousectl.print_usage
86
+ else
87
+ stage = ARGV[0]
88
+ pidfile = ARGV[1]
89
+ timeout = ARGV[2].to_i
90
+ timeout = Roundhousectl::DEFAULT_KILL_TIMEOUT if timeout == 0
91
+
92
+ Roundhousectl.new(stage, pidfile, timeout)
93
+ end
@@ -0,0 +1,9 @@
1
+ <% module_namespacing do -%>
2
+ class <%= class_name %>Worker
3
+ include Roundhouse::Worker
4
+
5
+ def perform(*args)
6
+ # Do something
7
+ end
8
+ end
9
+ <% end -%>
@@ -0,0 +1,6 @@
1
+ require 'rails_helper'
2
+ <% module_namespacing do -%>
3
+ RSpec.describe <%= class_name %>Worker, :type => :worker do
4
+ pending "add some examples to (or delete) #{__FILE__}"
5
+ end
6
+ <% end -%>
@@ -0,0 +1,8 @@
1
+ require_relative 'test_helper'
2
+ <% module_namespacing do -%>
3
+ class <%= class_name %>WorkerTest < MiniTest::Unit::TestCase
4
+ def test_example
5
+ skip "add some examples to (or delete) #{__FILE__}"
6
+ end
7
+ end
8
+ <% end -%>
@@ -0,0 +1,49 @@
1
+ require 'rails/generators/named_base'
2
+
3
+ module Roundhouse
4
+ module Generators # :nodoc:
5
+ class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
+ desc 'This generator creates a Roundhouse Worker in app/workers and a corresponding test'
7
+
8
+ check_class_collision suffix: 'Worker'
9
+
10
+ def self.default_generator_root
11
+ File.dirname(__FILE__)
12
+ end
13
+
14
+ def create_worker_file
15
+ template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
16
+ end
17
+
18
+ def create_test_file
19
+ if defined?(RSpec)
20
+ create_worker_spec
21
+ else
22
+ create_worker_test
23
+ end
24
+ end
25
+
26
+ private
27
+
28
+ def create_worker_spec
29
+ template_file = File.join(
30
+ 'spec/workers',
31
+ class_path,
32
+ "#{file_name}_worker_spec.rb"
33
+ )
34
+ template 'worker_spec.rb.erb', template_file
35
+ end
36
+
37
+ def create_worker_test
38
+ template_file = File.join(
39
+ 'test/workers',
40
+ class_path,
41
+ "#{file_name}_worker_test.rb"
42
+ )
43
+ template 'worker_test.rb.erb', template_file
44
+ end
45
+
46
+
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,39 @@
1
+ module Roundhouse
2
+ module Actor
3
+
4
+ module ClassMethods
5
+ def trap_exit(*args)
6
+ end
7
+ def new_link(*args)
8
+ new(*args)
9
+ end
10
+ end
11
+
12
+ module InstanceMethods
13
+ def current_actor
14
+ self
15
+ end
16
+ def after(interval)
17
+ end
18
+ def alive?
19
+ @dead = false unless defined?(@dead)
20
+ !@dead
21
+ end
22
+ def terminate
23
+ @dead = true
24
+ end
25
+ def defer
26
+ yield
27
+ end
28
+ end
29
+
30
+ def self.included(klass)
31
+ if $TESTING
32
+ klass.__send__(:include, InstanceMethods)
33
+ klass.__send__(:extend, ClassMethods)
34
+ else
35
+ klass.__send__(:include, Celluloid)
36
+ end
37
+ end
38
+ end
39
+ end