creeper 1.0.9 → 2.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (145) hide show
  1. data/.gitignore +1 -0
  2. data/.rvmrc +48 -0
  3. data/Gemfile +17 -1
  4. data/Guardfile +32 -0
  5. data/Rakefile +9 -1
  6. data/bin/creeper +10 -58
  7. data/bin/creeperctl +74 -0
  8. data/config.ru +18 -0
  9. data/creeper.gemspec +19 -9
  10. data/lib/creeper.rb +108 -413
  11. data/lib/creeper/beanstalk_connection.rb +35 -0
  12. data/lib/creeper/cli.rb +225 -0
  13. data/lib/creeper/client.rb +93 -0
  14. data/lib/creeper/core_ext.rb +54 -0
  15. data/lib/creeper/exception_handler.rb +30 -0
  16. data/lib/creeper/extensions/action_mailer.rb +33 -0
  17. data/lib/creeper/extensions/active_record.rb +30 -0
  18. data/lib/creeper/extensions/generic_proxy.rb +26 -0
  19. data/lib/creeper/fetch.rb +94 -0
  20. data/lib/creeper/legacy.rb +46 -0
  21. data/lib/creeper/logging.rb +46 -0
  22. data/lib/creeper/manager.rb +164 -0
  23. data/lib/creeper/middleware/chain.rb +100 -0
  24. data/lib/creeper/middleware/server/active_record.rb +13 -0
  25. data/lib/creeper/middleware/server/logging.rb +31 -0
  26. data/lib/creeper/middleware/server/retry_jobs.rb +79 -0
  27. data/lib/creeper/middleware/server/timeout.rb +21 -0
  28. data/lib/creeper/paginator.rb +31 -0
  29. data/lib/creeper/processor.rb +116 -0
  30. data/lib/creeper/rails.rb +21 -0
  31. data/lib/creeper/redis_connection.rb +28 -0
  32. data/lib/creeper/testing.rb +44 -0
  33. data/lib/creeper/util.rb +45 -0
  34. data/lib/creeper/version.rb +1 -1
  35. data/lib/creeper/web.rb +248 -0
  36. data/lib/creeper/worker.rb +62 -313
  37. data/spec/dummy/.gitignore +15 -0
  38. data/spec/dummy/Gemfile +51 -0
  39. data/spec/dummy/README.rdoc +261 -0
  40. data/spec/dummy/Rakefile +7 -0
  41. data/spec/dummy/app/assets/images/rails.png +0 -0
  42. data/spec/dummy/app/assets/javascripts/application.js +15 -0
  43. data/spec/dummy/app/assets/stylesheets/application.css +13 -0
  44. data/spec/dummy/app/controllers/application_controller.rb +3 -0
  45. data/spec/dummy/app/controllers/work_controller.rb +71 -0
  46. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  47. data/spec/dummy/app/mailers/.gitkeep +0 -0
  48. data/spec/dummy/app/mailers/user_mailer.rb +9 -0
  49. data/spec/dummy/app/models/.gitkeep +0 -0
  50. data/spec/dummy/app/models/post.rb +8 -0
  51. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  52. data/spec/dummy/app/views/user_mailer/greetings.html.erb +3 -0
  53. data/spec/dummy/app/views/work/index.html.erb +1 -0
  54. data/spec/dummy/app/workers/fast_worker.rb +10 -0
  55. data/spec/dummy/app/workers/hard_worker.rb +11 -0
  56. data/spec/dummy/app/workers/lazy_worker.rb +12 -0
  57. data/spec/dummy/app/workers/suicidal_worker.rb +33 -0
  58. data/spec/dummy/config.ru +4 -0
  59. data/spec/dummy/config/application.rb +68 -0
  60. data/spec/dummy/config/boot.rb +6 -0
  61. data/spec/dummy/config/creeper.yml +9 -0
  62. data/spec/dummy/config/database.yml +25 -0
  63. data/spec/dummy/config/environment.rb +5 -0
  64. data/spec/dummy/config/environments/development.rb +37 -0
  65. data/spec/dummy/config/environments/production.rb +67 -0
  66. data/spec/dummy/config/environments/test.rb +37 -0
  67. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  68. data/spec/dummy/config/initializers/creeper.rb +8 -0
  69. data/spec/dummy/config/initializers/inflections.rb +15 -0
  70. data/spec/dummy/config/initializers/mime_types.rb +5 -0
  71. data/spec/dummy/config/initializers/secret_token.rb +7 -0
  72. data/spec/dummy/config/initializers/session_store.rb +8 -0
  73. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  74. data/spec/dummy/config/locales/en.yml +5 -0
  75. data/spec/dummy/config/routes.rb +13 -0
  76. data/spec/dummy/db/migrate/20120123214055_create_posts.rb +10 -0
  77. data/spec/dummy/db/schema.rb +23 -0
  78. data/spec/dummy/db/seeds.rb +7 -0
  79. data/spec/dummy/lib/assets/.gitkeep +0 -0
  80. data/spec/dummy/lib/tasks/.gitkeep +0 -0
  81. data/spec/dummy/log/.gitkeep +0 -0
  82. data/spec/dummy/public/404.html +26 -0
  83. data/spec/dummy/public/422.html +26 -0
  84. data/spec/dummy/public/500.html +25 -0
  85. data/spec/dummy/public/favicon.ico +0 -0
  86. data/spec/dummy/public/index.html +241 -0
  87. data/spec/dummy/public/robots.txt +5 -0
  88. data/spec/dummy/script/rails +6 -0
  89. data/spec/dummy/vendor/assets/javascripts/.gitkeep +0 -0
  90. data/spec/dummy/vendor/assets/stylesheets/.gitkeep +0 -0
  91. data/spec/dummy/vendor/plugins/.gitkeep +0 -0
  92. data/spec/lib/creeper/cli_spec.rb +208 -0
  93. data/spec/lib/creeper/client_spec.rb +110 -0
  94. data/spec/lib/creeper/exception_handler_spec.rb +110 -0
  95. data/spec/lib/creeper/processor_spec.rb +92 -0
  96. data/spec/lib/creeper/testing_spec.rb +105 -0
  97. data/spec/lib/creeper_spec.rb +54 -120
  98. data/spec/spec_helper.rb +81 -7
  99. data/spec/support/config.yml +9 -0
  100. data/spec/support/fake_env.rb +0 -0
  101. data/spec/support/workers/base_worker.rb +11 -0
  102. data/spec/support/workers/my_worker.rb +4 -0
  103. data/spec/support/workers/queued_worker.rb +5 -0
  104. data/spec/support/workers/real_worker.rb +10 -0
  105. data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
  106. data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
  107. data/web/assets/javascripts/application.js +49 -0
  108. data/web/assets/javascripts/vendor/bootstrap.js +12 -0
  109. data/web/assets/javascripts/vendor/bootstrap/bootstrap-alert.js +91 -0
  110. data/web/assets/javascripts/vendor/bootstrap/bootstrap-button.js +98 -0
  111. data/web/assets/javascripts/vendor/bootstrap/bootstrap-carousel.js +154 -0
  112. data/web/assets/javascripts/vendor/bootstrap/bootstrap-collapse.js +136 -0
  113. data/web/assets/javascripts/vendor/bootstrap/bootstrap-dropdown.js +92 -0
  114. data/web/assets/javascripts/vendor/bootstrap/bootstrap-modal.js +210 -0
  115. data/web/assets/javascripts/vendor/bootstrap/bootstrap-popover.js +95 -0
  116. data/web/assets/javascripts/vendor/bootstrap/bootstrap-scrollspy.js +125 -0
  117. data/web/assets/javascripts/vendor/bootstrap/bootstrap-tab.js +130 -0
  118. data/web/assets/javascripts/vendor/bootstrap/bootstrap-tooltip.js +270 -0
  119. data/web/assets/javascripts/vendor/bootstrap/bootstrap-transition.js +51 -0
  120. data/web/assets/javascripts/vendor/bootstrap/bootstrap-typeahead.js +271 -0
  121. data/web/assets/javascripts/vendor/jquery.js +9266 -0
  122. data/web/assets/javascripts/vendor/jquery.timeago.js +148 -0
  123. data/web/assets/stylesheets/application.css +6 -0
  124. data/web/assets/stylesheets/layout.css +26 -0
  125. data/web/assets/stylesheets/vendor/bootstrap-responsive.css +567 -0
  126. data/web/assets/stylesheets/vendor/bootstrap.css +3365 -0
  127. data/web/views/_paging.slim +15 -0
  128. data/web/views/_summary.slim +9 -0
  129. data/web/views/_workers.slim +14 -0
  130. data/web/views/index.slim +10 -0
  131. data/web/views/layout.slim +37 -0
  132. data/web/views/poll.slim +3 -0
  133. data/web/views/queue.slim +15 -0
  134. data/web/views/queues.slim +19 -0
  135. data/web/views/retries.slim +31 -0
  136. data/web/views/retry.slim +52 -0
  137. data/web/views/scheduled.slim +27 -0
  138. metadata +341 -23
  139. data/lib/creeper/celluloid_ext.rb +0 -42
  140. data/lib/creeper/creep.rb +0 -25
  141. data/lib/creeper/err_logger.rb +0 -37
  142. data/lib/creeper/launcher.rb +0 -44
  143. data/lib/creeper/out_logger.rb +0 -39
  144. data/spec/lib/creeper/session_spec.rb +0 -15
  145. data/spec/lib/creeper/worker_spec.rb +0 -21
@@ -1,42 +0,0 @@
1
- module Celluloid
2
- class Actor
3
-
4
- def handle_crash(exception)
5
- prefix = (@subject.respond_to?(:prefix) rescue nil) ? ("#{@subject.prefix} " rescue nil) : nil
6
- Logger.crash("#{prefix}#{@subject.class} crashed!", exception)
7
- shutdown ExitEvent.new(@proxy, exception)
8
- rescue => ex
9
- Logger.crash("#{@subject.class}: ERROR HANDLER CRASHED!", ex)
10
- end
11
-
12
- end
13
- end
14
-
15
- module Celluloid
16
- class PoolManager
17
-
18
- # ensure that the provisioned worker is alive to prevent PoolManager from dying
19
- def execute(method, *args, &block)
20
- worker = provision_worker
21
-
22
- begin
23
- worker._send_ method, *args, &block
24
- rescue Celluloid::DeadActorError, Celluloid::MailboxError
25
- execute(method, *args, &block)
26
- ensure
27
- @idle << worker if worker && worker.alive?
28
- end
29
- end
30
-
31
- def crash_handler(actor, reason)
32
- return unless reason # don't restart workers that exit cleanly
33
- index = @idle.rindex(actor) # replace the old actor if possible
34
- if index
35
- @idle[index] = @worker_class.new_link(*@args)
36
- else
37
- @idle << @worker_class.new_link(*@args)
38
- end
39
- end
40
-
41
- end
42
- end
@@ -1,25 +0,0 @@
1
- module Creeper
2
- module Creep
3
-
4
- def enqueue(job, data = {}, options = {})
5
- Creeper.enqueue(job, data, options)
6
- end
7
-
8
- def job(name, &block)
9
- Creeper.job(name, &block)
10
- end
11
-
12
- def before(name = nil, &block)
13
- Creeper.before(name, &block)
14
- end
15
-
16
- def after(name = nil, &block)
17
- Creeper.after(name, &block)
18
- end
19
-
20
- def error(name = nil, &block)
21
- Creeper.error(name, &block)
22
- end
23
-
24
- end
25
- end
@@ -1,37 +0,0 @@
1
- module Creeper
2
- module ErrLogger
3
- module_function
4
-
5
- # Send a debug message
6
- def debug(string)
7
- Creeper.err_logger.debug(string) if Creeper.err_logger
8
- end
9
-
10
- # Send a info message
11
- def info(string)
12
- Creeper.err_logger.info(string) if Creeper.err_logger
13
- end
14
-
15
- # Send a warning message
16
- def warn(string)
17
- Creeper.err_logger.warn(string) if Creeper.err_logger
18
- end
19
-
20
- # Send an error message
21
- def error(string)
22
- Creeper.err_logger.error(string) if Creeper.err_logger
23
- end
24
-
25
- # Handle a crash
26
- def crash(string, exception)
27
- string << "\n" << format_exception(exception)
28
- error string
29
- end
30
-
31
- # Format an exception message
32
- def format_exception(exception)
33
- str = "#{exception.class}: #{exception.to_s}\n"
34
- str << exception.backtrace.join("\n")
35
- end
36
- end
37
- end
@@ -1,44 +0,0 @@
1
- $stdout.sync = $stderr.sync = true
2
- $stdin.binmode
3
- $stdout.binmode
4
- $stderr.binmode
5
-
6
- require 'creeper'
7
-
8
- module Creeper
9
- module Launcher
10
-
11
- extend self
12
-
13
- def launch!(options)
14
- $stdin.reopen("/dev/null")
15
-
16
- # grandparent - reads pipe, exits when master is ready
17
- # \_ parent - exits immediately ASAP
18
- # \_ creeper master - writes to pipe when ready
19
-
20
- rd, wr = IO.pipe
21
- grandparent = $$
22
- if fork
23
- wr.close # grandparent does not write
24
- else
25
- rd.close # creeper master does not read
26
- Process.setsid
27
- exit if fork # parent dies now
28
- end
29
-
30
- if grandparent == $$
31
- # this will block until Creeper.join runs (or it dies)
32
- creeper_pid = (rd.readpartial(16) rescue nil).to_i
33
- unless creeper_pid > 1
34
- warn "creeper failed to start, check stderr log for details"
35
- exit!(1)
36
- end
37
- exit 0
38
- else # creeper master process
39
- options[:ready_pipe] = wr
40
- end
41
- end
42
-
43
- end
44
- end
@@ -1,39 +0,0 @@
1
- module Creeper
2
- module OutLogger
3
-
4
- extend self
5
-
6
- # Send a debug message
7
- def debug(string)
8
- Creeper.out_logger.debug(string) if Creeper.out_logger
9
- end
10
-
11
- # Send a info message
12
- def info(string)
13
- Creeper.out_logger.info(string) if Creeper.out_logger
14
- end
15
-
16
- # Send a warning message
17
- def warn(string)
18
- Creeper.out_logger.warn(string) if Creeper.out_logger
19
- end
20
-
21
- # Send an error message
22
- def error(string)
23
- Creeper.out_logger.error(string) if Creeper.out_logger
24
- end
25
-
26
- # Handle a crash
27
- def crash(string, exception)
28
- string << "\n" << format_exception(exception)
29
- error string
30
- end
31
-
32
- # Format an exception message
33
- def format_exception(exception)
34
- str = "#{exception.class}: #{exception.to_s}\n"
35
- str << exception.backtrace.join("\n")
36
- end
37
-
38
- end
39
- end
@@ -1,15 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Creeper::Session do
4
-
5
- it "parses BEANSTALK_URL" do
6
- ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300"
7
- Creeper::Session.new.beanstalk_addresses.should == ["localhost:12300"]
8
- ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300/, beanstalk://localhost:12301/"
9
- Creeper::Session.new.beanstalk_addresses.should == ["localhost:12300","localhost:12301"]
10
- ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300 beanstalk://localhost:12301"
11
- Creeper::Session.new.beanstalk_addresses.should == ["localhost:12300","localhost:12301"]
12
- ENV['BEANSTALK_URL'] = nil
13
- end
14
-
15
- end
@@ -1,21 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Creeper::Worker do
4
-
5
- context 'a worker' do
6
-
7
- subject { Creeper::Worker.new }
8
-
9
- after(:each) do
10
- subject.clear!
11
- Creeper.clear!
12
- end
13
-
14
- it 'should have a session different from the Creeper.default_session' do
15
- subject.session.should_not == Creeper.default_session
16
- subject.session.beanstalk.should_not == Creeper.default_session.beanstalk
17
- end
18
-
19
- end
20
-
21
- end