creeper 1.0.9 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. data/.gitignore +1 -0
  2. data/.rvmrc +48 -0
  3. data/Gemfile +17 -1
  4. data/Guardfile +32 -0
  5. data/Rakefile +9 -1
  6. data/bin/creeper +10 -58
  7. data/bin/creeperctl +74 -0
  8. data/config.ru +18 -0
  9. data/creeper.gemspec +19 -9
  10. data/lib/creeper.rb +108 -413
  11. data/lib/creeper/beanstalk_connection.rb +35 -0
  12. data/lib/creeper/cli.rb +225 -0
  13. data/lib/creeper/client.rb +93 -0
  14. data/lib/creeper/core_ext.rb +54 -0
  15. data/lib/creeper/exception_handler.rb +30 -0
  16. data/lib/creeper/extensions/action_mailer.rb +33 -0
  17. data/lib/creeper/extensions/active_record.rb +30 -0
  18. data/lib/creeper/extensions/generic_proxy.rb +26 -0
  19. data/lib/creeper/fetch.rb +94 -0
  20. data/lib/creeper/legacy.rb +46 -0
  21. data/lib/creeper/logging.rb +46 -0
  22. data/lib/creeper/manager.rb +164 -0
  23. data/lib/creeper/middleware/chain.rb +100 -0
  24. data/lib/creeper/middleware/server/active_record.rb +13 -0
  25. data/lib/creeper/middleware/server/logging.rb +31 -0
  26. data/lib/creeper/middleware/server/retry_jobs.rb +79 -0
  27. data/lib/creeper/middleware/server/timeout.rb +21 -0
  28. data/lib/creeper/paginator.rb +31 -0
  29. data/lib/creeper/processor.rb +116 -0
  30. data/lib/creeper/rails.rb +21 -0
  31. data/lib/creeper/redis_connection.rb +28 -0
  32. data/lib/creeper/testing.rb +44 -0
  33. data/lib/creeper/util.rb +45 -0
  34. data/lib/creeper/version.rb +1 -1
  35. data/lib/creeper/web.rb +248 -0
  36. data/lib/creeper/worker.rb +62 -313
  37. data/spec/dummy/.gitignore +15 -0
  38. data/spec/dummy/Gemfile +51 -0
  39. data/spec/dummy/README.rdoc +261 -0
  40. data/spec/dummy/Rakefile +7 -0
  41. data/spec/dummy/app/assets/images/rails.png +0 -0
  42. data/spec/dummy/app/assets/javascripts/application.js +15 -0
  43. data/spec/dummy/app/assets/stylesheets/application.css +13 -0
  44. data/spec/dummy/app/controllers/application_controller.rb +3 -0
  45. data/spec/dummy/app/controllers/work_controller.rb +71 -0
  46. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  47. data/spec/dummy/app/mailers/.gitkeep +0 -0
  48. data/spec/dummy/app/mailers/user_mailer.rb +9 -0
  49. data/spec/dummy/app/models/.gitkeep +0 -0
  50. data/spec/dummy/app/models/post.rb +8 -0
  51. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  52. data/spec/dummy/app/views/user_mailer/greetings.html.erb +3 -0
  53. data/spec/dummy/app/views/work/index.html.erb +1 -0
  54. data/spec/dummy/app/workers/fast_worker.rb +10 -0
  55. data/spec/dummy/app/workers/hard_worker.rb +11 -0
  56. data/spec/dummy/app/workers/lazy_worker.rb +12 -0
  57. data/spec/dummy/app/workers/suicidal_worker.rb +33 -0
  58. data/spec/dummy/config.ru +4 -0
  59. data/spec/dummy/config/application.rb +68 -0
  60. data/spec/dummy/config/boot.rb +6 -0
  61. data/spec/dummy/config/creeper.yml +9 -0
  62. data/spec/dummy/config/database.yml +25 -0
  63. data/spec/dummy/config/environment.rb +5 -0
  64. data/spec/dummy/config/environments/development.rb +37 -0
  65. data/spec/dummy/config/environments/production.rb +67 -0
  66. data/spec/dummy/config/environments/test.rb +37 -0
  67. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  68. data/spec/dummy/config/initializers/creeper.rb +8 -0
  69. data/spec/dummy/config/initializers/inflections.rb +15 -0
  70. data/spec/dummy/config/initializers/mime_types.rb +5 -0
  71. data/spec/dummy/config/initializers/secret_token.rb +7 -0
  72. data/spec/dummy/config/initializers/session_store.rb +8 -0
  73. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  74. data/spec/dummy/config/locales/en.yml +5 -0
  75. data/spec/dummy/config/routes.rb +13 -0
  76. data/spec/dummy/db/migrate/20120123214055_create_posts.rb +10 -0
  77. data/spec/dummy/db/schema.rb +23 -0
  78. data/spec/dummy/db/seeds.rb +7 -0
  79. data/spec/dummy/lib/assets/.gitkeep +0 -0
  80. data/spec/dummy/lib/tasks/.gitkeep +0 -0
  81. data/spec/dummy/log/.gitkeep +0 -0
  82. data/spec/dummy/public/404.html +26 -0
  83. data/spec/dummy/public/422.html +26 -0
  84. data/spec/dummy/public/500.html +25 -0
  85. data/spec/dummy/public/favicon.ico +0 -0
  86. data/spec/dummy/public/index.html +241 -0
  87. data/spec/dummy/public/robots.txt +5 -0
  88. data/spec/dummy/script/rails +6 -0
  89. data/spec/dummy/vendor/assets/javascripts/.gitkeep +0 -0
  90. data/spec/dummy/vendor/assets/stylesheets/.gitkeep +0 -0
  91. data/spec/dummy/vendor/plugins/.gitkeep +0 -0
  92. data/spec/lib/creeper/cli_spec.rb +208 -0
  93. data/spec/lib/creeper/client_spec.rb +110 -0
  94. data/spec/lib/creeper/exception_handler_spec.rb +110 -0
  95. data/spec/lib/creeper/processor_spec.rb +92 -0
  96. data/spec/lib/creeper/testing_spec.rb +105 -0
  97. data/spec/lib/creeper_spec.rb +54 -120
  98. data/spec/spec_helper.rb +81 -7
  99. data/spec/support/config.yml +9 -0
  100. data/spec/support/fake_env.rb +0 -0
  101. data/spec/support/workers/base_worker.rb +11 -0
  102. data/spec/support/workers/my_worker.rb +4 -0
  103. data/spec/support/workers/queued_worker.rb +5 -0
  104. data/spec/support/workers/real_worker.rb +10 -0
  105. data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
  106. data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
  107. data/web/assets/javascripts/application.js +49 -0
  108. data/web/assets/javascripts/vendor/bootstrap.js +12 -0
  109. data/web/assets/javascripts/vendor/bootstrap/bootstrap-alert.js +91 -0
  110. data/web/assets/javascripts/vendor/bootstrap/bootstrap-button.js +98 -0
  111. data/web/assets/javascripts/vendor/bootstrap/bootstrap-carousel.js +154 -0
  112. data/web/assets/javascripts/vendor/bootstrap/bootstrap-collapse.js +136 -0
  113. data/web/assets/javascripts/vendor/bootstrap/bootstrap-dropdown.js +92 -0
  114. data/web/assets/javascripts/vendor/bootstrap/bootstrap-modal.js +210 -0
  115. data/web/assets/javascripts/vendor/bootstrap/bootstrap-popover.js +95 -0
  116. data/web/assets/javascripts/vendor/bootstrap/bootstrap-scrollspy.js +125 -0
  117. data/web/assets/javascripts/vendor/bootstrap/bootstrap-tab.js +130 -0
  118. data/web/assets/javascripts/vendor/bootstrap/bootstrap-tooltip.js +270 -0
  119. data/web/assets/javascripts/vendor/bootstrap/bootstrap-transition.js +51 -0
  120. data/web/assets/javascripts/vendor/bootstrap/bootstrap-typeahead.js +271 -0
  121. data/web/assets/javascripts/vendor/jquery.js +9266 -0
  122. data/web/assets/javascripts/vendor/jquery.timeago.js +148 -0
  123. data/web/assets/stylesheets/application.css +6 -0
  124. data/web/assets/stylesheets/layout.css +26 -0
  125. data/web/assets/stylesheets/vendor/bootstrap-responsive.css +567 -0
  126. data/web/assets/stylesheets/vendor/bootstrap.css +3365 -0
  127. data/web/views/_paging.slim +15 -0
  128. data/web/views/_summary.slim +9 -0
  129. data/web/views/_workers.slim +14 -0
  130. data/web/views/index.slim +10 -0
  131. data/web/views/layout.slim +37 -0
  132. data/web/views/poll.slim +3 -0
  133. data/web/views/queue.slim +15 -0
  134. data/web/views/queues.slim +19 -0
  135. data/web/views/retries.slim +31 -0
  136. data/web/views/retry.slim +52 -0
  137. data/web/views/scheduled.slim +27 -0
  138. metadata +341 -23
  139. data/lib/creeper/celluloid_ext.rb +0 -42
  140. data/lib/creeper/creep.rb +0 -25
  141. data/lib/creeper/err_logger.rb +0 -37
  142. data/lib/creeper/launcher.rb +0 -44
  143. data/lib/creeper/out_logger.rb +0 -39
  144. data/spec/lib/creeper/session_spec.rb +0 -15
  145. data/spec/lib/creeper/worker_spec.rb +0 -21
@@ -1,42 +0,0 @@
1
- module Celluloid
2
- class Actor
3
-
4
- def handle_crash(exception)
5
- prefix = (@subject.respond_to?(:prefix) rescue nil) ? ("#{@subject.prefix} " rescue nil) : nil
6
- Logger.crash("#{prefix}#{@subject.class} crashed!", exception)
7
- shutdown ExitEvent.new(@proxy, exception)
8
- rescue => ex
9
- Logger.crash("#{@subject.class}: ERROR HANDLER CRASHED!", ex)
10
- end
11
-
12
- end
13
- end
14
-
15
- module Celluloid
16
- class PoolManager
17
-
18
- # ensure that the provisioned worker is alive to prevent PoolManager from dying
19
- def execute(method, *args, &block)
20
- worker = provision_worker
21
-
22
- begin
23
- worker._send_ method, *args, &block
24
- rescue Celluloid::DeadActorError, Celluloid::MailboxError
25
- execute(method, *args, &block)
26
- ensure
27
- @idle << worker if worker && worker.alive?
28
- end
29
- end
30
-
31
- def crash_handler(actor, reason)
32
- return unless reason # don't restart workers that exit cleanly
33
- index = @idle.rindex(actor) # replace the old actor if possible
34
- if index
35
- @idle[index] = @worker_class.new_link(*@args)
36
- else
37
- @idle << @worker_class.new_link(*@args)
38
- end
39
- end
40
-
41
- end
42
- end
@@ -1,25 +0,0 @@
1
- module Creeper
2
- module Creep
3
-
4
- def enqueue(job, data = {}, options = {})
5
- Creeper.enqueue(job, data, options)
6
- end
7
-
8
- def job(name, &block)
9
- Creeper.job(name, &block)
10
- end
11
-
12
- def before(name = nil, &block)
13
- Creeper.before(name, &block)
14
- end
15
-
16
- def after(name = nil, &block)
17
- Creeper.after(name, &block)
18
- end
19
-
20
- def error(name = nil, &block)
21
- Creeper.error(name, &block)
22
- end
23
-
24
- end
25
- end
@@ -1,37 +0,0 @@
1
- module Creeper
2
- module ErrLogger
3
- module_function
4
-
5
- # Send a debug message
6
- def debug(string)
7
- Creeper.err_logger.debug(string) if Creeper.err_logger
8
- end
9
-
10
- # Send a info message
11
- def info(string)
12
- Creeper.err_logger.info(string) if Creeper.err_logger
13
- end
14
-
15
- # Send a warning message
16
- def warn(string)
17
- Creeper.err_logger.warn(string) if Creeper.err_logger
18
- end
19
-
20
- # Send an error message
21
- def error(string)
22
- Creeper.err_logger.error(string) if Creeper.err_logger
23
- end
24
-
25
- # Handle a crash
26
- def crash(string, exception)
27
- string << "\n" << format_exception(exception)
28
- error string
29
- end
30
-
31
- # Format an exception message
32
- def format_exception(exception)
33
- str = "#{exception.class}: #{exception.to_s}\n"
34
- str << exception.backtrace.join("\n")
35
- end
36
- end
37
- end
@@ -1,44 +0,0 @@
1
- $stdout.sync = $stderr.sync = true
2
- $stdin.binmode
3
- $stdout.binmode
4
- $stderr.binmode
5
-
6
- require 'creeper'
7
-
8
- module Creeper
9
- module Launcher
10
-
11
- extend self
12
-
13
- def launch!(options)
14
- $stdin.reopen("/dev/null")
15
-
16
- # grandparent - reads pipe, exits when master is ready
17
- # \_ parent - exits immediately ASAP
18
- # \_ creeper master - writes to pipe when ready
19
-
20
- rd, wr = IO.pipe
21
- grandparent = $$
22
- if fork
23
- wr.close # grandparent does not write
24
- else
25
- rd.close # creeper master does not read
26
- Process.setsid
27
- exit if fork # parent dies now
28
- end
29
-
30
- if grandparent == $$
31
- # this will block until Creeper.join runs (or it dies)
32
- creeper_pid = (rd.readpartial(16) rescue nil).to_i
33
- unless creeper_pid > 1
34
- warn "creeper failed to start, check stderr log for details"
35
- exit!(1)
36
- end
37
- exit 0
38
- else # creeper master process
39
- options[:ready_pipe] = wr
40
- end
41
- end
42
-
43
- end
44
- end
@@ -1,39 +0,0 @@
1
- module Creeper
2
- module OutLogger
3
-
4
- extend self
5
-
6
- # Send a debug message
7
- def debug(string)
8
- Creeper.out_logger.debug(string) if Creeper.out_logger
9
- end
10
-
11
- # Send a info message
12
- def info(string)
13
- Creeper.out_logger.info(string) if Creeper.out_logger
14
- end
15
-
16
- # Send a warning message
17
- def warn(string)
18
- Creeper.out_logger.warn(string) if Creeper.out_logger
19
- end
20
-
21
- # Send an error message
22
- def error(string)
23
- Creeper.out_logger.error(string) if Creeper.out_logger
24
- end
25
-
26
- # Handle a crash
27
- def crash(string, exception)
28
- string << "\n" << format_exception(exception)
29
- error string
30
- end
31
-
32
- # Format an exception message
33
- def format_exception(exception)
34
- str = "#{exception.class}: #{exception.to_s}\n"
35
- str << exception.backtrace.join("\n")
36
- end
37
-
38
- end
39
- end
@@ -1,15 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Creeper::Session do
4
-
5
- it "parses BEANSTALK_URL" do
6
- ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300"
7
- Creeper::Session.new.beanstalk_addresses.should == ["localhost:12300"]
8
- ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300/, beanstalk://localhost:12301/"
9
- Creeper::Session.new.beanstalk_addresses.should == ["localhost:12300","localhost:12301"]
10
- ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300 beanstalk://localhost:12301"
11
- Creeper::Session.new.beanstalk_addresses.should == ["localhost:12300","localhost:12301"]
12
- ENV['BEANSTALK_URL'] = nil
13
- end
14
-
15
- end
@@ -1,21 +0,0 @@
1
- require 'spec_helper'
2
-
3
- describe Creeper::Worker do
4
-
5
- context 'a worker' do
6
-
7
- subject { Creeper::Worker.new }
8
-
9
- after(:each) do
10
- subject.clear!
11
- Creeper.clear!
12
- end
13
-
14
- it 'should have a session different from the Creeper.default_session' do
15
- subject.session.should_not == Creeper.default_session
16
- subject.session.beanstalk.should_not == Creeper.default_session.beanstalk
17
- end
18
-
19
- end
20
-
21
- end