resque-kalashnikov 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +18 -0
- data/.rspec +2 -0
- data/.travis.yml +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +99 -0
- data/Rakefile +7 -0
- data/demo/.gitignore +15 -0
- data/demo/Gemfile +31 -0
- data/demo/README.rdoc +9 -0
- data/demo/Rakefile +7 -0
- data/demo/app/assets/images/rails.png +0 -0
- data/demo/app/assets/javascripts/application.js +13 -0
- data/demo/app/assets/stylesheets/application.css +13 -0
- data/demo/app/controllers/application_controller.rb +3 -0
- data/demo/app/controllers/test_controller.rb +15 -0
- data/demo/app/helpers/application_helper.rb +2 -0
- data/demo/app/mailers/.gitkeep +0 -0
- data/demo/app/models/.gitkeep +0 -0
- data/demo/app/workers/slow_http_request.rb +3 -0
- data/demo/config.ru +4 -0
- data/demo/config/application.rb +65 -0
- data/demo/config/boot.rb +6 -0
- data/demo/config/environment.rb +5 -0
- data/demo/config/environments/development.rb +26 -0
- data/demo/config/environments/production.rb +51 -0
- data/demo/config/environments/test.rb +35 -0
- data/demo/config/initializers/backtrace_silencers.rb +7 -0
- data/demo/config/initializers/inflections.rb +15 -0
- data/demo/config/initializers/mime_types.rb +5 -0
- data/demo/config/initializers/secret_token.rb +7 -0
- data/demo/config/initializers/session_store.rb +8 -0
- data/demo/config/initializers/wrap_parameters.rb +10 -0
- data/demo/config/locales/en.yml +5 -0
- data/demo/config/routes.rb +11 -0
- data/demo/db/seeds.rb +7 -0
- data/demo/lib/assets/.gitkeep +0 -0
- data/demo/lib/tasks/.gitkeep +0 -0
- data/demo/log/.gitkeep +0 -0
- data/demo/public/404.html +26 -0
- data/demo/public/422.html +26 -0
- data/demo/public/500.html +25 -0
- data/demo/public/favicon.ico +0 -0
- data/demo/public/index.html +12 -0
- data/demo/public/robots.txt +5 -0
- data/demo/script/rails +6 -0
- data/demo/script/resque_async.rb +29 -0
- data/demo/vendor/assets/javascripts/.gitkeep +0 -0
- data/demo/vendor/assets/stylesheets/.gitkeep +0 -0
- data/demo/vendor/plugins/.gitkeep +0 -0
- data/demo/zeus.json +22 -0
- data/lib/event_machine/forced_stop.rb +3 -0
- data/lib/resque/catridge.rb +87 -0
- data/lib/resque/plugins/resque_kalashnikov/resque_kalashnikov.rb +130 -0
- data/lib/resque_kalashnikov.rb +17 -0
- data/lib/resque_kalashnikov/delegation.rb +153 -0
- data/lib/resque_kalashnikov/http_request.rb +64 -0
- data/lib/resque_kalashnikov/railtie.rb +10 -0
- data/lib/resque_kalashnikov/server.rb +50 -0
- data/lib/resque_kalashnikov/server/views/catridges.erb +57 -0
- data/lib/resque_kalashnikov/version.rb +3 -0
- data/lib/tasks.rb +34 -0
- data/resque-kalashnikov.gemspec +30 -0
- data/screenshot.png +0 -0
- data/spec/catridge_spec.rb +42 -0
- data/spec/http_request_spec.rb +84 -0
- data/spec/server_spec.rb +46 -0
- data/spec/spec_helper.rb +56 -0
- data/spec/support/stub_server.rb +46 -0
- data/spec/worker_spec.rb +113 -0
- data/tasks/resque_kalashnikov.rake +2 -0
- metadata +267 -0
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Vlad Bokov
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
# Resque::Kalashnikov - fast, fiber-based URL hitter for Resque
|
2
|
+
|
3
|
+
![screenshot](https://raw.github.com/razum2um/resque-kalashnikov/master/screenshot.png)
|
4
|
+
|
5
|
+
## Warnings
|
6
|
+
|
7
|
+
It's EM, bro! Don't call sleep and other blocking stuff inside your
|
8
|
+
job's methods!
|
9
|
+
|
10
|
+
For database-related tasks consider running some of adapters
|
11
|
+
from https://github.com/igrigorik/em-synchrony (thanks @igrigorik!) or
|
12
|
+
just have 2 Resques running different queues and reschedule
|
13
|
+
|
14
|
+
This gem manually handles GC. *It must be disabled during the main loop.*
|
15
|
+
It's enabled internally each time before Redis poll. So, huge (ActiveRecord
|
16
|
+
fetch from database * many times at once) can be painful for RAM
|
17
|
+
|
18
|
+
Resque's INTERVAL is yet meaningless. It's 0. Redis is polled with blpop.
|
19
|
+
As such, please, set infinite timeout for server in /etc/redis.conf
|
20
|
+
|
21
|
+
timeout = 0
|
22
|
+
|
23
|
+
Currently, you cannot do QUEUE= * Please, list your queues
|
24
|
+
|
25
|
+
Beware hash ordering in enqueue options. If failed, these tasks would
|
26
|
+
be encountered *differently*. Retried twice, if applicable. Sort it yourself.
|
27
|
+
|
28
|
+
Resque.enqueue ResqueKalashnikov::HttpRequest, 'http://some-url', a:1, b:2
|
29
|
+
Resque.enqueue ResqueKalashnikov::HttpRequest, 'http://some-url', b:2, a:1
|
30
|
+
|
31
|
+
Be sure your Resque is not running while testing. And don't run tests
|
32
|
+
on production env
|
33
|
+
|
34
|
+
## Installation
|
35
|
+
|
36
|
+
Add this line to your application's Gemfile:
|
37
|
+
|
38
|
+
gem 'resque-kalashnikov', require: 'resque_kalashnikov'
|
39
|
+
|
40
|
+
And then execute:
|
41
|
+
|
42
|
+
$ bundle
|
43
|
+
|
44
|
+
Or install it yourself as:
|
45
|
+
|
46
|
+
$ gem install resque-kalashnikov
|
47
|
+
|
48
|
+
For WebUI:
|
49
|
+
|
50
|
+
gem 'resque-kalashnikov', require: ['resque_kalashnikov', 'resque_kalashnikov/server']
|
51
|
+
|
52
|
+
and be sure you have in config/routes.rb
|
53
|
+
|
54
|
+
mount Resque::Server, at: '/resque'
|
55
|
+
|
56
|
+
## Usage
|
57
|
+
|
58
|
+
Start it as normal Resque
|
59
|
+
|
60
|
+
QUEUE='async_queue,sync_queue' bundle exec rake environment resque:fire
|
61
|
+
|
62
|
+
Enqueue ResqueKalashnikov::HttpRequest.
|
63
|
+
|
64
|
+
Resque.enqueue ResqueKalashnikov::HttpRequest, 'http://localhost:8081/', :post, foo: 'bar'
|
65
|
+
|
66
|
+
By default it retries all http codes in range 300-600 3 times. For customizing it do your own job.
|
67
|
+
|
68
|
+
class SlowHttpRequest < ResqueKalashnikov::HttpRequest
|
69
|
+
@queue = :some_async_queue
|
70
|
+
@retry_limit = 5
|
71
|
+
end
|
72
|
+
|
73
|
+
Note, that @queue **must** match /async/
|
74
|
+
|
75
|
+
## Testing
|
76
|
+
|
77
|
+
Test suite is provided with a small EM test webserver. It can be run
|
78
|
+
manyally for acceptance tests without mocking the web.
|
79
|
+
|
80
|
+
Again, thanks @igrigorik!
|
81
|
+
|
82
|
+
Besides it's delay option, now it can also randomize HTTP anwser codes:
|
83
|
+
|
84
|
+
ruby spec/support/stub_server.rb 200 404 500
|
85
|
+
|
86
|
+
It runs on http://localhost:8081
|
87
|
+
|
88
|
+
## Build Status
|
89
|
+
|
90
|
+
[![Build Status](https://secure.travis-ci.org/razum2um/resque-kalashnikov.png?branch=master)](https://travis-ci.org/razum2um/resque-kalashnikov)
|
91
|
+
[![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/razum2um/resque-kalashnikov)
|
92
|
+
|
93
|
+
## Contributing
|
94
|
+
|
95
|
+
1. Fork it
|
96
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
97
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
98
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
99
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/demo/.gitignore
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
# See http://help.github.com/ignore-files/ for more about ignoring files.
|
2
|
+
#
|
3
|
+
# If you find yourself ignoring temporary files generated by your text editor
|
4
|
+
# or operating system, you probably want to add a global ignore instead:
|
5
|
+
# git config --global core.excludesfile ~/.gitignore_global
|
6
|
+
|
7
|
+
# Ignore bundler config
|
8
|
+
/.bundle
|
9
|
+
|
10
|
+
# Ignore the default SQLite database.
|
11
|
+
/db/*.sqlite3
|
12
|
+
|
13
|
+
# Ignore all logfiles and tempfiles.
|
14
|
+
/log/*.log
|
15
|
+
/tmp
|
data/demo/Gemfile
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
gem 'rails', '3.2.8'
|
4
|
+
|
5
|
+
# Bundle edge Rails instead:
|
6
|
+
# gem 'rails', :git => 'git://github.com/rails/rails.git'
|
7
|
+
|
8
|
+
gem 'resque', require: 'resque/server'
|
9
|
+
gem 'resque-kalashnikov', github: 'razum2um/resque-kalashnikov', branch: 'master', require: ['resque_kalashnikov', 'resque_kalashnikov/server']
|
10
|
+
|
11
|
+
group :development do
|
12
|
+
gem 'pry-rails'
|
13
|
+
gem 'pry-rescue'
|
14
|
+
gem 'pry-stack_explorer'
|
15
|
+
gem 'debugger'
|
16
|
+
end
|
17
|
+
|
18
|
+
# To use ActiveModel has_secure_password
|
19
|
+
# gem 'bcrypt-ruby', '~> 3.0.0'
|
20
|
+
|
21
|
+
# To use Jbuilder templates for JSON
|
22
|
+
# gem 'jbuilder'
|
23
|
+
|
24
|
+
# Use unicorn as the app server
|
25
|
+
# gem 'unicorn'
|
26
|
+
|
27
|
+
# Deploy with Capistrano
|
28
|
+
# gem 'capistrano'
|
29
|
+
|
30
|
+
# To use debugger
|
31
|
+
# gem 'debugger'
|
data/demo/README.rdoc
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
== Start
|
2
|
+
|
3
|
+
QUEUE='async_queue' bundle exec rake environment resque:fire
|
4
|
+
|
5
|
+
TestController#home enqueues jobs. Note if you uncomment errors they'll
|
6
|
+
retry twice!
|
7
|
+
|
8
|
+
See results here: http://httplogger.herokuapp.com/bvlog/
|
9
|
+
Use http://httplogger.herokuapp.com/bvlog/clear to reload
|
data/demo/Rakefile
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
# Add your own tasks in files placed in lib/tasks ending in .rake,
|
3
|
+
# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
|
4
|
+
|
5
|
+
require File.expand_path('../config/application', __FILE__)
|
6
|
+
|
7
|
+
ResqueKalashnikovDemo::Application.load_tasks
|
Binary file
|
@@ -0,0 +1,13 @@
|
|
1
|
+
// This is a manifest file that'll be compiled into application.js, which will include all the files
|
2
|
+
// listed below.
|
3
|
+
//
|
4
|
+
// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
|
5
|
+
// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
|
6
|
+
//
|
7
|
+
// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
|
8
|
+
// the compiled file.
|
9
|
+
//
|
10
|
+
// WARNING: THE FIRST BLANK LINE MARKS THE END OF WHAT'S TO BE PROCESSED, ANY BLANK LINE SHOULD
|
11
|
+
// GO AFTER THE REQUIRES BELOW.
|
12
|
+
//
|
13
|
+
//= require_tree .
|
@@ -0,0 +1,13 @@
|
|
1
|
+
/*
|
2
|
+
* This is a manifest file that'll be compiled into application.css, which will include all the files
|
3
|
+
* listed below.
|
4
|
+
*
|
5
|
+
* Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
|
6
|
+
* or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
|
7
|
+
*
|
8
|
+
* You're free to add application-wide styles to this file and they'll appear at the top of the
|
9
|
+
* compiled file, but it's generally better to create a new file per style scope.
|
10
|
+
*
|
11
|
+
*= require_self
|
12
|
+
*= require_tree .
|
13
|
+
*/
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class TestController < ApplicationController
|
2
|
+
layout false
|
3
|
+
|
4
|
+
def home
|
5
|
+
100.times do |n|
|
6
|
+
Resque.enqueue SlowHttpRequest, "http://httplogger.herokuapp.com/bvlog/get?n=#{n}", :get
|
7
|
+
#Resque.enqueue SlowHttpRequest, "http://httplogger.herokuapp.com/bvlog/get?n=#{n}", :get, {error: 404}
|
8
|
+
end
|
9
|
+
120.times do |n|
|
10
|
+
Resque.enqueue SlowHttpRequest, "http://httplogger.herokuapp.com/bvlog/post?n=#{n}", :post, {n10: n*10}
|
11
|
+
#Resque.enqueue SlowHttpRequest, "http://httplogger.herokuapp.com/bvlog/post?n=#{n}", :post, {error: 500}
|
12
|
+
end
|
13
|
+
redirect_to '/resque/kalashnikov'
|
14
|
+
end
|
15
|
+
end
|
File without changes
|
File without changes
|
data/demo/config.ru
ADDED
@@ -0,0 +1,65 @@
|
|
1
|
+
require File.expand_path('../boot', __FILE__)
|
2
|
+
|
3
|
+
# Pick the frameworks you want:
|
4
|
+
# require "active_record/railtie"
|
5
|
+
require "action_controller/railtie"
|
6
|
+
require "action_mailer/railtie"
|
7
|
+
require "active_resource/railtie"
|
8
|
+
# require "sprockets/railtie"
|
9
|
+
# require "rails/test_unit/railtie"
|
10
|
+
|
11
|
+
if defined?(Bundler)
|
12
|
+
# If you precompile assets before deploying to production, use this line
|
13
|
+
Bundler.require(*Rails.groups(:assets => %w(development test)))
|
14
|
+
# If you want your assets lazily compiled in production, use this line
|
15
|
+
# Bundler.require(:default, :assets, Rails.env)
|
16
|
+
end
|
17
|
+
|
18
|
+
module ResqueKalashnikovDemo
|
19
|
+
class Application < Rails::Application
|
20
|
+
# Settings in config/environments/* take precedence over those specified here.
|
21
|
+
# Application configuration should go into files in config/initializers
|
22
|
+
# -- all .rb files in that directory are automatically loaded.
|
23
|
+
|
24
|
+
# Custom directories with classes and modules you want to be autoloadable.
|
25
|
+
# config.autoload_paths += %W(#{config.root}/extras)
|
26
|
+
|
27
|
+
# Only load the plugins named here, in the order given (default is alphabetical).
|
28
|
+
# :all can be used as a placeholder for all plugins not explicitly named.
|
29
|
+
# config.plugins = [ :exception_notification, :ssl_requirement, :all ]
|
30
|
+
|
31
|
+
# Activate observers that should always be running.
|
32
|
+
# config.active_record.observers = :cacher, :garbage_collector, :forum_observer
|
33
|
+
|
34
|
+
# Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
|
35
|
+
# Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
|
36
|
+
# config.time_zone = 'Central Time (US & Canada)'
|
37
|
+
|
38
|
+
# The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
|
39
|
+
# config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
|
40
|
+
# config.i18n.default_locale = :de
|
41
|
+
|
42
|
+
# Configure the default encoding used in templates for Ruby 1.9.
|
43
|
+
config.encoding = "utf-8"
|
44
|
+
|
45
|
+
# Configure sensitive parameters which will be filtered from the log file.
|
46
|
+
config.filter_parameters += [:password]
|
47
|
+
|
48
|
+
# Enable escaping HTML in JSON.
|
49
|
+
config.active_support.escape_html_entities_in_json = true
|
50
|
+
|
51
|
+
# Use SQL instead of Active Record's schema dumper when creating the database.
|
52
|
+
# This is necessary if your schema can't be completely dumped by the schema dumper,
|
53
|
+
# like if you have constraints or database-specific column types
|
54
|
+
# config.active_record.schema_format = :sql
|
55
|
+
|
56
|
+
# Enforce whitelist mode for mass assignment.
|
57
|
+
# This will create an empty whitelist of attributes available for mass-assignment for all models
|
58
|
+
# in your app. As such, your models will need to explicitly whitelist or blacklist accessible
|
59
|
+
# parameters by using an attr_accessible or attr_protected declaration.
|
60
|
+
# config.active_record.whitelist_attributes = true
|
61
|
+
|
62
|
+
config.middleware.use PryRescue::Rack if Rails.env.development?
|
63
|
+
config.autoload_paths += Dir["#{Rails.root}/app/workers/"]
|
64
|
+
end
|
65
|
+
end
|
data/demo/config/boot.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
ResqueKalashnikovDemo::Application.configure do
|
2
|
+
# Settings specified here will take precedence over those in config/application.rb
|
3
|
+
|
4
|
+
# In the development environment your application's code is reloaded on
|
5
|
+
# every request. This slows down response time but is perfect for development
|
6
|
+
# since you don't have to restart the web server when you make code changes.
|
7
|
+
config.cache_classes = false
|
8
|
+
|
9
|
+
# Log error messages when you accidentally call methods on nil.
|
10
|
+
config.whiny_nils = true
|
11
|
+
|
12
|
+
# Show full error reports and disable caching
|
13
|
+
config.consider_all_requests_local = true
|
14
|
+
config.action_controller.perform_caching = false
|
15
|
+
|
16
|
+
# Don't care if the mailer can't send
|
17
|
+
config.action_mailer.raise_delivery_errors = false
|
18
|
+
|
19
|
+
# Print deprecation notices to the Rails logger
|
20
|
+
config.active_support.deprecation = :log
|
21
|
+
|
22
|
+
# Only use best-standards-support built into browsers
|
23
|
+
config.action_dispatch.best_standards_support = :builtin
|
24
|
+
|
25
|
+
|
26
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
ResqueKalashnikovDemo::Application.configure do
|
2
|
+
# Settings specified here will take precedence over those in config/application.rb
|
3
|
+
|
4
|
+
# Code is not reloaded between requests
|
5
|
+
config.cache_classes = true
|
6
|
+
|
7
|
+
# Full error reports are disabled and caching is turned on
|
8
|
+
config.consider_all_requests_local = false
|
9
|
+
config.action_controller.perform_caching = true
|
10
|
+
|
11
|
+
# Disable Rails's static asset server (Apache or nginx will already do this)
|
12
|
+
config.serve_static_assets = false
|
13
|
+
|
14
|
+
|
15
|
+
# Specifies the header that your server uses for sending files
|
16
|
+
# config.action_dispatch.x_sendfile_header = "X-Sendfile" # for apache
|
17
|
+
# config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for nginx
|
18
|
+
|
19
|
+
# Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
|
20
|
+
# config.force_ssl = true
|
21
|
+
|
22
|
+
# See everything in the log (default is :info)
|
23
|
+
# config.log_level = :debug
|
24
|
+
|
25
|
+
# Prepend all log lines with the following tags
|
26
|
+
# config.log_tags = [ :subdomain, :uuid ]
|
27
|
+
|
28
|
+
# Use a different logger for distributed setups
|
29
|
+
# config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new)
|
30
|
+
|
31
|
+
# Use a different cache store in production
|
32
|
+
# config.cache_store = :mem_cache_store
|
33
|
+
|
34
|
+
# Enable serving of images, stylesheets, and JavaScripts from an asset server
|
35
|
+
# config.action_controller.asset_host = "http://assets.example.com"
|
36
|
+
|
37
|
+
|
38
|
+
# Disable delivery errors, bad email addresses will be ignored
|
39
|
+
# config.action_mailer.raise_delivery_errors = false
|
40
|
+
|
41
|
+
# Enable threaded mode
|
42
|
+
# config.threadsafe!
|
43
|
+
|
44
|
+
# Enable locale fallbacks for I18n (makes lookups for any locale fall back to
|
45
|
+
# the I18n.default_locale when a translation can not be found)
|
46
|
+
config.i18n.fallbacks = true
|
47
|
+
|
48
|
+
# Send deprecation notices to registered listeners
|
49
|
+
config.active_support.deprecation = :notify
|
50
|
+
|
51
|
+
end
|