resque-kalashnikov 1.1.0

data/spec/http_request_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+
+describe 'ResqueKalashnikov::HttpRequest' do
+
+  def build(attrs={})
+    url = attrs.delete('url') || async_server_url
+    http_method = attrs.delete('http_method') || 'get'
+    ResqueKalashnikov::HttpRequest.new(url, http_method, attrs)
+  end
+
+  def success_response
+    { :status => 200, :body => "success", :headers => {} }
+  end
+
+  ['get', 'post', 'delete', 'put'].each do |http_method|
+    it "stores valid #{http_method} methods" do
+      stub_request(http_method.to_sym, async_server_url).to_return success_response
+      EM.synchrony do
+        build("http_method" => http_method).send(:http_request).response.should == 'success'
+        EM.stop
+      end
+    end
+  end
+
+  ['head', 'options', 'patch'].each do |http_method|
+    it "stores valid #{http_method} methods" do
+      stub_request(http_method.to_sym, async_server_url).to_return success_response
+      EM.synchrony do
+        build("http_method" => http_method).send(:http_request).response.should == ''
+        EM.stop
+      end
+    end
+  end
+
+  it 'stores invalid methods as GET' do
+    stub_request(:get, async_server_url).to_return success_response
+    EM.synchrony do
+      build.send(:http_request).response.should == 'success'
+      build('http_method' => 'foobar').send(:http_request).response.should == 'success'
+      EM.stop
+    end
+  end
+
+  it 'handles misc options' do
+    stub_request(:get, async_server_url)
+      .with(:query => {"a" => ["b", "c"], "d" => "e"})
+      .to_return(success_response)
+
+    stub_request(:post,  "http://127.0.0.1:8081/?d=e&n=1&kind=async")
+      .to_return(success_response)
+
+    EM.synchrony do
+      build(
+        "a" => ["b", "c"],
+        "d" => "e"
+      ).send(:http_request).response.should == 'success'
+
+      build(
+        "url" => async_server_url({n: 1, kind: 'async'}),
+        "http_method" => "post",
+        "d" => "e"
+      ).send(:http_request).response.should == 'success'
+      EM.stop
+    end
+  end
+
+  describe 'error handling' do
+    it 'handles 4xx responses' do
+      stub_request(:any, async_server_url).to_return(:status => [404, "Not Found"])
+      EM.synchrony do
+        build.send(:http_request).response_header.status.should == 404
+        EM.stop
+      end
+    end
+
+    it 'handles 5xx responses' do
+      stub_request(:any, async_server_url).to_return(:status => [500, "Internal Server Error"])
+      EM.synchrony do
+        build.send(:http_request).response_header.status.should == 500
+        EM.stop
+      end
+    end
+  end
+end

data/spec/server_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+require "resque_kalashnikov/server"
+
+ENV['RACK_ENV'] = 'test'
+
+describe ResqueKalashnikov::Server do
+  require 'rack/test'
+  include Rack::Test::Methods
+
+  def app
+    Resque::Server
+  end
+
+  it 'has Kalashnikov tab' do
+    get '/overview'
+    last_response.body.should =~ /kalashnikov/
+  end
+
+  it 'can show Kalashnikov tab' do
+    get '/kalashnikov'
+    last_response.status.should == 200
+  end
+
+  describe 'actions' do
+    before do
+      @request_key = Base64.encode64 Resque.encode [SlowHttpRequest, [async_server_url, {method: 'post'}]]
+      @fake_redis = double
+      Redis.stub(:connect).and_return(@fake_redis)
+    end
+
+    it 'can retry jobs' do
+      @fake_redis.should_receive(:rpush).once
+      post '/kalashnikov/retry/500', {request_key: @request_key}
+    end
+
+    it 'can reset misfire count' do
+      @fake_redis.should_receive(:rpush).once
+      post '/kalashnikov/retry/500', {request_key: @request_key}
+    end
+
+    it 'can reset everyhing' do
+      Resque::Catridge.should_receive(:reset_stats)
+      get '/kalashnikov/reset_stats'
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'resque'
+require 'em-synchrony'
+require 'em-synchrony/em-hiredis'
+require 'webmock/rspec'
+
+$dir = File.dirname(File.expand_path(__FILE__))
+$LOAD_PATH.unshift $dir + '/../lib'
+require 'resque_kalashnikov'
+require 'support/stub_server'
+$TESTING = true
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+  config.order = 'random'
+
+  config.before do
+    Redis.connect.flushall
+  end
+end
+
+class SlowHttpRequest < ResqueKalashnikov::HttpRequest
+  @queue = :async_queue
+
+  def handle http
+    super
+    File.open("/tmp/kalashnikov-#{$$}.log", "a") do |f|
+      f.write "#{DateTime.now}:#{http.response_header.status}:#{http.response}\n"
+    end
+  end
+end
+
+def async_server(response_status=200, delay=0)
+  WebMock.allow_net_connect!
+  EM.synchrony do
+    #EM::HttpRequest.new('http://httplogger.herokuapp.com/bvlog/clear').get if async_server_url['herokuapp']
+    Resque.redis = EM::Synchrony::ConnectionPool.new(size: 3) do
+      EM::Hiredis.connect
+    end
+    #Resque.redis.flushall # reset state in Resque object
+    s = StubServer.new response_status, delay
+    yield
+    s.stop
+  end
+end
+
+def async_server_url(attrs={})
+  #return "http://httplogger.herokuapp.com/bvlog/get"
+  if attrs.empty?
+    "http://127.0.0.1:8081"
+  else
+    "http://127.0.0.1:8081/?n=#{attrs[:n]}&kind=#{attrs[:kind]}"
+  end
+end

data/spec/support/stub_server.rb

@@ -0,0 +1,46 @@
+require 'eventmachine'
+
+# thanks @igrigorik
+# https://github.com/igrigorik/em-synchrony/blob/master/spec/helper/stub-http-server.rb
+class StubServer
+  module Server
+    attr_accessor :responses, :delay
+    def receive_data(data)
+      EM.add_timer(@delay) {
+        send_data @responses.sample
+        close_connection_after_writing
+      }
+    end
+  end
+
+  def initialize(response_code=[200], delay=0, port=8081)
+    @sig = EventMachine::start_server("127.0.0.1", port, Server) { |s|
+      response_map = {
+        "500" => "HTTP/1.0 500 Internal Server Error\r\nConnection: close\r\n\r\nFail: 500",
+        "404" => "HTTP/1.0 404 Not Found\r\nConnection: close\r\n\r\nNot Found: 404",
+        "200" => "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\nSuccess",
+      }
+      s.responses = response_map.select { |k,v| response_code.to_s.include? k } .values
+      s.delay = delay
+    }
+  end
+
+  def stop
+    EventMachine.stop_server @sig
+  end
+end
+
+if __FILE__ == $0
+  EM.run do
+    s = StubServer.new(ARGV || [200], 0)
+    puts 'Started on http://localhost:8081/'
+
+    ['TERM', 'INT', 'QUIT'].each do |signal|
+      trap(signal) do
+        puts 'Finished'
+        s.stop
+        EM.stop
+      end
+    end
+  end
+end

data/spec/worker_spec.rb

@@ -0,0 +1,113 @@
+require 'spec_helper'
+
+DELAY = 0.5
+
+describe 'Resque::Worker' do
+  before do
+    Resque.before_first_fork = nil
+    Resque.before_fork = nil
+    Resque.after_fork = nil
+    @worker = Resque::Worker.new(:async_queue, :sync_queue)
+    #@worker.verbose = true # useful to see sync/async difference
+  end
+
+  def now(); Time.now.to_f; end
+
+  def create_async_job(attrs={})
+    attrs.merge! kind: 'async'
+    Resque::Job.create(:async_queue, SlowHttpRequest, async_server_url(attrs), attrs)
+  end
+
+  def create_sync_job(attrs={})
+    attrs.merge! kind: 'sync'
+    Resque::Job.create(:sync_queue, SlowHttpRequest, async_server_url(attrs), attrs)
+  end
+
+  it 'has proper name' do
+    @worker.inspect.should =~ /KalashnikovWorker/
+  end
+
+  it 'makes acquires jobs with Hiredis#blpop' do
+    create_sync_job
+    @worker.reserve.should be_kind_of Resque::Job
+  end
+
+  it 'differs async jobs' do
+    create_async_job
+    @worker.can_do_job_async?(Resque::Job.reserve :async_queue).should be_true
+  end
+
+  it 'differs sync jobs' do
+    create_sync_job
+    @worker.can_do_job_async?(Resque::Job.reserve :sync_queue).should be_false
+  end
+
+  it 'handles async queues with fibers' do
+    queue_size = 20
+    queue_size.times { |n| create_async_job(n: n) }
+
+    async_server([200], DELAY) do
+      start = now
+      @worker.work(0) do |job|
+        # block yields in Worker#perform in "ensure" part
+        #
+        # this is too early - Fiber hasn't finished till this moment
+        # if theirs count is small, it's still ok
+        #
+        # if there were 100 jobs - they'll have managed ~60
+        # but uncommenting "if" statement leads to ever-hook in EM
+        # having ~30 alive fibers after having done 100 jobs
+        #
+        # TLDR: kalashnikov-worker is hard to stop with block - use signals
+        @worker.shutdown #if Resque.size(:async_queue) == 0
+      end
+
+      # O(1)
+      (now - start).should be_within(DELAY*2).of(DELAY)
+    end
+  end
+
+  it 'handles sync queues with processes' do
+    queue_size = 3
+    queue_size.times { |n| create_sync_job(n: n) }
+
+    async_server([200], DELAY) do
+      start = now
+      @worker.work(0) do |job|
+        @worker.shutdown if Resque.size(:sync_queue) == 0
+      end
+
+      # O(n)
+      (now - start).should be_within(DELAY*queue_size*2).of(DELAY*queue_size)
+    end
+  end
+
+  it 'handles 404 & 500 error async' do
+    queue_size = 20
+    queue_size.times { |n| create_async_job(n: n) }
+    async_server([500, 404], DELAY) do
+      start = now
+      @worker.work(0) do |job|
+        @worker.shutdown #if Resque.size(:async_queue) == 0
+      end
+
+      # O(1)
+      (now - start).should be_within(DELAY*2).of(DELAY)
+    end
+  end
+
+  it 'handles random async jobs' do
+    queue_size = 20
+    queue_size.times { |n| create_async_job(n: n) }
+
+    async_server([200, 404, 500], DELAY) do
+      start = now
+      @worker.work(0) do |job|
+        @worker.shutdown #if Resque.size(:async_queue) == 0
+      end
+
+      # O(1)
+      (now - start).should be_within(DELAY*2).of(DELAY)
+    end
+  end
+end

data/tasks/resque_kalashnikov.rake

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'resque_kalashnikov/tasks'

metadata

@@ -0,0 +1,267 @@
+--- !ruby/object:Gem::Specification
+name: resque-kalashnikov
+version: !ruby/object:Gem::Version
+  version: 1.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Vlad Bokov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.23.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.23.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>'
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-synchrony
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-hiredis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Handles your HTTP requests in background in non-blocking way using Resque
+  worker
+email:
+- bokov.vlad@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- demo/.gitignore
+- demo/Gemfile
+- demo/README.rdoc
+- demo/Rakefile
+- demo/app/assets/images/rails.png
+- demo/app/assets/javascripts/application.js
+- demo/app/assets/stylesheets/application.css
+- demo/app/controllers/application_controller.rb
+- demo/app/controllers/test_controller.rb
+- demo/app/helpers/application_helper.rb
+- demo/app/mailers/.gitkeep
+- demo/app/models/.gitkeep
+- demo/app/workers/slow_http_request.rb
+- demo/config.ru
+- demo/config/application.rb
+- demo/config/boot.rb
+- demo/config/environment.rb
+- demo/config/environments/development.rb
+- demo/config/environments/production.rb
+- demo/config/environments/test.rb
+- demo/config/initializers/backtrace_silencers.rb
+- demo/config/initializers/inflections.rb
+- demo/config/initializers/mime_types.rb
+- demo/config/initializers/secret_token.rb
+- demo/config/initializers/session_store.rb
+- demo/config/initializers/wrap_parameters.rb
+- demo/config/locales/en.yml
+- demo/config/routes.rb
+- demo/db/seeds.rb
+- demo/lib/assets/.gitkeep
+- demo/lib/tasks/.gitkeep
+- demo/log/.gitkeep
+- demo/public/404.html
+- demo/public/422.html
+- demo/public/500.html
+- demo/public/favicon.ico
+- demo/public/index.html
+- demo/public/robots.txt
+- demo/script/rails
+- demo/script/resque_async.rb
+- demo/vendor/assets/javascripts/.gitkeep
+- demo/vendor/assets/stylesheets/.gitkeep
+- demo/vendor/plugins/.gitkeep
+- demo/zeus.json
+- lib/event_machine/forced_stop.rb
+- lib/resque/catridge.rb
+- lib/resque/plugins/resque_kalashnikov/resque_kalashnikov.rb
+- lib/resque_kalashnikov.rb
+- lib/resque_kalashnikov/delegation.rb
+- lib/resque_kalashnikov/http_request.rb
+- lib/resque_kalashnikov/railtie.rb
+- lib/resque_kalashnikov/server.rb
+- lib/resque_kalashnikov/server/views/catridges.erb
+- lib/resque_kalashnikov/version.rb
+- lib/tasks.rb
+- resque-kalashnikov.gemspec
+- screenshot.png
+- spec/catridge_spec.rb
+- spec/http_request_spec.rb
+- spec/server_spec.rb
+- spec/spec_helper.rb
+- spec/support/stub_server.rb
+- spec/worker_spec.rb
+- tasks/resque_kalashnikov.rake
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: This is awesome
+test_files:
+- spec/catridge_spec.rb
+- spec/http_request_spec.rb
+- spec/server_spec.rb
+- spec/spec_helper.rb
+- spec/support/stub_server.rb
+- spec/worker_spec.rb