resque-kalashnikov 1.1.0

data/lib/resque_kalashnikov.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'em-synchrony'
+
+require "resque_kalashnikov/delegation"
+require 'resque/worker'
+require 'resque/catridge'
+require "resque/plugins/resque_kalashnikov/resque_kalashnikov"
+require "event_machine/forced_stop"
+require "resque_kalashnikov/http_request"
+require "resque_kalashnikov/railtie" if defined?(Rails)
+
+module ResqueKalashnikov
+  delegate :stats, :misfire_codes, :misfire_stats, :misfire_stats_reset, :reset_stats, to: Resque::Catridge, prefix: 'kalashnikov'
+end
+
+Resque.extend ResqueKalashnikov
+Resque::Worker.send(:include, Resque::Plugins::ResqueKalashnikov)

data/lib/resque_kalashnikov/delegation.rb

@@ -0,0 +1,153 @@
+class Module
+  # Provides a delegate class method to easily expose contained objects' methods
+  # as your own. Pass one or more methods (specified as symbols or strings)
+  # and the name of the target object via the <tt>:to</tt> option (also a symbol
+  # or string). At least one method and the <tt>:to</tt> option are required.
+  #
+  # Delegation is particularly useful with Active Record associations:
+  #
+  #   class Greeter < ActiveRecord::Base
+  #     def hello
+  #       "hello"
+  #     end
+  #
+  #     def goodbye
+  #       "goodbye"
+  #     end
+  #   end
+  #
+  #   class Foo < ActiveRecord::Base
+  #     belongs_to :greeter
+  #     delegate :hello, :to => :greeter
+  #   end
+  #
+  #   Foo.new.hello   # => "hello"
+  #   Foo.new.goodbye # => NoMethodError: undefined method `goodbye' for #<Foo:0x1af30c>
+  #
+  # Multiple delegates to the same target are allowed:
+  #
+  #   class Foo < ActiveRecord::Base
+  #     belongs_to :greeter
+  #     delegate :hello, :goodbye, :to => :greeter
+  #   end
+  #
+  #   Foo.new.goodbye # => "goodbye"
+  #
+  # Methods can be delegated to instance variables, class variables, or constants
+  # by providing them as a symbols:
+  #
+  #   class Foo
+  #     CONSTANT_ARRAY = [0,1,2,3]
+  #     @@class_array  = [4,5,6,7]
+  #
+  #     def initialize
+  #       @instance_array = [8,9,10,11]
+  #     end
+  #     delegate :sum, :to => :CONSTANT_ARRAY
+  #     delegate :min, :to => :@@class_array
+  #     delegate :max, :to => :@instance_array
+  #   end
+  #
+  #   Foo.new.sum # => 6
+  #   Foo.new.min # => 4
+  #   Foo.new.max # => 11
+  #
+  # Delegates can optionally be prefixed using the <tt>:prefix</tt> option. If the value
+  # is <tt>true</tt>, the delegate methods are prefixed with the name of the object being
+  # delegated to.
+  #
+  #   Person = Struct.new(:name, :address)
+  #
+  #   class Invoice < Struct.new(:client)
+  #     delegate :name, :address, :to => :client, :prefix => true
+  #   end
+  #
+  #   john_doe = Person.new("John Doe", "Vimmersvej 13")
+  #   invoice = Invoice.new(john_doe)
+  #   invoice.client_name    # => "John Doe"
+  #   invoice.client_address # => "Vimmersvej 13"
+  #
+  # It is also possible to supply a custom prefix.
+  #
+  #   class Invoice < Struct.new(:client)
+  #     delegate :name, :address, :to => :client, :prefix => :customer
+  #   end
+  #
+  #   invoice = Invoice.new(john_doe)
+  #   invoice.customer_name    # => "John Doe"
+  #   invoice.customer_address # => "Vimmersvej 13"
+  #
+  # If the delegate object is +nil+ an exception is raised, and that happens
+  # no matter whether +nil+ responds to the delegated method. You can get a
+  # +nil+ instead with the +:allow_nil+ option.
+  #
+  #  class Foo
+  #    attr_accessor :bar
+  #    def initialize(bar = nil)
+  #      @bar = bar
+  #    end
+  #    delegate :zoo, :to => :bar
+  #  end
+  #
+  #  Foo.new.zoo   # raises NoMethodError exception (you called nil.zoo)
+  #
+  #  class Foo
+  #    attr_accessor :bar
+  #    def initialize(bar = nil)
+  #      @bar = bar
+  #    end
+  #    delegate :zoo, :to => :bar, :allow_nil => true
+  #  end
+  #
+  #  Foo.new.zoo   # returns nil
+  #
+  def delegate(*methods)
+    options = methods.pop
+    unless options.is_a?(Hash) && to = options[:to]
+      raise ArgumentError, "Delegation needs a target. Supply an options hash with a :to key as the last argument (e.g. delegate :hello, :to => :greeter)."
+    end
+    prefix, to, allow_nil = options[:prefix], options[:to], options[:allow_nil]
+
+    if prefix == true && to.to_s =~ /^[^a-z_]/
+      raise ArgumentError, "Can only automatically set the delegation prefix when delegating to a method."
+    end
+
+    method_prefix =
+      if prefix
+        "#{prefix == true ? to : prefix}_"
+      else
+        ''
+      end
+
+    file, line = caller.first.split(':', 2)
+    line = line.to_i
+
+    methods.each do |method|
+      method = method.to_s
+
+      if allow_nil
+        module_eval(<<-EOS, file, line - 2)
+          def #{method_prefix}#{method}(*args, &block)        # def customer_name(*args, &block)
+            if #{to} || #{to}.respond_to?(:#{method})         #   if client || client.respond_to?(:name)
+              #{to}.__send__(:#{method}, *args, &block)       #     client.__send__(:name, *args, &block)
+            end                                               #   end
+          end                                                 # end
+        EOS
+      else
+        exception = %(raise "#{self}##{method_prefix}#{method} delegated to #{to}.#{method}, but #{to} is nil: \#{self.inspect}")
+
+        module_eval(<<-EOS, file, line - 1)
+          def #{method_prefix}#{method}(*args, &block)        # def customer_name(*args, &block)
+            #{to}.__send__(:#{method}, *args, &block)         #   client.__send__(:name, *args, &block)
+          rescue NoMethodError                                # rescue NoMethodError
+            if #{to}.nil?                                     #   if client.nil?
+              #{exception}                                    #     # add helpful message to the exception
+            else                                              #   else
+              raise                                           #     raise
+            end                                               #   end
+          end                                                 # end
+        EOS
+      end
+    end
+  end
+end

data/lib/resque_kalashnikov/http_request.rb

@@ -0,0 +1,64 @@
+require "em-synchrony/em-http"
+
+module ResqueKalashnikov
+  class HttpRequest
+    attr_accessor :url, :http_method, :opts
+
+    def initialize(*args)
+      case args.size
+        when 1 then @url = args[0]
+        when 2 then @url, @opts = args
+        when 3 then @url, @http_method, @opts = args
+      else
+        raise "insufficient params in #{self.class}: args=#{args}"
+      end
+      @http_method ||= 'get'
+      @opts ||= {}
+      @http_method.downcase!
+    end
+
+    # This method is invoked inside EM
+    # no blocking calls, please
+    def handle http
+      Resque::Catridge.new(self, http)
+    end
+
+    def retry_limit
+      instance_variable_get(:@retry_limit) || 2
+    end
+
+    def perform
+      catrige = handle http_request
+      reload if catrige.reload? && catrige.retries < retry_limit
+      http_request.response
+    end
+
+    def reload_opts
+      opts
+    end
+
+    def http_method
+      valid_methods.include?(@http_method) ? @http_method : 'get'
+    end
+
+    private
+
+    def reload
+      Resque.enqueue self.class, url, http_method, reload_opts
+    end
+
+    def http_request
+      EM::HttpRequest.new(url).send http_method, query: opts
+    end
+
+    def valid_methods
+      ['get', 'post', 'head', 'delete', 'put', 'options', 'patch']
+    end
+
+    class << self
+      def perform(*args)
+        new(*args).perform
+      end
+    end
+  end
+end

data/lib/resque_kalashnikov/railtie.rb

@@ -0,0 +1,10 @@
+require 'resque_kalashnikov'
+require 'rails'
+
+module ResqueKalashnikov
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      require 'resque_kalashnikov/../tasks'
+    end
+  end
+end

data/lib/resque_kalashnikov/server.rb

@@ -0,0 +1,50 @@
+require 'resque_kalashnikov'
+require 'resque/server'
+
+module ResqueKalashnikov
+
+  module Server
+
+    def render_erb(view)
+      erb File.read(File.join(File.dirname(__FILE__), view))
+    end
+
+    def self.included(base)
+      base.class_eval do
+        get "/kalashnikov" do
+          render_erb 'server/views/catridges.erb'
+        end
+
+        post "/kalashnikov/retry/:status" do
+          status = params[:status]
+          klass_name, args = Resque.decode Base64.decode64 params[:request_key]
+          klass = Resque::Job.constantize klass_name
+          queue = Resque.queue_from_class(klass)
+          redis = Redis.connect
+          redis.rpush "resque:queue:#{queue}", Resque.encode(:class => klass_name, :args => args)
+          sleep 3 # FIXME: if refreshed immediately - no effect seen - instead, can be polled 
+          redirect u('/kalashnikov')
+        end
+
+        post "/kalashnikov/reset/:status" do
+          status = params[:status]
+          request_key = Base64.decode64 params[:request_key]
+          redis = Redis.connect
+          redis.hset "resque:kalashnikov:misfires:#{status}", request_key, 0
+          redirect u('/kalashnikov')
+        end
+
+        get "/kalashnikov/reset_stats" do
+          Resque::Catridge.reset_stats
+          redirect u('/kalashnikov')
+        end
+      end
+    end
+
+    Resque::Server.tabs << 'Kalashnikov'
+  end
+end
+
+Resque::Server.class_eval do
+  include ResqueKalashnikov::Server
+end

data/lib/resque_kalashnikov/server/views/catridges.erb

@@ -0,0 +1,57 @@
+<% statuses = resque.kalashnikov_stats %>
+
+<h1 class='wi'>Kalashnikov</h1>
+<p class='intro'>The list below contains statistics of Kalashnikov</p>
+<table>
+  <tr>
+    <th>HTTP code</th>
+    <th>Jobs done</th>
+  </tr>
+  <% statuses.sort.each do |status, count| %>
+  <tr>
+    <td class='status'><%= status %></td>
+    <td class='size'><%= count %></td>
+  </tr>
+  <% end %>
+</table>
+
+<% resque.kalashnikov_misfire_codes.each do |misfire_stats_key| %>
+<% status = misfire_stats_key.split(':').last %>
+<h1 class='wi'>Error code: <%= status %></h1>
+<table>
+  <tr>
+    <th>Job</th>
+    <th>HTTP</th>
+    <th>URL</th>
+    <th>Options</th>
+    <th>Retries</th>
+    <th>Retry</th>
+    <th>Reset retries</th>
+  </tr>
+  <% resque.kalashnikov_misfire_stats(misfire_stats_key).sort_by { |k,v| v } .reverse.each do |misfire_json, count| %>
+  <tr>
+    <% job, payload = resque.decode misfire_json %>
+    <% url, http_method, options = payload %>
+    <td class='job'><%= job %></td>
+    <td class='http_method'><%= http_method.upcase %></td>
+    <td class='url'><a href="<%= url %>"><%= url %></a></td>
+    <td class='options'><%= options %></td>
+    <td class='size'><%= count %></td>
+    <td class='url'>
+      <form method="POST" action="<%= u %Q{kalashnikov/retry/#{status}} %>" >
+        <input name="request_key" type="hidden" value="<%= Base64.encode64 misfire_json %>" />
+        <input type='submit' name='' value='Retry' />
+      </form>
+     </td>
+    <td class='url'>
+      <form method="POST" action="<%= u %Q{kalashnikov/reset/#{status}} %>" >
+        <input name="request_key" type="hidden" value="<%= Base64.encode64 misfire_json %>" />
+        <input type='submit' name='' value='Reset' />
+      </form>
+     </td>
+  </tr>
+  <% end %>
+</table>
+<% end %>
+
+<a href="<%= u %Q{kalashnikov/reset_stats} %>">Reset Kalashnikov statistics</a>

data/lib/resque_kalashnikov/version.rb

@@ -0,0 +1,3 @@
+module ResqueKalashnikov
+  VERSION = '1.1.0'
+end

data/lib/tasks.rb

@@ -0,0 +1,34 @@
+namespace :resque do
+
+  desc "Fire Kalashnikov"
+  task :fire do
+
+    require 'resque'
+    require 'em-synchrony'
+    require 'em-synchrony/em-hiredis'
+    require 'resque_kalashnikov'
+
+    queues = (ENV['QUEUES'] || ENV['QUEUE']).to_s.split(',')
+
+    # FIXME: cannot start with clean redis
+    abort "QUEUE env var cannot be '*', please, list your queues" if queues.include? '*'
+    redis = Redis.connect
+    queues.each { |queue| redis.sadd "resque:queues", queue }
+
+
+    if defined?(Rails) && Rails.respond_to?(:application)
+      Rails.application.eager_load!
+    end
+
+    worker = Resque::Worker.new(*queues)
+    #worker.verbose = true
+
+    EM.synchrony do
+      Resque.redis = EM::Synchrony::ConnectionPool.new(size: 100) do
+        EM::Hiredis.connect
+      end
+      ['TERM', 'INT', 'QUIT'].each { |signal| trap(signal) { EM.stop } }
+      worker.work(0)
+    end
+  end
+end

data/resque-kalashnikov.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque_kalashnikov/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "resque-kalashnikov"
+  gem.version       = ResqueKalashnikov::VERSION
+  gem.authors       = ["Vlad Bokov"]
+  gem.email         = ["bokov.vlad@gmail.com"]
+  gem.summary       = %q{This is awesome}
+  gem.description   = %q{Handles your HTTP requests in background in non-blocking way using Resque worker}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_dependency('rake')
+  gem.add_dependency('resque', '~> 1.23.0')
+  gem.add_dependency('redis', '> 3.0.0')
+  gem.add_dependency('em-http-request')
+  gem.add_dependency('em-synchrony')
+  gem.add_dependency('em-hiredis')
+
+  gem.add_development_dependency('rack-test')
+  gem.add_development_dependency('webmock')
+  gem.add_development_dependency('rspec')
+end

data/spec/catridge_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+describe Resque::Catridge do
+  before do
+    @fake_redis = double
+    @fake_redis.stub(:hget)
+    @fake_redis.stub(:rpush)
+    @fake_redis.stub(:hincrby)
+    @fake_redis.stub(:keys)
+    Resque::Catridge.stub(:redis).and_return(@fake_redis)
+  end
+
+  def fake_response_with_status(status)
+    response = double
+    response.stub_chain(:response_header, :status).and_return(status)
+    response
+  end
+
+  def fake_request
+    request = double
+    request.stub(:url).and_return 'some-url'
+    request.stub(:http_method).and_return 'get'
+    request.stub(:reload_opts).and_return 'some-opts'
+    request
+  end
+
+  def build(response)
+    Resque::Catridge.new fake_request, response
+  end
+
+  it 'forces no reload on 200' do
+    build(fake_response_with_status 200).reload?.should be_false
+  end
+
+  it 'forces 404 for reload' do
+    build(fake_response_with_status 404).reload?.should be_true
+  end
+
+  it 'forces 500 for reload' do
+    build(fake_response_with_status 500).reload?.should be_true
+  end
+end