resque-kalashnikov 1.1.0

data/demo/config/environments/test.rb

@@ -0,0 +1,35 @@
+ResqueKalashnikovDemo::Application.configure do
+  # Settings specified here will take precedence over those in config/application.rb
+
+  # The test environment is used exclusively to run your application's
+  # test suite. You never need to work with it otherwise. Remember that
+  # your test database is "scratch space" for the test suite and is wiped
+  # and recreated between test runs. Don't rely on the data there!
+  config.cache_classes = true
+
+  # Configure static asset server for tests with Cache-Control for performance
+  config.serve_static_assets = true
+  config.static_cache_control = "public, max-age=3600"
+
+  # Log error messages when you accidentally call methods on nil
+  config.whiny_nils = true
+
+  # Show full error reports and disable caching
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Raise exceptions instead of rendering exception templates
+  config.action_dispatch.show_exceptions = false
+
+  # Disable request forgery protection in test environment
+  config.action_controller.allow_forgery_protection    = false
+
+  # Tell Action Mailer not to deliver emails to the real world.
+  # The :test delivery method accumulates sent emails in the
+  # ActionMailer::Base.deliveries array.
+  config.action_mailer.delivery_method = :test
+
+
+  # Print deprecation notices to the stderr
+  config.active_support.deprecation = :stderr
+end

data/demo/config/initializers/backtrace_silencers.rb

@@ -0,0 +1,7 @@
+# Be sure to restart your server when you modify this file.
+
+# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
+# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
+
+# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
+# Rails.backtrace_cleaner.remove_silencers!

data/demo/config/initializers/inflections.rb

@@ -0,0 +1,15 @@
+# Be sure to restart your server when you modify this file.
+
+# Add new inflection rules using the following format
+# (all these examples are active by default):
+# ActiveSupport::Inflector.inflections do |inflect|
+#   inflect.plural /^(ox)$/i, '\1en'
+#   inflect.singular /^(ox)en/i, '\1'
+#   inflect.irregular 'person', 'people'
+#   inflect.uncountable %w( fish sheep )
+# end
+#
+# These inflection rules are supported but not enabled by default:
+# ActiveSupport::Inflector.inflections do |inflect|
+#   inflect.acronym 'RESTful'
+# end

data/demo/config/initializers/mime_types.rb

@@ -0,0 +1,5 @@
+# Be sure to restart your server when you modify this file.
+
+# Add new mime types for use in respond_to blocks:
+# Mime::Type.register "text/richtext", :rtf
+# Mime::Type.register_alias "text/html", :iphone

data/demo/config/initializers/secret_token.rb

@@ -0,0 +1,7 @@
+# Be sure to restart your server when you modify this file.
+
+# Your secret key for verifying the integrity of signed cookies.
+# If you change this key, all old signed cookies will become invalid!
+# Make sure the secret is at least 30 characters and all random,
+# no regular words or you'll be exposed to dictionary attacks.
+ResqueKalashnikovDemo::Application.config.secret_token = '99d46a272754fe656bf667aee4b24ae1d3ae48ce1db44b1d4fa3d9eed079e009903a7b2c95517f4c44db074592423168e6815e3b742ce5cb905ee2ffbfeda32a'

data/demo/config/initializers/session_store.rb

@@ -0,0 +1,8 @@
+# Be sure to restart your server when you modify this file.
+
+ResqueKalashnikovDemo::Application.config.session_store :cookie_store, key: '_resque-kalashnikov-demo_session'
+
+# Use the database for sessions instead of the cookie-based default,
+# which shouldn't be used to store highly confidential information
+# (create the session table with "rails generate session_migration")
+# ResqueKalashnikovDemo::Application.config.session_store :active_record_store

data/demo/config/initializers/wrap_parameters.rb

@@ -0,0 +1,10 @@
+# Be sure to restart your server when you modify this file.
+#
+# This file contains settings for ActionController::ParamsWrapper which
+# is enabled by default.
+
+# Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
+ActiveSupport.on_load(:action_controller) do
+  wrap_parameters format: [:json]
+end
+

data/demo/config/locales/en.yml

@@ -0,0 +1,5 @@
+# Sample localization file for English. Add more files in this directory for other locales.
+# See https://github.com/svenfuchs/rails-i18n/tree/master/rails%2Flocale for starting points.
+
+en:
+  hello: "Hello world"

data/demo/config/routes.rb

@@ -0,0 +1,11 @@
+ResqueKalashnikovDemo::Application.routes.draw do
+
+  mount Resque::Server, at: '/resque'
+
+  match "test/home"
+
+  match "test/slow"
+
+  match "test/unreliable"
+
+end

data/demo/db/seeds.rb

@@ -0,0 +1,7 @@
+# This file should contain all the record creation needed to seed the database with its default values.
+# The data can then be loaded with the rake db:seed (or created alongside the db with db:setup).
+#
+# Examples:
+#
+#   cities = City.create([{ name: 'Chicago' }, { name: 'Copenhagen' }])
+#   Mayor.create(name: 'Emanuel', city: cities.first)

data/demo/public/404.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The page you were looking for doesn't exist (404)</title>
+  <style type="text/css">
+    body { background-color: #fff; color: #666; text-align: center; font-family: arial, sans-serif; }
+    div.dialog {
+      width: 25em;
+      padding: 0 4em;
+      margin: 4em auto 0 auto;
+      border: 1px solid #ccc;
+      border-right-color: #999;
+      border-bottom-color: #999;
+    }
+    h1 { font-size: 100%; color: #f00; line-height: 1.5em; }
+  </style>
+</head>
+
+<body>
+  <!-- This file lives in public/404.html -->
+  <div class="dialog">
+    <h1>The page you were looking for doesn't exist.</h1>
+    <p>You may have mistyped the address or the page may have moved.</p>
+  </div>
+</body>
+</html>

data/demo/public/422.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The change you wanted was rejected (422)</title>
+  <style type="text/css">
+    body { background-color: #fff; color: #666; text-align: center; font-family: arial, sans-serif; }
+    div.dialog {
+      width: 25em;
+      padding: 0 4em;
+      margin: 4em auto 0 auto;
+      border: 1px solid #ccc;
+      border-right-color: #999;
+      border-bottom-color: #999;
+    }
+    h1 { font-size: 100%; color: #f00; line-height: 1.5em; }
+  </style>
+</head>
+
+<body>
+  <!-- This file lives in public/422.html -->
+  <div class="dialog">
+    <h1>The change you wanted was rejected.</h1>
+    <p>Maybe you tried to change something you didn't have access to.</p>
+  </div>
+</body>
+</html>

data/demo/public/500.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>We're sorry, but something went wrong (500)</title>
+  <style type="text/css">
+    body { background-color: #fff; color: #666; text-align: center; font-family: arial, sans-serif; }
+    div.dialog {
+      width: 25em;
+      padding: 0 4em;
+      margin: 4em auto 0 auto;
+      border: 1px solid #ccc;
+      border-right-color: #999;
+      border-bottom-color: #999;
+    }
+    h1 { font-size: 100%; color: #f00; line-height: 1.5em; }
+  </style>
+</head>
+
+<body>
+  <!-- This file lives in public/500.html -->
+  <div class="dialog">
+    <h1>We're sorry, but something went wrong.</h1>
+  </div>
+</body>
+</html>

data/demo/public/index.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset='utf-8'>
+  </head>
+  <body>
+    <h1>Test#home</h1>
+    <form method='POST' action='/test/home'>
+      <input type='submit' value='Begin!' />
+    </form>
+  </body>
+</html>

data/demo/public/robots.txt

@@ -0,0 +1,5 @@
+# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
+#
+# To ban all spiders from the entire site uncomment the next two lines:
+# User-Agent: *
+# Disallow: /

data/demo/script/rails

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# This command will automatically be run when you run "rails" with Rails 3 gems installed from the root of your application.
+
+APP_PATH = File.expand_path('../../config/application',  __FILE__)
+require File.expand_path('../../config/boot',  __FILE__)
+require 'rails/commands'

data/demo/script/resque_async.rb

@@ -0,0 +1,29 @@
+RAILS_ENV = ENV['RAILS_ENV'] || 'development_async'
+RAILS_ROOT = Dir.pwd
+
+require 'rubygems'
+require 'yaml'
+require 'uri'
+require 'em-resque'
+require 'em-resque/worker_machine'
+require 'em-resque/task_helper'
+
+#require 'resque-kalashnikov'
+#require 'resque/plugins/kalashnikov'
+require '/www/resque-kalashnikov/lib/resque/plugins/kalashnikov'
+
+#require 'resque-retry'
+#require 'em-synchrony'
+#require 'em-synchrony/connection_pool'
+#require 'em-synchrony/mysql2'
+
+require 'debugger'
+
+#Dir.glob(File.join(RAILS_ROOT, 'lib', 'async_worker', '**', '*.rb')).sort.each{|f| require File.expand_path(f)}
+
+#resque_config = YAML.load_file("#{RAILS_ROOT}/config/resque.yml")
+#proxy_config = YAML.load_file("#{RAILS_ROOT}/config/proxy.yml")
+#PROXY = proxy_config ? proxy_config[RAILS_ENV] : nil
+
+opts = TaskHelper.parse_opts_from_env #.merge(:redis => resque_config[RAILS_ENV])
+EM::Resque::WorkerMachine.new(opts).start

data/demo/zeus.json

@@ -0,0 +1,22 @@
+{
+  "command": "ruby -rubygems -rzeus/rails -eZeus.go",
+
+  "plan": {
+    "boot": {
+      "default_bundle": {
+        "development_environment": {
+          "prerake": {"rake": []},
+          "runner": ["r"],
+          "console": ["c"],
+          "server": ["s"],
+          "generate": ["g"],
+          "dbconsole": []
+        },
+        "test_environment": {
+          "cucumber_environment": {"cucumber": []},
+          "test_helper": {"test": ["rspec", "testrb"]}
+        }
+      }
+    }
+  }
+}

data/lib/event_machine/forced_stop.rb

@@ -0,0 +1,3 @@
+module EventMachine
+  class ForcedStop < Exception; end
+end

data/lib/resque/catridge.rb

@@ -0,0 +1,87 @@
+module Resque
+  class Catridge
+    attr_reader :request, :response
+
+    def initialize(request, response)
+      @request = request
+      @response = response
+      log if misfire?
+      inc_stat_counter
+    end
+
+    def reload?
+      misfire? && !ran_out_of_ammo? 
+    end
+
+    def retries
+      self.class.redis.hget("#{self.class.ns}:misfires:#{status}", serialized_request).to_i
+    end
+
+    private
+
+    def serialized_request
+      Resque.encode([request.class.to_s, [request.url, request.http_method, request.reload_opts]])
+    end
+
+    def log
+      self.class.redis.hincrby "#{self.class.ns}:misfires:#{status}", serialized_request, 1
+    end
+
+    def inc_stat_counter
+      self.class.redis.hincrby "#{self.class.ns}:stat", status, 1
+    end
+
+    def status
+      @status ||= response.response_header.status
+    end
+
+    def misfire?
+      case status
+      when 300..600 then true
+      else
+        false
+      end
+    end
+
+    def ran_out_of_ammo?
+      # do something meaningfull
+      false
+    end
+
+    class << self
+      def ns
+        'kalashnikov'
+      end
+
+      # DI here, please
+      def redis
+        Resque.redis
+      end
+
+      def stats
+        redis.hgetall "#{ns}:stat"
+      end
+
+      def misfire_codes
+        redis.keys "#{ns}:misfires:*"
+      end
+
+      def misfire_stats(status)
+        redis.hgetall "#{status}"
+        #redis.hgetall "#{ns}:misfires:#{status}"
+      end
+
+      def misfire_stats_reset(status, request)
+        redis.hdel "#{status}", "#{request}"
+        #redis.hgetall "#{ns}:misfires:#{status}"
+      end
+
+      def reset_stats
+        stats.keys.each do |http_code|
+          redis.del "#{ns}:misfires:#{http_code}"
+        end
+        redis.del "#{ns}:stat"
+      end
+    end
+  end
+end

data/lib/resque/plugins/resque_kalashnikov/resque_kalashnikov.rb

@@ -0,0 +1,130 @@
+module Resque::Plugins
+  module ResqueKalashnikov
+
+    def work_with_kalashnikov(interval=5.0, &block)
+      interval = Float(interval)
+      @fibers = []
+      startup
+
+      loop do
+        break if shutdown?
+        job = reserve
+
+        log "got job in worker fiber: #{job.inspect}"
+        job.worker = self
+
+        working_on job
+
+        if can_do_job_async? job
+          @fibers << work_async_on(job, &block)
+        else
+          work_sync_on(job, &block)
+          @child = nil
+        end
+        monitor(interval)
+        done_working
+      end
+      unregister_worker
+
+    rescue EM::ForcedStop => e
+      # happens in fiber-mode
+      # EM has stopped but we need 
+      # to reconnect to report it
+      Resque.redis = Redis.connect
+      unregister_worker
+    rescue Resque::Helpers::DecodeException => e
+      # agian, happens in fork-mode
+      raise e unless e.to_s['Redis disconnected']
+      Resque.redis = Redis.connect
+      unregister_worker
+
+    rescue Exception => exception
+      log exception.to_s
+      log exception.backtrace.to_s
+      unregister_worker(exception)
+    end
+
+    def inspect_with_kalashnikov
+      "#<KalashnikovWorker #{to_s}>"
+    end
+
+    def work_sync_on(job, &block)
+      log 'work sync'
+      if @child = fork(job)
+        srand # Reseeding
+        procline "Forked #{@child} at #{Time.now.to_i}"
+        begin
+          Process.waitpid(@child)
+        rescue SystemCallError
+          nil
+        end
+        job.fail(DirtyExit.new($?.to_s)) if $?.signaled?
+      else
+        unregister_signal_handlers if will_fork? && term_child
+        procline "Processing #{job.queue} since #{Time.now.to_i}"
+        #reconnect # cannot do it with hiredis
+        perform(job, &block)
+        exit!(true) if will_fork?
+      end
+    end
+
+    def work_async_on(job, &block)
+      log "work async"
+      Fiber.new do
+        perform(job, &block)
+      end.tap &:resume
+    end
+
+    # if resque worker gonna to stop - stop EM
+    # essentially, fiber-singleton
+    def monitor(interval)
+      @monitor ||= Fiber.new do
+        EM.add_periodic_timer(interval) do
+          # monitor itself doesnt count in @fibers
+          if (@fibers = @fibers.select(&:alive?)).empty?
+            EM.stop if shutdown?
+          else
+            log! "Big brother says: #{@fibers.size} fibers alive"
+            log! ObjectSpace.count_objects.to_s
+          end
+        end
+      end.tap &:resume
+    end
+
+    # test whenether we can do job async
+    # based on its name
+    def can_do_job_async?(job)
+      !! job.queue['async']
+    end
+
+    def reserve_with_kalashnikov
+      queues = Resque.queues.map { |q| "queue:#{q}" }
+
+      # NO block for EM since using hiredis + em-synchrony
+      GC.enable
+      queue, value = redis.blpop(*queues, 0)
+      GC.disable
+
+      # shit happens if monitor fiber stops EM
+      # it should happen only in tests
+      raise EM::ForcedStop.new(queue) if queue['Redis disconnected']
+
+      log "popped: q=#{queue} v=#{value}"
+      payload = decode value
+      Resque::Job.new queue, payload
+    end
+
+    def self.included(receiver)
+      receiver.class_eval do
+        alias work_without_kalashnikov work
+        alias work work_with_kalashnikov
+
+        alias inspect_without_kalashnikov inspect
+        alias inspect inspect_with_kalashnikov
+
+        alias reserve_without_kalashnikov reserve
+        alias reserve reserve_with_kalashnikov
+      end
+    end
+  end # ResqueKalashnikov
+end # Resque::Plugins