queueing_proxy 0.0.4 → 0.0.5

data/.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+pkg/

data/.rvmrc

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional.
+environment_id="ruby-1.9.2-p290@queueing_proxy"
+
+#
+# Uncomment following line if you want options to be set only for given project.
+#
+# PROJECT_JRUBY_OPTS=( --1.9 )
+
+#
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+#
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments" \
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+
+  if [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]]
+  then
+    . "${rvm_path:-$HOME/.rvm}/hooks/after_use"
+  fi
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  if ! rvm --create  "$environment_id"
+  then
+    echo "Failed to create RVM environment '${environment_id}'."
+    exit 1
+  fi
+fi
+
+#
+# If you use an RVM gemset file to install a list of gems (*.gems), you can have
+# it be automatically loaded. Uncomment the following and adjust the filename if
+# necessary.
+#
+# filename=".gems"
+# if [[ -s "$filename" ]] ; then
+#   rvm gemset import "$filename" | grep -v already | grep -v listed | grep -v complete | sed '/^$/d'
+# fi
+

data/Gemfile.lock

@@ -0,0 +1,28 @@
+PATH
+  remote: .
+  specs:
+    queueing_proxy (0.0.4)
+      em-jack
+      eventmachine
+      http_parser.rb
+      thin
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    daemons (1.1.4)
+    em-jack (0.1.3)
+      eventmachine
+    eventmachine (0.12.10)
+    http_parser.rb (0.5.2)
+    rack (1.3.2)
+    thin (1.2.11)
+      daemons (>= 1.0.9)
+      eventmachine (>= 0.12.6)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  queueing_proxy!

data/Rakefile

@@ -1,2 +1 @@
-require 'ext/gem_rake'
-Bundler::GemHelper.install_tasks
+require 'bundler/gem_tasks'

data/bin/queueing_proxy

@@ -4,5 +4,33 @@ $LOAD_PATH << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'queueing_proxy'
 
-QueueingProxy::CLI.start
+EM.run{
+  # many-to-many routing topology. Whoa!
+  QueueingProxy {
+    logger Logger.new($stdout)
 
+    from '0.0.0.0', 11000
+    from '0.0.0.0', 10007
+
+    to '127.0.0.1', 10001, do
+      queue_with '0.0.0.0', 'http-queue'
+      workers 1
+    end
+
+    to '127.0.0.1', 10009 do
+      queue_with '0.0.0.0', 'message-gateway-queue'
+      workers 1
+    end
+  }.run
+
+  QueueingProxy {
+    logger Logger.new($stdout)
+
+    from '0.0.0.0', 10002
+
+    to('127.0.0.1', 10001) do
+      workers 1
+      queue_with('0.0.0.0', 'http-queue2')
+    end
+  }.run
+}

data/bin/upstream

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'thin'
+
+# curl "http://localhost:10001/?wait=5&status=500"
+# ---
+# wait - How many seconds should I wait before returning the status?
+# status - HTTP return code
+
+Thin::Server.start('127.0.0.1', 10001, Proc.new{|env|
+  req = Rack::Request.new(env)
+  puts "GET /?#{req.query_string}"
+  EventMachine::Timer.new(req.params['wait'].to_i) {
+    response = [(req.params['status'] || 200).to_i, {}, '']
+    env['async.callback'].call response
+  }
+  # Tell thin to chill, we're gonna run this stack async-style
+  [-1, {}, []]
+})

data/lib/queueing_proxy.rb

@@ -3,18 +3,15 @@ require 'eventmachine'
 require 'em-jack'
 require 'json'
 
-require 'queueing_proxy/cli'
-require 'queueing_proxy/dsl'
-require 'queueing_proxy/queuer'
 require 'queueing_proxy/version'
-require 'queueing_proxy/dispatcher'
+require 'queueing_proxy/dsl'
+require 'queueing_proxy/frontend'
+require 'queueing_proxy/worker'
 
 module QueueingProxy
-
   def self.from(host, port)
     DSL.new.from(host, port)
   end
-
 end
 
 def QueueingProxy(&blk)

data/lib/queueing_proxy/dsl.rb

@@ -2,32 +2,47 @@ require 'logger'
 
 module QueueingProxy
   class DSL
+    class Backend
+      attr_reader :tube, :host
+
+      def initialize(host='localhost', port=80, workers=4, &block)
+        @host, @port, @workers, @beanstalk_host, @tube = host, port, workers, 'localhost', 'default'
+        instance_exec(&block) if block_given?
+        self
+      end
+
+      def queue_with(beanstalk, tube)
+        @beanstalk, @tube = beanstalk, tube
+        self
+      end
+
+      def workers(workers=nil)
+        if workers
+          @workers = workers
+          self
+        else
+          (1..@workers).map { Worker.new(@logger, @host, @port, @beanstalk_host, @tube) }
+        end
+      end
+
+      def logger(logger)
+        @logger = logger
+        self
+      end
+    end
+
     def initialize(&block)
-      @dispatcher_count = 1
-      @logger = Logger.new(STDOUT)
+      @logger = Logger.new($stdout)
       instance_eval(&block) if block
     end
     
-    def to(host, port)
-      @to_host = host
-      @to_port = port
-      self
-    end
-
-    def from(host, port)
-      @from_host = host
-      @from_port = port
-      self
-    end
-
-    def times(count)
-      @dispatcher_count = count
+    def to(host, port=80, &block)
+      backends << Backend.new(host, port, &block)
       self
     end
 
-    def queue_with(beanstalk_host, tube)
-      @beanstalk_host = beanstalk_host
-      @tube = tube
+    def from(host, port=80, &block)
+      frontends << [host, port]
       self
     end
     
@@ -35,14 +50,29 @@ module QueueingProxy
       @logger = logger
       self
     end
-    
+
     def run
       unless EM.reactor_running?
         EM.run{ run }
       else
-        Queuer.new(@logger, @from_host, @from_port, @beanstalk_host, @tube).run
-        @dispatcher_count.times { Dispatcher.new(@logger, @to_host, @to_port, @beanstalk_host, @tube).run }
+        frontends.each do |host, port|
+          Frontend.new(@logger, host, port, backends).run
+        end
+        # Setup multiple backends
+        backends.each {|b|
+          b.logger @logger
+          b.workers.each(&:run) 
+        }
       end
     end
+
+  private
+    def frontends
+      @frontends ||= []
+    end
+
+    def backends
+      @backends ||= []
+    end
   end
 end

data/lib/queueing_proxy/frontend.rb

@@ -0,0 +1,71 @@
+module QueueingProxy
+  class Frontend
+    attr_reader :logger
+
+    def initialize(logger, host, port, backends)
+      @logger, @host, @port, @backends = logger, host, port, backends
+      logger.info "Starting queuer on #{host}:#{port} using beanstalk at #{backends.map{|b| "#{b.tube}@#{b.host}"}.join(', ')}"
+    end
+
+    def beanstalks
+      @beanstalks ||= @backends.map {|b|
+        connection = EMJack::Connection.new(:host => b.host, :tube => b.tube)
+        connection.errback { logger.error "Couldn't connect to beanstalk" }
+        connection
+      }
+    end
+
+    def run
+      app = proc do |env|
+        logger.info "Frontend frontend #{env['HTTP_VERSION']} #{env['PATH_INFO']} #{env['REQUEST_METHOD']}"
+        [204, {}, []]
+      end
+      backend = FakeBackend.new
+
+      EM.start_server(@host, @port, Queuer) do |conn|
+        conn.beanstalks = beanstalks
+        conn.app = app
+        conn.backend = backend
+        conn.logger = logger
+      end
+    end
+
+    # We need this to make Thin happy
+    class FakeBackend
+      def connection_finished(conn)
+      end
+
+      def ssl?
+      end
+    end
+
+    class Queuer < Thin::Connection
+      attr_accessor :beanstalks, :logger
+
+      def post_init
+        @data = ''
+        super
+      end
+
+      def receive_data(data)
+        @data << data
+        super(data)
+      end
+
+      def unbind
+        queue_data
+        super
+      end
+
+      def queue_data
+        if @data != ''
+          beanstalks.each {|b|
+            b.put(@data) {|id|
+              logger.info "Frontend enqueued job #{id} to #{b}"
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/queueing_proxy/version.rb

@@ -1,3 +1,3 @@
 module QueueingProxy
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

data/lib/queueing_proxy/worker.rb

@@ -0,0 +1,126 @@
+require "http/parser"
+
+module QueueingProxy
+  class Worker
+    # 4294967295 is from the Beanstalkd protocol as the least important 
+    # possible job priority. 0 is the highest pri.
+    module Priority
+      Lowest = 4294967295
+      Highest = 0
+    end
+
+    attr_reader :logger
+
+    def initialize(logger, to_host, to_port, beanstalk_host, tube, retries=3)
+      @logger, @to_host, @to_port, @beanstalk_host, @tube, @retries = logger, to_host, to_port, beanstalk_host, tube, retries
+    end
+
+    # Setup a beanstalk connection
+    def beanstalk
+      @beanstalk ||= EMJack::Connection.new(:host => @beanstalk_host, :tube => @tube)
+    end
+
+    # Run the beanstalk consumer that pops the job off the queue and passes it 
+    # to a connection object that makes an upstream connection
+    def run
+      logger.debug "Worker #{object_id} reporting again for duty"
+      beanstalk.reserve {|job|
+        logger.info "Worker #{object_id} reserved #{job}"
+        upstream = Upstream.new(job.body, @to_host, @to_port, 15, logger).request
+        upstream.errback{
+          logger.info "Worker #{object_id} upstream connection timed-out with #{job}."
+          # If there's an upstream problem, try this a fe more times, then bury it
+          job.stats{|stats|
+            if stats['reserves'] < @retries
+              logger.info "Worker #{object_id} delayed for 5 seconds."
+              job.release(:delay => 5){ run }
+            else
+              logger.info "Worker #{object_id} max #{@retries} retries. Burying."
+              job.bury(Priority::Lowest) { run }
+            end
+          }
+        }
+        upstream.callback {
+          case status = Integer(upstream.response.status_code)
+          when 200..299
+            logger.info "Worker #{object_id}. Deleting #{job.jobid}."
+            job.delete { run}
+          else
+            logger.info "Worker #{object_id}. Burying #{job} for inspection."
+            job.bury(Priority::Lowest) { run }
+          end
+        }
+      }.errback{|status|
+        case status
+        when :disconnected
+          logger.error "Worker #{object_id} reservation error. Rescheduling."
+        else
+          logger.error "Worker #{object_id} unhandled error #{status}."
+        end
+        run # Keep on chuggin partner!
+      }
+    end
+
+    # Defferable upstream connection
+    class Upstream
+      include EventMachine::Deferrable
+
+      # The connection handler for EM.
+      module Client
+        attr_accessor :upstream
+
+        # Succesful connection!
+        def connection_completed
+          # Setup our callback for when the upstream response hits us
+          upstream.response.on_headers_complete = Proc.new {
+            upstream.succeed
+            close_connection
+            :stop
+          }
+
+          # Send the HTTP request upstream
+          send_data upstream.payload
+        end
+
+        # Read the response of the upstream proxy
+        def receive_data(data)
+          begin
+            upstream.response << data
+          rescue HTTP::Parser::Error
+            upstream.fail
+            close_connection
+          end
+        end
+
+        # If something bad happens to the connection, bind gets called
+        def unbind
+          upstream.fail
+        end
+      end
+
+      attr_accessor :payload, :host, :port, :timeout, :logger
+      attr_reader :response
+
+      def initialize(payload, host, port, timeout, logger)
+        @response = Http::Parser.new
+        @payload, @host, @port, @timeout, @logger = payload, host, port, timeout, logger
+      end
+
+      # Connect to the upstream server and send the HTTP payload
+      def request
+        begin
+          EventMachine.connect(host, port, Client) {|c|
+            c.upstream = self
+            c.comm_inactivity_timeout = timeout
+            c.pending_connect_timeout = timeout
+          }
+          # TODO - Is this async/deferrable?
+        rescue => e
+          logger.error e
+          fail # If something explodes here, fail the defferable
+        end
+        self
+      end
+    end
+  end
+end

data/queueing_proxy.gemspec

@@ -24,8 +24,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'eventmachine'
   s.add_runtime_dependency 'em-jack'
   s.add_runtime_dependency 'thin'
-  s.add_runtime_dependency 'json'
-  s.add_runtime_dependency 'thor', '>= 0.13.8'
+  s.add_runtime_dependency 'http_parser.rb'
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
@@ -36,5 +35,4 @@ Gem::Specification.new do |s|
     end
   else
   end
-end
-
+end

metadata

@@ -1,148 +1,103 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: queueing_proxy
-version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: false
-  segments: 
-  - 0
-  - 0
-  - 4
-  version: 0.0.4
+version: !ruby/object:Gem::Version
+  version: 0.0.5
+  prerelease: 
 platform: ruby
-authors: 
+authors:
 - Joshua Hull
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-07-21 00:00:00 -04:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2010-07-21 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: eventmachine
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70269188984240 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: em-jack
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *70269188984240
+- !ruby/object:Gem::Dependency
+  name: em-jack
+  requirement: &70269188983560 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: thin
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *70269188983560
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: &70269188982800 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: json
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *70269188982800
+- !ruby/object:Gem::Dependency
+  name: http_parser.rb
+  requirement: &70269188981260 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: thor
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 59
-        segments: 
-        - 0
-        - 13
-        - 8
-        version: 0.13.8
-  type: :runtime
-  version_requirements: *id005
+  version_requirements: *70269188981260
 description: Queueing proxy backed by EM/Beanstalk for a very weird purpose.
 email: joshbuddy@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
+- .gitignore
+- .rvmrc
 - Gemfile
+- Gemfile.lock
 - Rakefile
 - bin/queueing_proxy
+- bin/upstream
 - ext/gem_rake.rb
 - lib/queueing_proxy.rb
-- lib/queueing_proxy/cli.rb
-- lib/queueing_proxy/dispatcher.rb
 - lib/queueing_proxy/dsl.rb
-- lib/queueing_proxy/queuer.rb
+- lib/queueing_proxy/frontend.rb
 - lib/queueing_proxy/version.rb
+- lib/queueing_proxy/worker.rb
 - queueing_proxy.gemspec
-has_rdoc: true
 homepage: http://github.com/joshbuddy/queueing_proxy
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --charset=UTF-8
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: queueing_proxy
-rubygems_version: 1.3.7
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: Queueing proxy backed by EM/Beanstalk for a very weird purpose.
 test_files: []
-

data/lib/queueing_proxy/cli.rb

@@ -1,17 +0,0 @@
-require 'thor'
-
-module QueueingProxy
-  class CLI < Thor
-    
-    def initialize
-      super
-    end
-    
-    default_task :start
-    
-    desc "start", "Start a proxy"
-    def start(config = '/etc/queueing_proxy.yml')
-      
-    end
-  end
-end

data/lib/queueing_proxy/dispatcher.rb

@@ -1,60 +0,0 @@
-module QueueingProxy
-  class Dispatcher
-    attr_reader :logger
-
-    def initialize(logger, to_host, to_port, beanstalk_host, tube)
-      @logger, @to_host, @to_port, @beanstalk_host, @tube = logger, to_host, to_port, beanstalk_host, tube
-      @beanstalk = EMJack::Connection.new(:host => @beanstalk_host, :tube => @tube)
-      @beanstalk.watch(@tube)
-      logger.info "Starting dispatcher on #{to_host}:#{to_port} using beanstalk at #{tube}@#{beanstalk_host}"
-    end
-    
-    def run
-      @beanstalk.reserve do |job|
-        logger.info "Dispatching #{job.jobid}"
-        parsed_job = JSON.parse(job.body)
-        begin
-          EventMachine.connect(@to_host, @to_port, DispatchClient) { |c|
-            c.payload = parsed_job['data']
-            c.dispatcher = self
-            c.logger = logger
-            c.job = job
-            c.dispatcher = self
-          }
-        rescue EventMachine::ConnectionError
-          job.release(:delay => 5)
-          logger.info("Problem connecting")
-          EM.add_timer(5){ run }
-        end
-      end
-    end
-    
-    class DispatchClient < EventMachine::Connection
-      attr_accessor :payload, :dispatcher, :logger, :dispatcher, :job
-      
-      def connection_completed
-        send_data(payload)
-      end
-
-      def receive_data(data)
-        status = Integer(data[/^HTTP\/(1\.1|1\.0) (\d+)/, 2])
-        close_connection
-        case status
-        when 200
-          logger.info "Done dispatching #{job.jobid}"
-          job.delete
-        when 500..599
-          logger.info "Error #{status}"
-          job.release(:delay => 5)
-        else
-          logger.info "Done dispatching #{job.jobid} -- #{status}"
-          job.delete
-        end
-      end
-      
-      def unbind
-        dispatcher.run
-      end
-    end
-  end
-end

data/lib/queueing_proxy/queuer.rb

@@ -1,57 +0,0 @@
-module QueueingProxy
-  class Queuer
-    attr_reader :logger
-
-    def initialize(logger, host, port, beanstalk_host, tube)
-      @logger, @host, @port, @beanstalk_host, @tube = logger, host, port, beanstalk_host, tube
-      logger.info "Starting queuer on #{host}:#{port} using beanstalk at #{tube}@#{beanstalk_host}"
-    end
-
-    def run
-      beanstalk = EMJack::Connection.new(:host => @beanstalk_host, :tube => @tube)
-      app = proc do |env|
-        logger.info "Queueing #{env['HTTP_VERSION']} #{env['PATH_INFO']} #{env['REQUEST_METHOD']}"
-        [200, {}, []]
-      end
-      backend = FakeBackend.new
-      EM.start_server(@host, @port, QueuerConnection) do |conn|
-        conn.beanstalk = beanstalk
-        conn.app = app
-        conn.backend = backend
-        conn.logger = logger
-      end
-    end
-
-    class FakeBackend
-      def connection_finished(conn)
-      end
-    end
-    
-    class QueuerConnection < Thin::Connection
-      attr_accessor :beanstalk, :logger
-
-      def post_init
-        @data = ''
-        super
-      end
-
-      def receive_data(data)
-        @data << data
-        super(data)
-      end
-
-      def unbind
-        queue_data
-        super
-      end
-
-      def queue_data
-        if @data != ''
-          beanstalk.put({:data => @data, :time => Time.new.to_i}.to_json) { |id|
-            logger.info "Job queued #{id}"
-          }
-        end
-      end
-    end
-  end
-end