yup 0.0.3 → 0.0.4

data/Gemfile

@@ -2,11 +2,15 @@ source "http://rubygems.org"
 
 gem "eventmachine"
 gem "em-http-request"
+gem "http_request.rb"
 gem "http_parser.rb"
+gem "tuple"
+gem "dk-bdb"
+gem "yajl-ruby"
 
 group :development do
-  gem "shoulda", ">= 0"
   gem "yard", "~> 0.6.0"
+  gem "minitest", ">= 0"
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.5.2"
   gem "rcov", ">= 0"

data/Gemfile.lock

@@ -2,6 +2,7 @@ GEM
   remote: http://rubygems.org/
   specs:
     addressable (2.2.6)
+    dk-bdb (0.2.4.2)
     em-http-request (0.3.0)
       addressable (>= 2.0.0)
       escape_utils
@@ -10,13 +11,16 @@ GEM
     eventmachine (0.12.10)
     git (1.2.5)
     http_parser.rb (0.5.1)
+    http_request.rb (1.1.11)
     jeweler (1.5.2)
       bundler (~> 1.0.0)
       git (>= 1.2.5)
       rake
+    minitest (2.5.0)
     rake (0.9.2)
     rcov (0.9.10)
-    shoulda (2.11.3)
+    tuple (0.1.2)
+    yajl-ruby (0.8.3)
     yard (0.6.8)
 
 PLATFORMS
@@ -24,10 +28,14 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.0.0)
+  dk-bdb
   em-http-request
   eventmachine
   http_parser.rb
+  http_request.rb
   jeweler (~> 1.5.2)
+  minitest
   rcov
-  shoulda
+  tuple
+  yajl-ruby
   yard (~> 0.6.0)

data/README.rdoc

@@ -4,10 +4,16 @@ This is the small daemon for transparent asynchronous delegating HTTP requests w
 
 When a http request is arrived the yup daemon (yupd) answers to the client configured answer (by default 200 OK). Then the yupd forwards the http request to the specified host and retries if a timeout error was happend.
 
+== Non-persistent queue
+By default, nonpersistent queue is used. A limit (the option --watermark) at which new requests will be dropped.
+
+== Persistent queue
+If you want use persistent queue you need to specify the option --persistent with a path to a database.
+
 == One of use cases
 
 For example we can have a rails app which send exceptions to an errbit by the gem hoptoad_notifier. We know the errbit can be not available by network issues or some else reasons, but we do not want lose exceptions. To resolve this problem we can start yupd on the same host with the rails app:
-    yupd --listen localhost:8081 --status-code 201 errbit.host.somewhere
+    yupd --listen localhost:8081 --status-code 201 --persistent /var/db/yupd-errbit errbit.host.somewhere
 
 Reconfiguration of hoptoad_notifier is very ease:
     HoptoadNotifier.configure do |config|
@@ -20,9 +26,9 @@ Now problem of availability of errbit is assigned to the yupd.
 
 == Roadmap to 0.1
 
+* More better documentation
 * Daemonize
 * Preforking
-* Persistent HTTP requests queue
 * A configurable map of different delegating rules
 * Tests...
 

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.0.4

data/bin/yupd

@@ -18,6 +18,7 @@ Options:
   --watermark <number>       Maximum of concurrent requests (default 100)
   --loglevel <level>         Logging severity (default fatal).
                              Available log levels: fatal, error, warn, info, debug.
+  --persistent <path>        Use persistent queue.
 
 Examples:
   yupd --listen 0.0.0.0:8081 --status-code 201 errbit.host.somewhere
@@ -39,6 +40,7 @@ opts = GetoptLong.new(*[['--help',         '-h', GetoptLong::NO_ARGUMENT],
                         ['--resend-delay', '-d', GetoptLong::REQUIRED_ARGUMENT],
                         ['--watermark',          GetoptLong::REQUIRED_ARGUMENT],
                         ['--loglevel',           GetoptLong::REQUIRED_ARGUMENT],
+                        ['--persistent',         GetoptLong::REQUIRED_ARGUMENT],
                         ])
 config = {:loglevel => LEVELS["fatal"]}
 
@@ -63,6 +65,8 @@ opts.each do |opt, arg|
       usage
       exit 1
     end
+  when '--persistent'
+    config[:persistent] = arg || "/tmp/queue"
   end
 end
 
@@ -77,4 +81,10 @@ config[:forward_to] = ARGV.shift
 Yup.logger.level = config[:loglevel]
 Yup.watermark    = config[:watermark] if config.has_key?(:watermark)
 Yup.resend_delay = config[:resend_delay] if config.has_key?(:resend_delay)
-Yup.run(config)
+
+if config.has_key?(:persistent)
+  # Yup::State.repair_if_need(config[:persistent])
+  Yup.run_with_state(config)
+else
+  Yup.run(config)
+end

data/lib/yup.rb

@@ -1,9 +1,12 @@
 require 'rubygems'
 require 'eventmachine'
 require 'logger'
+require 'yajl'
+require 'tmpdir'
 
 require 'yup/request_forwarder'
 require 'yup/request_handler'
+require 'yup/state'
 
 module Yup
   @@resend_delay = 5.0
@@ -24,10 +27,42 @@ module Yup
     status_code = config[:status_code] || 200
     forward_to  = config[:forward_to]
 
-    EventMachine::run do
-      EventMachine::start_server(host, port, RequestHandler,
-                                 forward_to, status_code)
+    EventMachine.run do
+      EventMachine.start_server(host, port, RequestHandler, forward_to, status_code, nil)
       logger.info { "listening on #{host}:#{port}" }
     end
   end
+
+  def self.run_with_state(config)
+    host = config[:listen_host] || 'localhost'
+    port = config[:listen_port] || 8080
+    status_code = config[:status_code] || 200
+    forward_to  = config[:forward_to]
+    dbpath = config[:persistent]
+    feedback_channel = File.join(Dir.tmpdir, "yupd-#{$$}-feedback")
+    state            = Yup::State.new(dbpath, forward_to, feedback_channel)
+
+    pid = Process.fork do
+      State::RequestForwarder.new(state, forward_to).run_loop
+    end
+
+    if pid
+      db_closer = proc do
+        Yup.logger.info { "Terminating consumer #{$$}" }
+        Process.kill("KILL", pid)
+        state.close
+        exit 0
+      end
+      Signal.trap("TERM", &db_closer)
+      Signal.trap("INT", &db_closer)
+    end
+
+    EventMachine.run do
+      EventMachine.start_server(host, port, RequestHandler, forward_to, status_code, state)
+      logger.info { "Listening on #{host}:#{port}" }
+
+      EventMachine.start_unix_domain_server(feedback_channel, State::FeedbackHandler, state)
+      logger.info { "Feedback through #{feedback_channel}" }
+    end
+  end
 end

data/lib/yup/request_forwarder.rb

@@ -1,22 +1,25 @@
 require 'em-http-request'
+require 'http_request'
 
 module Yup
   class RequestForwarder
-    def initialize(parser, body, forward_to)
-      @parser     = parser
-      @body       = body
-      @forward_to = forward_to
+    def initialize(http_method, request_url, headers, body, forward_to)
+      @http_method = http_method
+      @request_url = request_url
+      @headers     = headers
+      @body        = body
+      @forward_to  = forward_to
 
       @logger = Yup.logger
     end
 
     def run
-      http_method = @parser.http_method.downcase.to_sym
-      http_url    = "http://#{@forward_to}#{@parser.request_url}"
+      http_method = @http_method.to_sym
+      http_url    = "http://#{@forward_to}#{@request_url}"
       http = EventMachine::HttpRequest.
         new(http_url).
         send(http_method,
-             :head => @parser.headers.merge('Host' => @forward_to),
+             :head => @headers.merge('Host' => @forward_to),
              :body => @body)
 
       http.callback do
@@ -35,9 +38,88 @@ module Yup
       http.errback do
         @logger.info '--- ERROR'
         @logger.debug { http.inspect }
+        @logger.debug { http.response_header.inspect }
+        @logger.debug { http.response.inspect }
 
         EventMachine.add_timer(Yup.resend_delay) { self.run }
       end
     end
   end
+
+  class State
+    class RequestForwarder
+      def initialize(state, forward_to)
+        @state      = state
+        @forward_to = forward_to
+
+        @logger = Yup.logger
+        @yajl   = Yajl::Parser.new(:symbolize_keys => true)
+        @yajl.on_parse_complete = method(:make_request)
+      end
+
+      def run_loop
+        loop do
+          data = @state.bpop
+          begin
+            @yajl << data
+          rescue Yajl::ParseError
+            @logger.error { "Error while parsing \"#{data}\"" }
+          end
+        end
+      end
+
+      def make_request(req)
+        begin
+          http_method, request_url, headers, body = req
+          headers = Hash[headers.to_a.flatten.map(&:to_s)]
+
+          http_method = http_method.to_sym
+          http_url    = "http://#{@forward_to}#{request_url}"
+          http = HttpRequest.
+            send(http_method,
+                 :url => http_url,
+                 :headers => headers.merge('Host' => @forward_to),
+                 :parameters => body)
+
+          if http.code_2xx?
+            @logger.info '--- SUCCESS'
+          else
+            @logger.info '--- FAIL'
+            @logger.debug { http.inspect }
+            @logger.debug { http.response_header.inspect }
+            @logger.debug { http.response.inspect }
+          end
+        rescue Exception => e
+          @logger.info '--- ERROR'
+          @logger.debug { e }
+
+          @state.to_feedback(Yajl::Encoder.encode([http_method.downcase, request_url, headers, body]))
+
+          sleep Yup.resend_delay
+        end
+      end
+    end
+
+    class FeedbackHandler < EM::Connection
+      def initialize(state)
+        @state      = state
+
+        @logger = Yup.logger
+        @yajl   = Yajl::Parser.new(:symbolize_keys => true)
+        @yajl.on_parse_complete = method(:on_message)
+      end
+
+      def receive_data(data)
+        begin
+          @yajl << data
+        rescue Yajl::ParseError
+          @logger.error { "Error while parsing \"#{data}\"" }
+        end
+      end
+
+      def on_message(req)
+        @state.push(Yajl::Encoder.encode(req))
+      end
+    end
+  end
 end

data/lib/yup/request_handler.rb

@@ -5,9 +5,10 @@ module Yup
   class RequestHandler < EM::Connection
     attr_reader :queue
 
-    def initialize(forward_to, status_code)
+    def initialize(forward_to, status_code, state)
       @forward_to  = forward_to
       @status_code = status_code
+      @state       = state
 
       @logger = Yup.logger
       @chunks = []
@@ -31,24 +32,37 @@ module Yup
 
     def on_message_complete
       @logger.info  "-- got request"
-      @logger.info  { "HTTP version : "     + @parser.http_version.join('.') }
-      @logger.info  { "HTTP method : "      + @parser.http_method }
+      @logger.info  { "HTTP version     : " + @parser.http_version.join('.') }
+      @logger.info  { "HTTP method      : " + @parser.http_method }
       @logger.info  { "HTTP request_url : " + @parser.request_url }
-      @logger.debug { "HTTP headers : "     + @parser.headers.inspect }
+      @logger.debug { "HTTP headers     : " + @parser.headers.inspect }
+      @logger.debug { "HTTP body        : " + @body }
 
+      send_answer
+      shedule_request
+    end
+
+    private
+    def send_answer
       resp = WEBrick::HTTPResponse.new(:HTTPVersion => '1.1')
       resp.status = @status_code
       resp['Server'] = 'yupd'
       send_data resp.to_s
+    end
 
-      unless Yup.watermark.zero?
-        Yup.watermark -= 1
+    def shedule_request
+      if @state
+        @state.push(Yajl::Encoder.encode([@parser.http_method.downcase, @parser.request_url, @parser.headers, @body, @forward_to]))
+      else
+        unless Yup.watermark.zero?
+          Yup.watermark -= 1
 
-        EventMachine.next_tick do
-          RequestForwarder.new(@parser, @body, @forward_to).run
+          EventMachine.next_tick do
+            RequestForwarder.new(@parser.http_method.downcase, @parser.request_url, @parser.headers, @body, @forward_to).run
+          end
+        else
+          @logger.error "-- watermark is reached, drop"
         end
-      else
-        @logger.error "-- watermark is reached, drop"
       end
     end
   end

data/lib/yup/state.rb

@@ -0,0 +1,54 @@
+require 'bdb'
+require 'bdb/database'
+
+module Yup
+  class State
+    RE_LEN = 1000
+
+    attr_reader :queue
+
+    def self.repair_if_need(path)
+      env = Bdb::Env.new(0)
+      env.open(path, Bdb::DB_CREATE | Bdb::DB_INIT_TXN | Bdb::DB_RECOVER, 0)
+      env.close()
+    end
+
+    def initialize(path, name, feedback_channel)
+      @path = path
+      @name = name
+      @feedback_channel = feedback_channel
+
+      @env   = Bdb::Env.new(0)
+      @env   = @env.open(@path, Bdb::DB_CREATE | Bdb::DB_INIT_MPOOL | Bdb::DB_INIT_CDB, 0)
+      @queue = @env.db
+      @queue.re_len = RE_LEN
+      @queue.open(nil, @name, nil, Bdb::Db::QUEUE, Bdb::DB_CREATE, 0)
+    end
+
+    def push(data)
+      Yup.logger.debug { "Pushing \"#{data}\"" }
+      i = 0
+      until (chunk = data.slice(i, RE_LEN)).nil?
+        @queue.put(nil, "", chunk, Bdb::DB_APPEND)
+        i += @queue.re_len
+      end
+    end
+
+    def bpop
+      data = @queue.get(nil, "", nil, Bdb::DB_CONSUME_WAIT)
+      Yup.logger.debug { "Bpoping \"#{data}\"" }
+      data
+    end
+
+    def to_feedback(data)
+      Yup.logger.debug { "Pushing to the feedback channel \"#{data}\"" }
+      sock = UNIXSocket.new(@feedback_channel)
+      sock.send(data, 0)
+      sock.close
+    end
+
+    def close
+      @queue.close(0)
+    end
+  end
+end

data/test/helper.rb

@@ -7,12 +7,13 @@ rescue Bundler::BundlerError => e
   $stderr.puts "Run `bundle install` to install missing gems"
   exit e.status_code
 end
-require 'test/unit'
-require 'shoulda'
+require 'minitest/unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'yup'
 
-class Test::Unit::TestCase
+class MiniTest::Unit::TestCase
 end
+
+MiniTest::Unit.autorun

data/test/test_yup.rb

@@ -1,7 +1,11 @@
 require 'helper'
 
-class TestYup < Test::Unit::TestCase
-  should "probably rename this file and start testing for real" do
-    flunk "hey buddy, you should probably rename this file and start testing for real"
+class TestYup < MiniTest::Unit::TestCase
+  def test_request_handler
+    flunk
+  end
+
+  def test_request_forwarder
+    flunk
   end
 end

data/yup.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{yup}
-  s.version = "0.0.3"
+  s.version = "0.0.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Denis Sukhonin"]
-  s.date = %q{2011-08-17}
+  s.date = %q{2011-09-07}
   s.default_executable = %q{yupd}
   s.description = %q{Just answers 200 (or specified) to a client and asynchronously forwards HTTP request to a configured host}
   s.email = %q{d.sukhonin@gmail.com}
@@ -30,6 +30,7 @@ Gem::Specification.new do |s|
     "lib/yup.rb",
     "lib/yup/request_forwarder.rb",
     "lib/yup/request_handler.rb",
+    "lib/yup/state.rb",
     "test/helper.rb",
     "test/test_yup.rb",
     "yup.gemspec"
@@ -51,18 +52,26 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<eventmachine>, [">= 0"])
       s.add_runtime_dependency(%q<em-http-request>, [">= 0"])
+      s.add_runtime_dependency(%q<http_request.rb>, [">= 0"])
       s.add_runtime_dependency(%q<http_parser.rb>, [">= 0"])
-      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_runtime_dependency(%q<tuple>, [">= 0"])
+      s.add_runtime_dependency(%q<dk-bdb>, [">= 0"])
+      s.add_runtime_dependency(%q<yajl-ruby>, [">= 0"])
       s.add_development_dependency(%q<yard>, ["~> 0.6.0"])
+      s.add_development_dependency(%q<minitest>, [">= 0"])
       s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_development_dependency(%q<rcov>, [">= 0"])
     else
       s.add_dependency(%q<eventmachine>, [">= 0"])
       s.add_dependency(%q<em-http-request>, [">= 0"])
+      s.add_dependency(%q<http_request.rb>, [">= 0"])
       s.add_dependency(%q<http_parser.rb>, [">= 0"])
-      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<tuple>, [">= 0"])
+      s.add_dependency(%q<dk-bdb>, [">= 0"])
+      s.add_dependency(%q<yajl-ruby>, [">= 0"])
       s.add_dependency(%q<yard>, ["~> 0.6.0"])
+      s.add_dependency(%q<minitest>, [">= 0"])
       s.add_dependency(%q<bundler>, ["~> 1.0.0"])
       s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
       s.add_dependency(%q<rcov>, [">= 0"])
@@ -70,9 +79,13 @@ Gem::Specification.new do |s|
   else
     s.add_dependency(%q<eventmachine>, [">= 0"])
     s.add_dependency(%q<em-http-request>, [">= 0"])
+    s.add_dependency(%q<http_request.rb>, [">= 0"])
     s.add_dependency(%q<http_parser.rb>, [">= 0"])
-    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<tuple>, [">= 0"])
+    s.add_dependency(%q<dk-bdb>, [">= 0"])
+    s.add_dependency(%q<yajl-ruby>, [">= 0"])
     s.add_dependency(%q<yard>, ["~> 0.6.0"])
+    s.add_dependency(%q<minitest>, [">= 0"])
     s.add_dependency(%q<bundler>, ["~> 1.0.0"])
     s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
     s.add_dependency(%q<rcov>, [">= 0"])

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 4
+  version: 0.0.4
 platform: ruby
 authors: 
 - Denis Sukhonin
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-17 00:00:00 +04:00
+date: 2011-09-07 00:00:00 +04:00
 default_executable: yupd
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -44,7 +44,7 @@ dependencies:
   prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: http_parser.rb
+  name: http_request.rb
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -57,7 +57,7 @@ dependencies:
   prerelease: false
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: shoulda
+  name: http_parser.rb
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -66,12 +66,51 @@ dependencies:
         segments: 
         - 0
         version: "0"
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: yard
+  name: tuple
   requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: dk-bdb
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: yajl-ruby
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  prerelease: false
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: yard
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -83,10 +122,23 @@ dependencies:
         version: 0.6.0
   type: :development
   prerelease: false
-  version_requirements: *id005
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: minitest
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id009
 - !ruby/object:Gem::Dependency 
   name: bundler
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id010 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -98,10 +150,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *id006
+  version_requirements: *id010
 - !ruby/object:Gem::Dependency 
   name: jeweler
-  requirement: &id007 !ruby/object:Gem::Requirement 
+  requirement: &id011 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -113,10 +165,10 @@ dependencies:
         version: 1.5.2
   type: :development
   prerelease: false
-  version_requirements: *id007
+  version_requirements: *id011
 - !ruby/object:Gem::Dependency 
   name: rcov
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  requirement: &id012 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -126,7 +178,7 @@ dependencies:
         version: "0"
   type: :development
   prerelease: false
-  version_requirements: *id008
+  version_requirements: *id012
 description: Just answers 200 (or specified) to a client and asynchronously forwards HTTP request to a configured host
 email: d.sukhonin@gmail.com
 executables: 
@@ -148,6 +200,7 @@ files:
 - lib/yup.rb
 - lib/yup/request_forwarder.rb
 - lib/yup/request_handler.rb
+- lib/yup/state.rb
 - test/helper.rb
 - test/test_yup.rb
 - yup.gemspec
@@ -165,7 +218,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -3474122509911024601
+      hash: -4031501533023442341
       segments: 
       - 0
       version: "0"