evdispatch 0.2.2 → 0.2.4

data/ext/revdispatch/revdispatch.cc

@@ -24,14 +24,11 @@
  * # start the event loop thread
  *
  * # send a dispatch http request and store a handle to the request
- * google_id = d.request_http("http://www.google.com/")
+ * google_id = d.request("http://www.google.com/")
  *
  * # do some processing and later on check for the response
- * while d.wait_for_response( google_id, 1, 0 )
- *   res = d.response_for( google_id )
- *   break if res
- * end
- * res = d.response_for( google_id ) unless res
+ *
+ * res = d.response( google_id )
  *
  * puts res[:body]
  *
@@ -51,6 +48,12 @@
 static VALUE rb_Evdispatch;
 static VALUE rb_Loop;
 
+/**
+ * call-seq: 
+ *    loop.start -> loop
+ * 
+ * Starts up the background event loop
+ */
 static
 VALUE Loop_start( VALUE self )
 {
@@ -62,6 +65,12 @@ VALUE Loop_start( VALUE self )
   return self;
 }
 
+/**
+ * call-seq: 
+ *    loop.request_http(url) -> request_id
+ * 
+ * Notifies the background thread of a new request. Call this method to start a basic HTTP GET request.
+ */
 static
 VALUE Loop_request_http( VALUE self, VALUE url )
 {
@@ -77,7 +86,7 @@ VALUE Loop_request_http( VALUE self, VALUE url )
 {\
   VALUE obj = rb_hash_aref( options, ID2SYM(rb_intern(name)) ); \
   if( !NIL_P(obj) ) {\
-    snprintf( VALUE_BUFFER, VALUE_BUFFER_SIZE, "%d", FIX2LONG(obj) );\
+    snprintf( VALUE_BUFFER, VALUE_BUFFER_SIZE, "%ld", FIX2LONG(obj) );\
     req->set_opt(name, VALUE_BUFFER);\
   }\
 }
@@ -90,6 +99,29 @@ VALUE Loop_request_http( VALUE self, VALUE url )
   }\
 }
 
+/**
+ * call-seq:
+ *    loop.request( url, options ) -> request_id
+ * 
+ * Notify the background of a new request. Can send a few options to influence the request:
+ * 
+ * options ( from libcurl ):
+ *
+ *   :port: port to connect to host with
+ *   :autoreferer: Pass a non-zero parameter to enable this. When enabled, libcurl will automatically set the Referer: field in requests where it follows a Location: redirect.
+ *   :followlocation: A non-zero parameter tells the library to follow any Location: header that the server sends as part of an HTTP header.
+ *                    This means that the library will re-send the same request on the new location and follow new Location: headers all the way until no more such headers are returned. 'maxredirs' can be used to limit the number of redirects libcurl will follow.
+ *   :maxredirs: Pass a long. The set number will be the redirection limit. If that many redirections have been followed, the next redirect will cause an error (CURLE_TOO_MANY_REDIRECTS). This option only makes sense if the CURLOPT_FOLLOWLOCATION is used at the same time. Added in 7.15.1: Setting the limit to 0 will make libcurl refuse any redirect. Set it to -1 for an infinite number of redirects (which is the default)
+ *   :referer: Pass a pointer to a zero terminated string as parameter. It will be used to set the Referer: header in the http request sent to the remote server. This can be used to fool servers or scripts. You can also set any custom header with CURLOPT_HTTPHEADER.
+ *   :useragent: Pass a pointer to a zero terminated string as parameter. It will be used to set the User-Agent: header in the http request sent to the remote server. This can be used to fool servers or scripts. You can also set any custom header with CURLOPT_HTTPHEADER.
+ *   :cookie: Pass a pointer to a zero terminated string as parameter. It will be used to set a cookie in the http request. The format of the string should be NAME=CONTENTS, where NAME is the cookie name and CONTENTS is what the cookie should contain.
+ *            If you need to set multiple cookies, you need to set them all using a single option and thus you need to concatenate them all in one single string. Set multiple cookies in one string like this: "name1=content1; name2=content2;" etc.
+ *            Note that this option sets the cookie header explictly in the outgoing request(s). If multiple requests are done due to authentication, followed redirections or similar, they will all get this cookie passed on.
+ *            Using this option multiple times will only make the latest string override the previous ones.
+ *
+ *   :post: Pass a string and sets CURLOPT_POST, CURLOPT_POSTFIELDSIZE, and CURLOPT_COPYPOSTFIELDS
+ *
+ */
 static
 VALUE Loop_request( int argc, VALUE *argv, VALUE self )
 {
@@ -115,6 +147,7 @@ VALUE Loop_request( int argc, VALUE *argv, VALUE self )
   SET_STR_VAL("referer");
   SET_STR_VAL("useragent");
   SET_STR_VAL("cookie");
+  SET_STR_VAL("post");
 
   return rb_int_new( d->request(req) );
 }
@@ -128,7 +161,11 @@ VALUE Loop_wait_for_response( VALUE self, VALUE id, VALUE timeout_seconds, VALUE
 
   Data_Get_Struct( self, EVD::Dispatch, d );
 
-  rstate = d->wait_for_response_by_id( rid, EVD::Timer(FIX2LONG(timeout_seconds), (FIX2LONG(timeout_mseconds)*1000*1000)) );
+  long seconds = FIX2LONG(timeout_seconds);
+  long nanoseconds = FIX2LONG(timeout_mseconds)*1000*1000;
+  //printf( "wait seconds %ld, nanoseconds %ld\n", seconds, nanoseconds );
+
+  rstate = d->wait_for_response_by_id( rid, EVD::Timer(seconds, nanoseconds) );
 
   return rb_int_new(rstate);
 }
@@ -160,6 +197,54 @@ VALUE Loop_response_for( VALUE self, VALUE id )
   return Qnil;
 }
 
+static
+VALUE Loop_blocking_response_for( int argc, VALUE *argv, VALUE self )
+{
+  VALUE req_id, options;
+  EVD::Dispatch *dispatcher;
+  Data_Get_Struct( self, EVD::Dispatch, dispatcher );
+
+  // required 1 argument the 'url' and  1 optional the hash of options
+  if( rb_scan_args( argc, argv, "11", &req_id, &options ) == 1 ) {
+    options = rb_hash_new();
+    rb_hash_aset( options, ID2SYM(rb_intern("timeout")), rb_float_new( 2.0 ) );
+  }
+
+  EVD::request_t id = FIX2LONG(req_id);
+  VALUE timeout_value = rb_hash_aref( options, ID2SYM(rb_intern("timeout")) );
+  double timeout = RFLOAT( timeout_value )->value;
+
+  struct timeval start;
+  EVD::Timer::current_time(&start);
+  long int secs = (int)timeout;
+  long int msecs = (secs == 0) ? 500 : 0; // XXX: timer hack for small timeout
+
+  //printf( "waiting on id: %d, %d:%d\n", id, secs, msecs );
+
+  while( dispatcher->wait_for_response_by_id( id, EVD::Timer(secs,msecs*1000*1000) ) ) {
+    if( EVD::Timer::elapsed_time( &start ) > timeout ){
+      printf("exceeded max elasped time...\n");
+      break;
+    }
+  }
+  EVD::HttpResponse *res = (EVD::HttpResponse *)dispatcher->response_for( id );
+  if( res ) {
+
+    VALUE result = rb_hash_new();
+
+    rb_hash_aset( result, ID2SYM(rb_intern("name")), rb_str_new( res->name.c_str(), res->name.length() ) );
+    rb_hash_aset( result, ID2SYM(rb_intern("body")), rb_str_new( res->body.c_str(), res->body.length() ) );
+    rb_hash_aset( result, ID2SYM(rb_intern("id")), rb_int_new( res->id ) );
+    rb_hash_aset( result, ID2SYM(rb_intern("response_time")), rb_float_new( res->response_time ) );
+    rb_hash_aset( result, ID2SYM(rb_intern("header")), rb_str_new( res->m_header.c_str(), res->m_header.length() ) );
+
+    delete res;
+
+    return result;
+  }
+  return Qnil;
+}
+
 static 
 VALUE Loop_flush( VALUE self )
 {
@@ -177,12 +262,14 @@ VALUE Loop_stop( VALUE self )
   Data_Get_Struct( self, EVD::Dispatch, d );
 
   d->stop();
+
   return Qnil;
 }
 
 static
 void Loop_free( EVD::Dispatch *d )
 {
+  fprintf(stderr,"Freeing loop\n");
   delete d;
 }
 
@@ -209,6 +296,7 @@ extern "C" void Init_revdispatch()
   rb_define_method( rb_Loop, "request_http", (VALUE (*)(...))Loop_request_http, 1 );
   rb_define_method( rb_Loop, "request", (VALUE (*)(...))Loop_request, -1 );
   rb_define_method( rb_Loop, "flush", (VALUE (*)(...))Loop_flush, 0 );
+  rb_define_method( rb_Loop, "blocking_response_for", (VALUE (*)(...))Loop_blocking_response_for, -1 );
   rb_define_method( rb_Loop, "response_for", (VALUE (*)(...))Loop_response_for, 1 );
   rb_define_method( rb_Loop, "wait_for_response", (VALUE (*)(...))Loop_wait_for_response, 3 );
   rb_define_method( rb_Loop, "stop", (VALUE (*)(...))Loop_stop, 0 );

data/ext/revdispatch/server.rb

@@ -5,15 +5,18 @@ Ebb.log = File.open('/dev/null','w')
 
 TEST_PORT = 4044
 
+$req_count = 0
 
 class HelperApp
   def call(env)
     commands = env['PATH_INFO'].split('/')
     extras = {}
+    $req_count += 1
+    puts "#{$req_count} requests"
 
     if commands.include?('delay')
-      n = commands.last.to_i
-      raise "delay called with n <= 0" if n < 0
+      n = commands.last.to_f
+      raise "delay called with n <= 0" if n < 0.0
       sleep n
       body = "delayed #{n} seconds"
       status = 200

data/ext/revdispatch/stest.rb

@@ -1,7 +1,7 @@
-$:.unshift File.join(File.dirname(__FILE__),'..','..','lib')
+require 'test/unit'
 
+$:.unshift File.join(File.dirname(__FILE__),'..','..','lib')
 require 'evdispatch'
-require 'test/unit'
 
 $d = Evdispatch::Loop.new
 # start the event loop thread
@@ -29,15 +29,21 @@ class TestRequests < Test::Unit::TestCase
   end
 
   def test_delayed_with_flush
-    id = $d.request("http://127.0.0.1:4044/delay/0")
-    tid = $d.request("http://127.0.0.1:4044/delay/3")
-    res = $d.response( id, 1.0, 1 )
-    res2 = $d.response( tid, 1.0, 1 )
+    id = $d.request("http://127.0.0.1:4044/delay/0.1")
+    tid = $d.request("http://127.0.0.1:4044/delay/3.0")
+    res = $d.blocking_response_for( id, :timeout => 0.5 )
+    res2 = $d.blocking_response_for( tid, :timeout => 0.5 )#, 1.0, 1 )
     if !res or !res2
       $d.flush # flush because we aborted before we finished, still the issue of the request is still running, but any previous responses that were delayed will be flushed
     end
-    puts res.inspect
-    puts res2.inspect
+    assert_not_nil res
+    assert_nil res2
+  end
+
+  def test_post
+    id = $d.request("http://127.0.0.1:4044/test_post_length", :post => "hello there world")
+    res = $d.blocking_response_for( id )
+    assert_not_nil res
   end
 end
 

data/ext/revdispatch/test.rb

@@ -16,14 +16,18 @@ require 'ostruct'
 
 # NOTE: we define this part in ruby to make sure other ruby threads can execute while we wait
 class Evdispatch::Loop
-  def response(id)
+
+  def response(id, timeout=2.0)
+    timer = Time.now
     while wait_for_response( id, 1, 0 )
       res = response_for( id )
       break if res
+      break if( (Time.now - timer) > timeout )
     end
     res = response_for( id ) unless res
     res
   end
+
 end
 
 def request_bytes_from( d, base, amount, range )
@@ -58,11 +62,11 @@ def run_trial
 
   # do some processing and later on check for the response
   response = d.response( google_id )
-  puts "Requested: #{response[0]} in #{Time.now - timer} seconds"
+  puts "Requested: #{response[:name]} in #{Time.now - timer} seconds"
 
   ebbbase = "http://127.0.0.1:4044/"
   timer = Time.now
-  ids = request_bytes_from( d, ebbbase, 400, 10000 )
+  ids = request_bytes_from( d, ebbbase, 300, 10000 )
   #ids += request_delay_from( d, ebbbase, 100, 1 )
 
   # wait for each response

data/lib/evdispatch/loop.rb

@@ -4,21 +4,13 @@ require 'revdispatch'
 # ruby threads can execute while we waiting for a response
 module Evdispatch
   class Loop
-    def response(id, timeout = 1.0, max_attempts = 100)
-      attempts = 0
-      ms = timeout * 1000 # convert to miliseconds
-      ms_i = ms.to_i
-      seconds = ms_i / 1000
-      mseconds = (ms - (seconds*1000)).to_i
-      #puts "Waiting #{seconds} seconds and #{mseconds} miliseconds"
-
-      while (wait_for_response( id, seconds, mseconds ) and attempts < max_attempts )
+    def response(id, timeout = 1.0)
+      timer = Time.now
+      while wait_for_response( id, 1, 0 )
         res = response_for( id )
         break if res
-        attempts += 1
-        #puts "Attempt: #{attempts}"
+        break if( (Time.now - timer) > timeout )
       end
-      return nil if !res and attempts == max_attempts 
       res = response_for( id ) unless res
       res
     end

data/lib/evdispatch/version.rb

@@ -2,7 +2,7 @@ module Evdispatch #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 2
+    TINY  = 4
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/test/test_evdispatch.rb

@@ -2,50 +2,40 @@ require File.dirname(__FILE__) + '/test_helper.rb'
 
 class TestEvdispatch < Test::Unit::TestCase
 
-  def is_linux?
-    !(`uname`.grep(/linux/i)).empty?
-  end
-
   def test_object_test
-    # start up the test server
-    if is_linux?
-      pid = fork do
-        require File.dirname(__FILE__) + '/../ext/revdispatch/server'
-        start_test_server
-      end
-      sleep 1
-    else
-      STDERR.puts "make sure '#{File.dirname(__FILE__) + '/../ext/revdispatch/server'} start' is running..."
-    end
-
     d = Evdispatch::Loop.new
 
     # start the event loop thread
     d.start
 
-    begin
+    # increase this to verify
+    1.times do
 
-      ObjectSpace.garbage_collect
+      begin
 
-      10.times do
-        begin
-          start_count = ObjectSpace.each_object { }
-          duration =  run_trial(d, 100)
-          new_count = ObjectSpace.each_object { }
-          puts "10 trials: #{duration} seconds, new objects #{new_count - start_count}, #{new_count} - #{start_count}"
-        end
+        ObjectSpace.garbage_collect
+
+        10.times do
+          begin
+            start_count = ObjectSpace.each_object { }
+            duration =  run_trial(d, 100)
+            new_count = ObjectSpace.each_object { }
+            puts "10 trials: #{duration} seconds, new objects #{new_count - start_count}, #{new_count} - #{start_count}"
+          end
+
+          begin
+            start_count = ObjectSpace.each_object { }
+            duration = run_trial(d, 200)
+            new_count = ObjectSpace.each_object { }
+            puts "100 trials: #{duration} seconds, new objects #{new_count - start_count}, #{new_count} - #{start_count}"
+          end
 
-        begin
-          start_count = ObjectSpace.each_object { }
-          duration = run_trial(d, 200)
-          new_count = ObjectSpace.each_object { }
-          puts "100 trials: #{duration} seconds, new objects #{new_count - start_count}, #{new_count} - #{start_count}"
         end
 
+      rescue => e
+        puts e.message, e.backtrace
       end
 
-    rescue => e
-      puts e.message, e.backtrace
     end
 
     count = ObjectSpace.each_object { }
@@ -57,8 +47,6 @@ class TestEvdispatch < Test::Unit::TestCase
  
     # sometime later you can stop the event loop
     d.stop
-  ensure
-    Process.kill('HUP', pid) if is_linux?
   end
 
   def request_bytes_from( d, base, amount, range )

data/website/index.html

@@ -18,7 +18,7 @@
     <h1>evdispatch</h1>
     <div id="version" class="clickable" onclick='document.location = "http://rubyforge.org/projects/evdispatch"; return false'>
       <p>Get Version</p>
-      <a href="http://rubyforge.org/projects/evdispatch" class="numbers">0.2.1</a>
+      <a href="http://rubyforge.org/projects/evdispatch" class="numbers">0.2.2</a>
     </div>
     <h4 style="float:right;padding-right:10px;">&#x2192; &#8216;evdispatch&#8217;</h4>
 

data/website/index.txt

@@ -58,6 +58,9 @@ chances are high that the news_id will have returned it's response
 by the time it gets finished with the blogs. This is because the background thread does not block while ruby waits for the first response.
 One thing to keep in mind is the background thread will block if it has to resolve DNS names.
 
+In my testing I've successfully run 3 million requests.  Doing 10000 iterations of 300 concurrent requests, on a Linux FC7 duel P4 machine.
+Memory utilization stays flat at around 14 to 19 megs with about 2,000 - 12,000 objects.
+
 h2. Demonstration of usage
 
 <pre syntax="ruby">
@@ -97,10 +100,15 @@ The trunk repository is <a href="http://evdispatch.rubyforge.org/svn/trunk/">htt
 h2. F.A.Q.
 
 <ul>
-<li><h5> I get an error when installing the gem: <strong>"error: 'curl_socket_t' has not been
-  declared"</strong></h5>
-  <p> You need to have at least version 7.16 of <a href="http://curl.haxx.se/download.html">libcurl</a>.  I recommend at least version: 7.18.1.</p>
-</li>
+  <li><h5>I get an error when installing the gem: <strong>"error: 'curl_socket_t' has not been
+    declared"</strong></h5>
+    <p> You need to have at least version 7.17.1 of <a href="http://curl.haxx.se/download.html">libcurl</a>.  I recommend at least version: 7.18.1.</p>
+  </li>
+  <li><h5>Mac OSX (Darwin) Crashing on HTTP POST request</h5>
+    <p>There are some issues with crashes on darwin builds when making an HTTP
+    POST.  This doesn't happen on Linux and I'm actively looking into the
+    issue on my apple.</p>
+  </li>
 </ul>
 
 h2. License

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: evdispatch
 version: !ruby/object:Gem::Version 
-  version: 0.2.2
+  version: 0.2.4
 platform: ruby
 authors: 
 - Todd A. Fisher
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-04-14 00:00:00 -04:00
+date: 2008-04-16 00:00:00 -04:00
 default_executable: 
 dependencies: []
 
@@ -69,7 +69,6 @@ files:
 - ext/revdispatch/libdispatch-0.1/Makefile.am
 - ext/revdispatch/libdispatch-0.1/Makefile.in
 - ext/revdispatch/libdispatch-0.1/README
-- ext/revdispatch/libdispatch-0.1/TODO
 - ext/revdispatch/libdispatch-0.1/aclocal.m4
 - ext/revdispatch/libdispatch-0.1/autogen.sh
 - ext/revdispatch/libdispatch-0.1/config.guess
@@ -125,6 +124,8 @@ files:
 - ext/revdispatch/libdispatch-0.1/test/next_test.cc
 - ext/revdispatch/libdispatch-0.1/test/pipe_test.cc
 - ext/revdispatch/libdispatch-0.1/test/opt_test.cc
+- ext/revdispatch/libdispatch-0.1/test/post_test.cc
+- ext/revdispatch/libdispatch-0.1/test/stress_test.cc
 has_rdoc: true
 homepage: http://evdispatch.rubyforge.org
 post_install_message: 

data/ext/revdispatch/libdispatch-0.1/TODO

@@ -1,5 +0,0 @@
-write gnu autoconf scripts
-  include test for curl_socket_t to detect valid curl library
-write ruby interface
-write test suite
-write python interface