evdispatch 0.2.6 → 0.3.0

data/ext/revdispatch/revdispatch.cc

@@ -1,15 +1,5 @@
-#include <ruby.h>
-#include "ev_dispatch.h"
-#include "ev_http.h"
-
-/* ruby 1.9 compat */
-#ifndef RSTRING_PTR
-#define RSTRING_PTR(str) RSTRING(str)->ptr
-#endif
-
-#ifndef RSTRING_LEN
-#define RSTRING_LEN(str) RSTRING(str)->len
-#endif
+#include "util.h"
+#include "rhttp.h"
 
 
 /*
@@ -47,6 +37,9 @@
  */
 static VALUE rb_Evdispatch;
 static VALUE rb_Loop;
+static VALUE rb_Response;
+
+static void Response_free( rResponse *res );
 
 /**
  * call-seq: 
@@ -77,6 +70,8 @@ VALUE Loop_request_http( VALUE self, VALUE url )
   EVD::Dispatch *d;
   Data_Get_Struct( self, EVD::Dispatch, d );
 
+  Check_Type( url, T_STRING );
+
   EVD::request_t id = d->request( new EVD::HttpRequest( *d, RSTRING_PTR(url) ) );
 
   return rb_int_new(id);
@@ -98,6 +93,16 @@ VALUE Loop_request_http( VALUE self, VALUE url )
     req->set_opt(name, RSTRING_PTR(obj));\
   }\
 }
+// set options
+#define SET_REQ_OPTS \
+  SET_LONG_VAL("port"); \
+  SET_STR_VAL("autoreferer"); \
+  SET_LONG_VAL("followlocation"); \
+  SET_LONG_VAL("maxredirs"); \
+  SET_STR_VAL("referer"); \
+  SET_STR_VAL("useragent"); \
+  SET_STR_VAL("cookie"); \
+  SET_STR_VAL("post");
 
 /**
  * call-seq:
@@ -107,20 +112,21 @@ VALUE Loop_request_http( VALUE self, VALUE url )
  * 
  * options ( from libcurl ):
  *
- *   :port: port to connect to host with
- *   :autoreferer: Pass a non-zero parameter to enable this. When enabled, libcurl will automatically set the Referer: field in requests where it follows a Location: redirect.
- *   :followlocation: A non-zero parameter tells the library to follow any Location: header that the server sends as part of an HTTP header.
+ *   :port: (Fixnum) port to connect to host with
+ *   :autoreferer: (Fixnum) Pass a non-zero parameter to enable this. When enabled, libcurl will automatically set the Referer: field in requests where it follows a Location: redirect.
+ *   :followlocation: (Fixnum) A non-zero parameter tells the library to follow any Location: header that the server sends as part of an HTTP header.
  *                    This means that the library will re-send the same request on the new location and follow new Location: headers all the way until no more such headers are returned. 'maxredirs' can be used to limit the number of redirects libcurl will follow.
- *   :maxredirs: Pass a long. The set number will be the redirection limit. If that many redirections have been followed, the next redirect will cause an error (CURLE_TOO_MANY_REDIRECTS). This option only makes sense if the CURLOPT_FOLLOWLOCATION is used at the same time. Added in 7.15.1: Setting the limit to 0 will make libcurl refuse any redirect. Set it to -1 for an infinite number of redirects (which is the default)
- *   :referer: Pass a pointer to a zero terminated string as parameter. It will be used to set the Referer: header in the http request sent to the remote server. This can be used to fool servers or scripts. You can also set any custom header with CURLOPT_HTTPHEADER.
- *   :useragent: Pass a pointer to a zero terminated string as parameter. It will be used to set the User-Agent: header in the http request sent to the remote server. This can be used to fool servers or scripts. You can also set any custom header with CURLOPT_HTTPHEADER.
- *   :cookie: Pass a pointer to a zero terminated string as parameter. It will be used to set a cookie in the http request. The format of the string should be NAME=CONTENTS, where NAME is the cookie name and CONTENTS is what the cookie should contain.
+ *   :maxredirs: (Fixnum) Pass a long. The set number will be the redirection limit. If that many redirections have been followed, the next redirect will cause an error (CURLE_TOO_MANY_REDIRECTS). This option only makes sense if the CURLOPT_FOLLOWLOCATION is used at the same time. Added in 7.15.1: Setting the limit to 0 will make libcurl refuse any redirect. Set it to -1 for an infinite number of redirects (which is the default)
+ *   :referer: (String) Pass a pointer to a zero terminated string as parameter. It will be used to set the Referer: header in the http request sent to the remote server. This can be used to fool servers or scripts. You can also set any custom header with CURLOPT_HTTPHEADER.
+ *   :useragent: (String) Pass a pointer to a zero terminated string as parameter. It will be used to set the User-Agent: header in the http request sent to the remote server. This can be used to fool servers or scripts. You can also set any custom header with CURLOPT_HTTPHEADER.
+ *   :cookie: (String) Pass a pointer to a zero terminated string as parameter. It will be used to set a cookie in the http request. The format of the string should be NAME=CONTENTS, where NAME is the cookie name and CONTENTS is what the cookie should contain.
  *            If you need to set multiple cookies, you need to set them all using a single option and thus you need to concatenate them all in one single string. Set multiple cookies in one string like this: "name1=content1; name2=content2;" etc.
  *            Note that this option sets the cookie header explictly in the outgoing request(s). If multiple requests are done due to authentication, followed redirections or similar, they will all get this cookie passed on.
  *            Using this option multiple times will only make the latest string override the previous ones.
  *
- *   :post: Pass a string and sets CURLOPT_POST, CURLOPT_POSTFIELDSIZE, and CURLOPT_COPYPOSTFIELDS
+ *   :post: (String) Pass a string and sets CURLOPT_POST, CURLOPT_POSTFIELDSIZE, and CURLOPT_COPYPOSTFIELDS
  *
+ *   :stream: (Evdispatch::Response) Send the response to the IO object.  Ignore the response id returned it'd invalid
  */
 static
 VALUE Loop_request( int argc, VALUE *argv, VALUE self )
@@ -137,19 +143,34 @@ VALUE Loop_request( int argc, VALUE *argv, VALUE self )
     options = rb_hash_new();
   }
 
-  EVD::HttpRequest *req = new EVD::HttpRequest( *d, RSTRING_PTR(url) );
+  Check_Type(url, T_STRING );
+  Check_Type(options, T_HASH );
 
-  // check for some specific options
-  SET_LONG_VAL("port");
-  SET_STR_VAL("autoreferer");
-  SET_LONG_VAL("followlocation");
-  SET_LONG_VAL("maxredirs");
-  SET_STR_VAL("referer");
-  SET_STR_VAL("useragent");
-  SET_STR_VAL("cookie");
-  SET_STR_VAL("post");
+  VALUE response = rb_hash_aref( options, ID2SYM(rb_intern("stream")) );
+  if( !NIL_P(response) && TYPE(response) == T_TRUE ) {
+ 
+    rResponse *res = new rResponse;
+    VALUE response_object = Data_Wrap_Struct( rb_Response, NULL, Response_free, res );
 
-  return rb_int_new( d->request(req) );
+    if( !res->init() ) {
+      rb_raise(rb_eIOError, "Failed to create new pipe");
+    }
+
+    EVD::HttpRequest *req = new EVD::HttpRequest( *d, RSTRING_PTR(url), res->m_pipe[1] );
+
+    SET_REQ_OPTS 
+
+    res->m_response = req->m_response;
+
+    d->request(req);
+    return response_object;
+  }
+  else {
+    EVD::HttpRequest *req = new EVD::HttpRequest( *d, RSTRING_PTR(url) );
+    SET_REQ_OPTS 
+
+    return rb_int_new( d->request(req) );
+  }
 }
 
 static 
@@ -160,6 +181,10 @@ VALUE Loop_wait_for_response( VALUE self, VALUE id, VALUE timeout_seconds, VALUE
   EVD::request_t rid = FIX2LONG(id);
 
   Data_Get_Struct( self, EVD::Dispatch, d );
+  
+  Check_Type( id, T_FIXNUM );
+  Check_Type( timeout_seconds, T_FIXNUM );
+  Check_Type( timeout_mseconds, T_FIXNUM );
 
   long seconds = FIX2LONG(timeout_seconds);
   long nanoseconds = FIX2LONG(timeout_mseconds)*1000*1000;
@@ -175,6 +200,7 @@ VALUE Loop_response_for( VALUE self, VALUE id )
 {
   EVD::Dispatch *d;
   Data_Get_Struct( self, EVD::Dispatch, d );
+  Check_Type( id, T_FIXNUM );
 
   EVD::HttpResponse *res = NULL;
   EVD::request_t rid = FIX2LONG(id);
@@ -209,6 +235,8 @@ VALUE Loop_blocking_response_for( int argc, VALUE *argv, VALUE self )
     options = rb_hash_new();
     rb_hash_aset( options, ID2SYM(rb_intern("timeout")), rb_float_new( 2.0 ) );
   }
+  Check_Type( req_id, T_FIXNUM );
+  Check_Type( options, T_HASH );
 
   EVD::request_t id = FIX2LONG(req_id);
   VALUE timeout_value = rb_hash_aref( options, ID2SYM(rb_intern("timeout")) );
@@ -269,7 +297,6 @@ VALUE Loop_stop( VALUE self )
 static
 void Loop_free( EVD::Dispatch *d )
 {
-  fprintf(stderr,"Freeing loop\n");
   delete d;
 }
 
@@ -284,6 +311,65 @@ VALUE Loop_alloc(VALUE klass)
   return object;
 }
 
+static
+void Response_free( rResponse *res )
+{
+  delete res;
+}
+
+static
+VALUE Response_alloc(VALUE klass)
+{
+  VALUE object;
+
+  rResponse *res = new rResponse;
+  object = Data_Wrap_Struct( klass, NULL, Response_free, res );
+
+  return object;
+}
+
+  /*
+  This code might be interesting later if we decide to wire in direct writing to a Ruby socket or file
+  const char *klass_name = rb_class2name(CLASS_OF(stream)); 
+  if( !strcmp("IO", klass_name) || !strcmp("Socket", klass_name) || !strcmp("File", klass_name) ) {
+    RFile *fstream = RFILE(stream);
+    struct OpenFile *fptr = fstream->fptr;
+    FILE *file = fptr->f;
+    int fd = fileno(file);
+    write(fd,"hi",2);
+  }*/
+
+static
+VALUE Response_read( VALUE self )
+{
+  rResponse *res;
+  Data_Get_Struct( self, rResponse, res );
+
+  int status = res->read_body_partial();
+
+  if( status == -1 ) {
+    rb_raise(rb_eIOError, "Failed to read pipe");
+  }
+
+  return rb_int_new(status);
+}
+
+static
+VALUE Response_body( VALUE self )
+{
+  rResponse *res;
+  Data_Get_Struct( self, rResponse, res );
+  return rb_str_new(res->m_buffer.c_str(), res->m_buffer.length());
+}
+
+static
+VALUE Response_headers( VALUE self )
+{
+  rResponse *res;
+  Data_Get_Struct( self, rResponse, res );
+  return rb_str_new(res->m_response->m_header.c_str(), res->m_response->m_header.length() );
+}
+
 extern "C" void Init_revdispatch()
 {
   rb_Evdispatch = rb_define_module( "Evdispatch" );
@@ -292,12 +378,22 @@ extern "C" void Init_revdispatch()
   // setup the Loop object
   rb_define_alloc_func( rb_Loop, Loop_alloc );
   
-  rb_define_method( rb_Loop, "start", (VALUE (*)(...))Loop_start, 0 );
-  rb_define_method( rb_Loop, "request_http", (VALUE (*)(...))Loop_request_http, 1 );
-  rb_define_method( rb_Loop, "request", (VALUE (*)(...))Loop_request, -1 );
-  rb_define_method( rb_Loop, "flush", (VALUE (*)(...))Loop_flush, 0 );
-  rb_define_method( rb_Loop, "blocking_response_for", (VALUE (*)(...))Loop_blocking_response_for, -1 );
-  rb_define_method( rb_Loop, "response_for", (VALUE (*)(...))Loop_response_for, 1 );
-  rb_define_method( rb_Loop, "wait_for_response", (VALUE (*)(...))Loop_wait_for_response, 3 );
-  rb_define_method( rb_Loop, "stop", (VALUE (*)(...))Loop_stop, 0 );
+  rb_define_method( rb_Loop, "start",                 RB_METHOD(Loop_start), 0 );
+  rb_define_method( rb_Loop, "request_http",          RB_METHOD(Loop_request_http), 1 );
+  rb_define_method( rb_Loop, "request",               RB_METHOD(Loop_request), -1 );
+  rb_define_method( rb_Loop, "flush",                 RB_METHOD(Loop_flush), 0 );
+  rb_define_method( rb_Loop, "blocking_response_for", RB_METHOD(Loop_blocking_response_for), -1 );
+  rb_define_method( rb_Loop, "response_for",          RB_METHOD(Loop_response_for), 1 );
+  rb_define_method( rb_Loop, "wait_for_response",     RB_METHOD(Loop_wait_for_response), 3 );
+  rb_define_method( rb_Loop, "stop",                  RB_METHOD(Loop_stop), 0 );
+
+  // define the response object
+  rb_Response = rb_define_class_under( rb_Evdispatch, "Response", rb_cObject );
+  
+  // setup the Response object
+  rb_define_alloc_func( rb_Response, Response_alloc );
+
+  rb_define_method( rb_Response, "read", RB_METHOD(Response_read), 0 );
+  rb_define_method( rb_Response, "body", RB_METHOD(Response_body), 0 );
+  rb_define_method( rb_Response, "headers", RB_METHOD(Response_headers), 0 );
 }

data/ext/revdispatch/rhttp.cc

@@ -0,0 +1,60 @@
+#include "util.h"
+
+rResponse::rResponse()
+  : has_headers(false), m_response(NULL), m_use_pipe(false)
+{
+}
+
+rResponse::~rResponse()
+{
+  if( m_use_pipe) close(m_pipe[0]);
+  if( m_response ) delete m_response;
+}
+
+bool rResponse::init()
+{
+  if( pipe(m_pipe) ) { perror("pipe"); return false; }
+  m_use_pipe = true;
+  return true;
+}
+
+static const int READ_SIZE = 1024;
+static char READ_BUFFER[READ_SIZE];
+
+int rResponse::read_body_partial()
+{
+  int fd = m_pipe[0];
+  fd_set rd;
+  struct timeval tv;
+
+  tv.tv_sec = 1;
+  tv.tv_usec = 0; //1000000; // timeout within a 10th of a second
+  memset(READ_BUFFER, '\0', READ_SIZE);
+
+  FD_ZERO(&rd);
+  FD_SET(fd, &rd);
+
+  int retval = select( (fd+1), &rd, NULL, NULL, &tv );
+
+  if( retval == 0 ) { printf("select timedout\n"); return 1; } // timeout
+  if( retval == -1 && errno == EINTR ) { return 1; } // not ready yet
+  if( retval < 0 ){ perror("select"); return -1; } // something went wrong
+
+  if( FD_ISSET(fd, &rd) ) {
+    retval = read(fd,READ_BUFFER,READ_SIZE);
+    if( retval >= 0 ) {
+      // write to the string
+      m_buffer.append(READ_BUFFER, retval);
+      if( retval == 0 ) {
+        FD_CLR(fd,&rd); // remove the file descriptor
+      }
+      return retval;
+    }
+    else {
+      perror("read");
+      return -1;
+    }
+  }
+
+  return 0;
+}

data/ext/revdispatch/rhttp.h

@@ -0,0 +1,30 @@
+#ifndef RUBY_EV_HTTP_H
+#define RUBY_EV_HTTP_H
+
+#include <string>
+#include "ev_dispatch.h"
+#include "ev_http.h"
+
+struct rResponse {
+  rResponse();
+  ~rResponse();
+
+  volatile bool has_headers;
+
+  // return values
+  // 1: read again
+  // 0: finished
+  // -1: error
+  int read_body_partial();
+
+  bool init();
+
+  EVD::HttpResponse *m_response;
+  int m_pipe[2];  // might use this if the IO object does not have a file descriptor
+  bool m_use_pipe;
+
+  std::string m_buffer;
+  
+};
+
+#endif

data/ext/revdispatch/stest.rb

@@ -1,17 +1,25 @@
 require 'test/unit'
-
 $:.unshift File.join(File.dirname(__FILE__),'..','..','lib')
 require 'evdispatch'
 
-$d = Evdispatch::Loop.new
-# start the event loop thread
-$d.start
-
 class TestRequests < Test::Unit::TestCase
 
+  def setup
+    @loop = Evdispatch::Loop.new
+    @loop.start
+  end
+
+  def teardown
+    @loop.stop
+  end
+
+  def loop
+    @loop
+  end
+
   def test_response
-    id = $d.request_http("http://127.0.0.1:4044/bytes/10")
-    response = $d.response( id )
+    id = loop.request_http("http://127.0.0.1:4044/bytes/10")
+    response = loop.response( id )
     assert_equal("http://127.0.0.1:4044/bytes/10", response[:name])
     assert_equal("CCCCCCCCCC", response[:body])
     assert_match(/Content-Type: text\/json/, response[:header])
@@ -22,29 +30,48 @@ class TestRequests < Test::Unit::TestCase
   end
 
   def test_options_request
-    id = $d.request("http://127.0.0.1:4044/redir/1", :followlocation => 1, :referer => 'pizza')
-    response = $d.response( id )
+    id = loop.request("http://127.0.0.1:4044/redir/1", :followlocation => 1, :referer => 'pizza')
+    response = loop.response( id )
+    assert_not_nil( response )
     assert_match(/ 302 Moved Temporarily/,response[:header])
     assert_match(/ 200 OK/,response[:header])
   end
 
-  def test_delayed_with_flush
-    id = $d.request("http://127.0.0.1:4044/delay/0.1")
-    tid = $d.request("http://127.0.0.1:4044/delay/3.0")
-    res = $d.blocking_response_for( id, :timeout => 0.5 )
-    res2 = $d.blocking_response_for( tid, :timeout => 0.5 )#, 1.0, 1 )
-    if !res or !res2
-      $d.flush # flush because we aborted before we finished, still the issue of the request is still running, but any previous responses that were delayed will be flushed
+  def test_redir_with_stream
+    stream = loop.request("http://127.0.0.1:4044/redir/2", :followlocation => 1, :stream => true )
+    res = stream.read
+    until res == 0
+      res = stream.read
     end
-    assert_not_nil res
-    assert_nil res2
+    assert_equal( 0, res )
+    assert_equal( "C"*10, stream.body )
+    headers = stream.headers
+    assert_match( "Content-Length: 10", headers )
+    assert_match( "Content-Type: text/json", headers )
+    assert_match( "HTTP/1.1 200 OK", headers )
   end
 
   def test_post
-    id = $d.request("http://127.0.0.1:4044/test_post_length", :post => "hello there world")
-    res = $d.blocking_response_for( id )
+    id = loop.request("http://127.0.0.1:4044/test_post_length", :post => "hello there world")
+    res = loop.blocking_response_for( id )
     assert_not_nil res
   end
+
+  def test_streaming
+    count = 10000
+    stream = loop.request("http://127.0.0.1:4044/bytes/#{count}", :stream => true )
+    res = stream.read
+    until res == 0
+      res = stream.read
+    end
+    assert_equal( 0, res )
+    assert_equal( "C"*count, stream.body )
+    headers = stream.headers
+    assert_match( "Content-Length: #{count}", headers )
+    assert_match( "Content-Type: text/json", headers )
+    assert_match( "HTTP/1.1 200 OK", headers )
+  end
+
 end
 
 # not bothering to cleanup

data/ext/revdispatch/util.h

@@ -0,0 +1,26 @@
+#ifndef EVD_UTIL_H
+#define EVD_UTIL_H
+
+#include <stdio.h>
+#include <errno.h>
+#include <ruby.h>
+#include <rubyio.h>
+#include "ev_dispatch.h"
+#include "ev_http.h"
+#include "rhttp.h"
+
+
+/* ruby 1.9 compat */
+#ifndef RSTRING_PTR
+  #define RSTRING_PTR(str) RSTRING(str)->ptr
+#endif
+
+#ifndef RSTRING_LEN
+  #define RSTRING_LEN(str) RSTRING(str)->len
+#endif
+
+#ifndef RB_METHOD
+  #define RB_METHOD(f) ((VALUE (*)(...))(f))
+#endif
+
+#endif

data/lib/evdispatch/version.rb

@@ -1,8 +1,8 @@
 module Evdispatch #:nodoc:
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 2
-    TINY  = 6
+    MINOR = 3
+    TINY  = 0
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/test/test_evdispatch.rb

@@ -2,7 +2,36 @@ require File.dirname(__FILE__) + '/test_helper.rb'
 
 class TestEvdispatch < Test::Unit::TestCase
 
-  def test_object_test
+  def test_streaming
+    d = Evdispatch::Loop.new
+    d.start
+
+    count = 10000
+
+    streams = []
+    # create 100 streams
+    100.times do|i|
+      stream = d.request("http://127.0.0.1:4044/bytes/#{count}", :stream => true )
+      streams << stream
+    end
+
+    # TODO: we should expose the pipe so someone can select on ready responses
+    for stream in streams do
+      res = stream.read
+      res = stream.read until res == 0
+
+      assert_equal( 0, res )
+      assert_equal( "C"*count, stream.body )
+      headers = stream.headers
+      assert_match( "Content-Length: 10000", headers )
+      assert_match( "Content-Type: text/json", headers )
+      assert_match( "HTTP/1.1 200 OK", headers )
+    end
+
+    d.stop
+  end
+
+  def test_object_allocation
     d = Evdispatch::Loop.new
 
     # start the event loop thread

data/website/index.html

@@ -18,7 +18,7 @@
     <h1>evdispatch</h1>
     <div id="version" class="clickable" onclick='document.location = "http://rubyforge.org/projects/evdispatch"; return false'>
       <p>Get Version</p>
-      <a href="http://rubyforge.org/projects/evdispatch" class="numbers">0.2.6</a>
+      <a href="http://rubyforge.org/projects/evdispatch" class="numbers">0.3.0</a>
     </div>
     <h4 style="float:right;padding-right:10px;">&#x2192; &#8216;evdispatch&#8217;</h4>
 
@@ -31,7 +31,13 @@
 	<h2>Installing</h2>
 
 
-	<p><pre class='syntax'><span class="ident">sudo</span> <span class="ident">gem</span> <span class="ident">install</span> <span class="ident">evdispatch</span></pre></p>
+	<p>Download: <a href="http://curl.haxx.se/download/curl-7.18.1.tar.gz">curl-7.18.1.tar.gz</a>
+<pre class='syntax'>
+  tar -zxf curl-7.18.1.tar.gz
+  cd curl-7.18.1/
+  ./configure &amp;&amp; make &amp;&amp; sudo make install
+  sudo gem install evdispatch
+</pre></p>
 
 
 	<h2>The basics</h2>

data/website/index.txt

@@ -9,7 +9,13 @@ A library for making HTTP requests in parallel.
 
 h2. Installing
 
-<pre syntax="ruby">sudo gem install evdispatch</pre>
+Download: <a href="http://curl.haxx.se/download/curl-7.18.1.tar.gz">curl-7.18.1.tar.gz</a>
+<pre syntax="bash">
+  tar -zxf curl-7.18.1.tar.gz
+  cd curl-7.18.1/
+  ./configure && make && sudo make install
+  sudo gem install evdispatch
+</pre>
 
 h2. The basics
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: evdispatch
 version: !ruby/object:Gem::Version 
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors: 
 - Todd A. Fisher
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-04-21 00:00:00 -04:00
+date: 2008-04-22 00:00:00 -04:00
 default_executable: 
 dependencies: []
 
@@ -61,6 +61,9 @@ files:
 - ext/revdispatch/stest.rb
 - ext/revdispatch/extconf.rb
 - ext/revdispatch/revdispatch.cc
+- ext/revdispatch/rhttp.cc
+- ext/revdispatch/rhttp.h
+- ext/revdispatch/util.h
 - ext/revdispatch/libdispatch-0.1/
 - ext/revdispatch/libdispatch-0.1/src/
 - ext/revdispatch/libdispatch-0.1/test/