curb 0.1.4 → 0.3.1

data/ext/curb_easy.h

@@ -24,8 +24,8 @@ typedef struct {
   
   VALUE body_proc;
   VALUE header_proc;  
-  VALUE body_data;      /* These have the result from the last curl_easy_perform */
-  VALUE header_data;    /* unless a block is supplied (when they'll be nil)      */
+  VALUE body_data;      /* Holds the response body from the last call to curl_easy_perform */
+  VALUE header_data;    /* unless a block is supplied (they'll be nil)      */
   VALUE progress_proc;
   VALUE debug_proc;
   VALUE interface;
@@ -33,7 +33,13 @@ typedef struct {
   VALUE proxypwd;
   VALUE headers;        /* ruby array of strings with headers to set */
   VALUE cookiejar;      /* filename */
-  
+  VALUE cert;
+  VALUE encoding;
+
+  VALUE success_proc;
+  VALUE failure_proc;
+  VALUE complete_proc;
+
   /* Other opts */
   unsigned short local_port;       // 0 is no port
   unsigned short local_port_range; // "  "  " "
@@ -65,10 +71,24 @@ typedef struct {
    * and in case it's asked for before the next call.
    */
   VALUE postdata_buffer;  
+
+  /* when added to a multi handle these buffers are needed
+   * when the easy handle isn't supplied the body proc
+   * or a custom http header is passed.
+   */
+  VALUE bodybuf;
+  VALUE headerbuf;
+  struct curl_slist *curl_headers;
+
+  VALUE self; /* pointer to self, used by multi interface */
+
 } ruby_curl_easy;
 
 extern VALUE cCurlEasy;
 
+VALUE ruby_curl_easy_setup(ruby_curl_easy *rbce, VALUE *bodybuf, VALUE *headerbuf, struct curl_slist **headers);
+VALUE ruby_curl_easy_cleanup(VALUE self, ruby_curl_easy *rbce, VALUE bodybuf, VALUE headerbuf, struct curl_slist *headers);
+
 void init_curb_easy();
 
 #endif

data/ext/curb_errors.c

@@ -98,9 +98,19 @@ VALUE eCurlErrTFTPUnknownID;
 VALUE eCurlErrTFTPFileExists;
 VALUE eCurlErrTFTPNoSuchUser;
 
+/* multi errors */
+VALUE mCurlErrCallMultiPerform;
+VALUE mCurlErrBadHandle;
+VALUE mCurlErrBadEasyHandle;
+VALUE mCurlErrOutOfMemory;
+VALUE mCurlErrInternalError;
+VALUE mCurlErrBadSocket;
+VALUE mCurlErrUnknownOption;
+
 /* binding errors */
 VALUE eCurlErrInvalidPostField;
 
+
 /* rb_raise an approriate exception for the supplied CURLcode */
 void raise_curl_easy_error_exception(CURLcode code) {
   VALUE exclz;
@@ -299,48 +309,54 @@ void raise_curl_easy_error_exception(CURLcode code) {
     case CURLE_FTP_SSL_FAILED:          /* 64 - Requested FTP SSL level failed */
       exclz = eCurlErrFTPSSLFailed;
       break;          
+#ifdef HAVE_CURLE_SEND_FAIL_REWIND 
     case CURLE_SEND_FAIL_REWIND:        /* 65 - Sending the data requires a rewind that failed */
       exclz = eCurlErrSendFailedRewind;
-      break;          
+      break;
+#endif
+#ifdef HAVE_CURLE_SSL_ENGINE_INITFAILED
     case CURLE_SSL_ENGINE_INITFAILED:   /* 66 - failed to initialise ENGINE */
       exclz = eCurlErrSSLEngineInitFailed;
       break;          
+#endif
+#ifdef HAVE_CURLE_LOGIN_DENIED
     case CURLE_LOGIN_DENIED:            /* 67 - user, password or similar was not accepted and we failed to login */
       exclz = eCurlErrLoginDenied;
       break;      
+#endif
       
       // recent additions, may not be present in all supported versions
-#ifdef CURLE_TFTP_NOTFOUND
+#ifdef HAVE_CURLE_TFTP_NOTFOUND
     case CURLE_TFTP_NOTFOUND:           /* 68 - file not found on server */
       exclz = eCurlErrTFTPNotFound;
       break;          
 #endif
-#ifdef CURLE_TFTP_PERM      
+#ifdef HAVE_CURLE_TFTP_PERM      
     case CURLE_TFTP_PERM:               /* 69 - permission problem on server */
       exclz = eCurlErrTFTPPermission;
       break;
 #endif
-#ifdef CURLE_TFTP_DISKFULL
+#ifdef HAVE_CURLE_TFTP_DISKFULL
     case CURLE_TFTP_DISKFULL:           /* 70 - out of disk space on server */
       exclz = eCurlErrTFTPDiskFull;
       break;    
 #endif
-#ifdef CURLE_TFTP_ILLEGAL
+#ifdef HAVE_CURLE_TFTP_ILLEGAL
     case CURLE_TFTP_ILLEGAL:            /* 71 - Illegal TFTP operation */
       exclz = eCurlErrTFTPIllegalOperation;
       break;    
 #endif     
-#ifdef CURLE_TFTP_UNKNOWNID
+#ifdef HAVE_CURLE_TFTP_UNKNOWNID
     case CURLE_TFTP_UNKNOWNID:          /* 72 - Unknown transfer ID */
       exclz = eCurlErrTFTPUnknownID;
       break;
 #endif
-#ifdef CURLE_TFTP_EXISTS
+#ifdef HAVE_CURLE_TFTP_EXISTS
     case CURLE_TFTP_EXISTS:             /* 73 - File already exists */
       exclz = eCurlErrTFTPFileExists;
       break;    
 #endif
-#ifdef CURLE_TFTP_NOSUCHUSER
+#ifdef HAVE_CURLE_TFTP_NOSUCHUSER
     case CURLE_TFTP_NOSUCHUSER:         /* 74 - No such user */
       exclz = eCurlErrTFTPNotFound;
       break;
@@ -356,6 +372,43 @@ void raise_curl_easy_error_exception(CURLcode code) {
   
   rb_raise(exclz, exmsg);
 }
+void raise_curl_multi_error_exception(CURLMcode code) {
+  VALUE exclz;
+  const char *exmsg = NULL;
+
+  switch(code) {
+    case CURLM_CALL_MULTI_PERFORM: /* -1 */
+      exclz = mCurlErrCallMultiPerform;
+      break;
+    case CURLM_BAD_HANDLE: /* 1 */
+      exclz = mCurlErrBadHandle;
+      break;
+    case CURLM_BAD_EASY_HANDLE: /* 2 */
+      exclz = mCurlErrBadEasyHandle;
+      break;
+    case CURLM_OUT_OF_MEMORY: /* 3 */
+      exclz = mCurlErrOutOfMemory;
+      break;
+    case CURLM_INTERNAL_ERROR: /* 4 */
+      exclz = mCurlErrInternalError;
+      break;
+    case CURLM_BAD_SOCKET: /* 5 */
+      exclz = mCurlErrBadSocket;
+      break;
+    case CURLM_UNKNOWN_OPTION: /* 6 */
+      exclz = mCurlErrUnknownOption;
+      break;
+    default:
+      exclz = eCurlErrError;
+      exmsg = "Unknown error result from libcurl";
+  }
+  
+  if (!exmsg) {
+    exmsg = curl_multi_strerror(code);
+  }
+
+  rb_raise(exclz, exmsg);
+}
 
 void init_curb_errors() {
   mCurlErr = rb_define_module_under(mCurl, "Err");

data/ext/curb_errors.h

@@ -12,6 +12,7 @@
 /* base errors */
 extern VALUE cCurlErr;
 
+/* easy errors */
 extern VALUE mCurlErr;
 extern VALUE eCurlErrError;
 extern VALUE eCurlErrFTPError;
@@ -97,10 +98,20 @@ extern VALUE eCurlErrTFTPUnknownID;
 extern VALUE eCurlErrTFTPFileExists;
 extern VALUE eCurlErrTFTPNoSuchUser;
 
+/* multi errors */
+extern VALUE mCurlErrCallMultiPerform;
+extern VALUE mCurlErrBadHandle;
+extern VALUE mCurlErrBadEasyHandle;
+extern VALUE mCurlErrOutOfMemory;
+extern VALUE mCurlErrInternalError;
+extern VALUE mCurlErrBadSocket;
+extern VALUE mCurlErrUnknownOption;
+
 /* binding errors */
 extern VALUE eCurlErrInvalidPostField;
 
 void init_curb_errors();
 void raise_curl_easy_error_exception(CURLcode code);
+void raise_curl_multi_error_exception(CURLMcode code);
 
 #endif

data/ext/curb_multi.c

@@ -0,0 +1,353 @@
+/* curb_easy.c - Curl easy mode
+ * Copyright (c)2008 Todd A. Fisher. 
+ * Licensed under the Ruby License. See LICENSE for details.
+ * 
+ * $Id$
+ */
+
+#include "curb_config.h"
+#ifdef HAVE_RUBY19_ST_H
+#include <ruby/st.h>
+#else
+#include <st.h>
+#endif
+#include "curb_easy.h"
+#include "curb_errors.h"
+#include "curb_postfield.h"
+#include "curb_multi.h"
+
+#include <errno.h>
+
+extern VALUE mCurl;
+static VALUE idCall;
+
+#ifdef RDOC_NEVER_DEFINED
+  mCurl = rb_define_module("Curl");
+#endif
+
+VALUE cCurlMulti;
+
+static VALUE ruby_curl_multi_remove(VALUE , VALUE );
+static void rb_curl_multi_remove(ruby_curl_multi *rbcm, VALUE easy);
+static void rb_curl_multi_read_info(VALUE self, CURLM *mptr);
+
+static void rb_curl_multi_mark_all_easy(VALUE key, VALUE rbeasy, ruby_curl_multi *rbcm) {
+  //printf( "mark easy: 0x%X\n", (long)rbeasy );
+  rb_gc_mark(rbeasy);
+}
+
+static void curl_multi_mark(ruby_curl_multi *rbcm) {
+  rb_gc_mark(rbcm->requests);
+  rb_hash_foreach( rbcm->requests, (int (*)())rb_curl_multi_mark_all_easy, (VALUE)rbcm );
+}
+
+static void curl_multi_flush_easy(VALUE key, VALUE easy, ruby_curl_multi *rbcm) {
+  rb_curl_multi_remove(rbcm, easy);
+}
+
+static void curl_multi_free(ruby_curl_multi *rbcm) {
+  //printf("hash entries: %d\n", RHASH(rbcm->requests)->tbl->num_entries );
+  if (rbcm && RHASH_LEN(rbcm->requests) > 0) {
+    rb_hash_foreach( rbcm->requests, (int (*)())curl_multi_flush_easy, (VALUE)rbcm );
+
+    curl_multi_cleanup(rbcm->handle);
+    //rb_hash_clear(rbcm->requests)
+    rbcm->requests = Qnil;
+  }
+}
+
+/*
+ * call-seq:
+ *   Curl::Multi.new                                   => #&lt;Curl::Easy...&gt;
+ * 
+ * Create a new Curl::Multi instance
+ */
+static VALUE ruby_curl_multi_new(VALUE self) {
+  VALUE new_curlm;
+  
+  ruby_curl_multi *rbcm = ALLOC(ruby_curl_multi);
+
+  rbcm->handle = curl_multi_init();
+
+  rbcm->requests = rb_hash_new();
+
+  rbcm->active = 0;
+  rbcm->running = 0;
+  
+  new_curlm = Data_Wrap_Struct(cCurlMulti, curl_multi_mark, curl_multi_free, rbcm);
+
+  return new_curlm;
+}
+
+/*
+ * call-seq:
+ * multi = Curl::Multi.new
+ * multi.max_connects = 800
+ *
+ * Set the max connections in the cache for a multi handle
+ */
+static VALUE ruby_curl_multi_max_connects(VALUE self, VALUE count) {
+  ruby_curl_multi *rbcm;
+
+  Data_Get_Struct(self, ruby_curl_multi, rbcm);
+  curl_multi_setopt(rbcm->handle, CURLMOPT_MAXCONNECTS, NUM2INT(count));
+
+  return self;
+}
+
+/*
+ * call-seq:
+ * multi = Curl::Multi.new
+ * easy = Curl::Easy.new('url')
+ *
+ * multi.add(easy)
+ *
+ * Add an easy handle to the multi stack
+ */
+static VALUE ruby_curl_multi_add(VALUE self, VALUE easy) {
+  CURLMcode mcode;
+  ruby_curl_easy *rbce;
+  ruby_curl_multi *rbcm;
+
+  Data_Get_Struct(self, ruby_curl_multi, rbcm);
+  Data_Get_Struct(easy, ruby_curl_easy, rbce);
+
+  mcode = curl_multi_add_handle(rbcm->handle, rbce->curl);
+  if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
+    raise_curl_multi_error_exception(mcode);
+  }
+
+  /* save a pointer to self */
+  rbce->self = easy;
+
+  /* setup the easy handle */
+  ruby_curl_easy_setup( rbce, &(rbce->bodybuf), &(rbce->headerbuf), &(rbce->curl_headers) );
+
+  rbcm->active++;
+  if (mcode == CURLM_CALL_MULTI_PERFORM) {
+    curl_multi_perform(rbcm->handle, &(rbcm->running));
+  }
+
+  rb_hash_aset( rbcm->requests, rb_int_new((long)rbce->curl), easy );
+  // active should equal INT2FIX(RHASH(rbcm->requests)->tbl->num_entries)
+
+  if (rbcm->active > rbcm->running) {
+    rb_curl_multi_read_info(self, rbcm->handle);
+  }
+
+  return self;
+}
+
+/*
+ * call-seq:
+ * multi = Curl::Multi.new
+ * easy = Curl::Easy.new('url')
+ *
+ * multi.add(easy)
+ * 
+ * # sometime later
+ * multi.remove(easy)
+ *
+ * Remove an easy handle from a multi stack
+ *
+ * Will raise an exception if the easy handle is not found
+ */
+static VALUE ruby_curl_multi_remove(VALUE self, VALUE easy) {
+  ruby_curl_multi *rbcm;
+
+  Data_Get_Struct(self, ruby_curl_multi, rbcm);
+
+  ruby_curl_easy *rbce;
+  Data_Get_Struct(easy, ruby_curl_easy, rbce);
+
+  rb_curl_multi_remove(rbcm,easy);
+  // active should equal INT2FIX(RHASH(rbcm->requests)->tbl->num_entries)
+  rb_hash_delete( rbcm->requests, rb_int_new((long)rbce->curl) );
+
+  return self;
+}
+static void rb_curl_multi_remove(ruby_curl_multi *rbcm, VALUE easy) {
+  CURLMcode result;
+  ruby_curl_easy *rbce;
+  Data_Get_Struct(easy, ruby_curl_easy, rbce);
+ 
+  rbcm->active--;
+
+  //printf( "calling rb_curl_multi_remove: 0x%X, active: %d\n", (long)easy, rbcm->active );
+
+  result = curl_multi_remove_handle(rbcm->handle, rbce->curl);
+  if (result != 0) {
+    raise_curl_multi_error_exception(result);
+  }
+ 
+  ruby_curl_easy_cleanup( easy, rbce, rbce->bodybuf, rbce->headerbuf, rbce->curl_headers );
+  rbce->headerbuf = Qnil;
+  rbce->bodybuf = Qnil;
+}
+
+static void rb_curl_multi_read_info(VALUE self, CURLM *multi_handle) {
+  int msgs_left, result;
+  CURLMsg *msg;
+  CURLcode ecode;
+  CURL *easy_handle;
+  ruby_curl_easy *rbce = NULL;
+//  VALUE finished = rb_ary_new();
+
+  /* check for finished easy handles and remove from the multi handle */
+  while ((msg = curl_multi_info_read(multi_handle, &msgs_left))) {
+
+    if (msg->msg != CURLMSG_DONE) {
+      continue;
+    }
+
+    easy_handle = msg->easy_handle;
+    result = msg->data.result;
+    if (easy_handle) {
+      ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, (char**)&rbce);
+      if (ecode != 0) {
+        raise_curl_easy_error_exception(ecode);
+      }
+      //printf( "finished: 0x%X\n", (long)rbce->self );
+      //rb_ary_push(finished, rbce->self);
+      ruby_curl_multi_remove( self, rbce->self );
+
+      if (rbce->complete_proc != Qnil) {
+        rb_funcall( rbce->complete_proc, idCall, 1, self );
+      }
+
+      long response_code = -1;
+      curl_easy_getinfo(rbce->curl, CURLINFO_RESPONSE_CODE, &response_code);
+ 
+      if (result != 0) {
+        if (rbce->failure_proc != Qnil) {
+          rb_funcall( rbce->failure_proc, idCall, 1, rbce->self );
+        }
+      }
+      else if (rbce->success_proc != Qnil &&
+              ((response_code >= 200 && response_code < 300) || response_code == 0)) {
+        /* NOTE: we allow response_code == 0, in the case the file is being read from disk */
+        rb_funcall( rbce->success_proc, idCall, 1, rbce->self );
+      }
+      else if (rbce->failure_proc != Qnil &&
+              (response_code >= 300 && response_code <= 999)) {
+        rb_funcall( rbce->failure_proc, idCall, 1, rbce->self );
+      }
+    }
+    else {
+      //printf( "missing easy handle\n" );
+    }
+  }
+
+  /*
+  while (RARRAY(finished)->len > 0) {
+    //printf( "finished handle\n" );
+    ruby_curl_multi_remove( self, rb_ary_pop(finished) );
+  }
+   */
+}
+
+/* called within ruby_curl_multi_perform */
+static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_running) {
+  CURLMcode mcode;
+
+  do {
+    mcode = curl_multi_perform(multi_handle, still_running);
+  } while (mcode == CURLM_CALL_MULTI_PERFORM);
+
+  if (mcode != CURLM_OK) {
+    raise_curl_multi_error_exception(mcode);
+  }
+
+  rb_curl_multi_read_info( self, multi_handle );
+}
+
+/*
+ * call-seq:
+ * multi = Curl::Multi.new
+ * easy1 = Curl::Easy.new('url')
+ * easy2 = Curl::Easy.new('url')
+ *
+ * multi.add(easy1)
+ * multi.add(easy2)
+ *
+ * multi.perform do
+ *  # while idle other code my execute here
+ * end
+ *
+ * Run multi handles, looping selecting when data can be transfered
+ */
+static VALUE ruby_curl_multi_perform(VALUE self) {
+  CURLMcode mcode;
+  ruby_curl_multi *rbcm;
+  int maxfd, rc;
+  fd_set fdread, fdwrite, fdexcep;
+
+  long timeout;
+  struct timeval tv = {0, 0};
+
+  Data_Get_Struct(self, ruby_curl_multi, rbcm);
+  //rb_gc_mark(self);
+
+  rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
+
+  while(rbcm->running) { 
+    FD_ZERO(&fdread);
+    FD_ZERO(&fdwrite);
+    FD_ZERO(&fdexcep);
+
+    /* load the fd sets from the multi handle */
+    mcode = curl_multi_fdset(rbcm->handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+    if (mcode != CURLM_OK) {
+      raise_curl_multi_error_exception(mcode);
+    }
+
+    /* get the curl suggested time out */
+    mcode = curl_multi_timeout(rbcm->handle, &timeout);
+    if (mcode != CURLM_OK) {
+      raise_curl_multi_error_exception(mcode);
+    }
+
+    if (timeout == 0) { /* no delay */
+      rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
+      continue;
+    }
+    else if (timeout == -1) {
+      timeout = 1; /* You must not wait too long 
+                     (more than a few seconds perhaps) before 
+                     you call curl_multi_perform() again */
+    }
+
+    if (rb_block_given_p()) {
+      rb_yield(self);
+    }
+ 
+    tv.tv_sec = timeout / 1000;
+    tv.tv_usec = (timeout * 1000) % 1000000;
+
+    rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
+    if (rc < 0) {
+      rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno));
+    }
+
+    rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
+
+  }
+
+  return Qnil;
+}
+
+/* =================== INIT LIB =====================*/
+void init_curb_multi() {
+  idCall = rb_intern("call");
+
+  cCurlMulti = rb_define_class_under(mCurl, "Multi", rb_cObject);
+
+  /* Class methods */
+  rb_define_singleton_method(cCurlMulti, "new", ruby_curl_multi_new, -1);
+
+  /* Instnace methods */
+  rb_define_method(cCurlMulti, "max_connects=", ruby_curl_multi_max_connects, 1);
+  rb_define_method(cCurlMulti, "add", ruby_curl_multi_add, 1);
+  rb_define_method(cCurlMulti, "remove", ruby_curl_multi_remove, 1);
+  rb_define_method(cCurlMulti, "perform", ruby_curl_multi_perform, 0);
+}