rhack 0.4.1 → 1.0.0.rc4

data/config/rhack.yml.template

@@ -0,0 +1,40 @@
+# By default, Scout uses randomly chosen ua (string from specified fiel)
+# for User-Agent header
+useragents: 
+  desktop: config/uas.txt
+  mobile: config/uas.mobile.txt
+
+# Options for RMTools::Logger. The logger will be initialized as RHACK::L constant
+logger:
+  :out: log/rhack.log
+  :color_log: true
+
+scout:
+  timeout: 60
+  # Curl::Error subclasses names for <domain> that will retrigger Scout#load!
+  # By default an error will keep Scout from having request done
+  retry:
+    example.com:
+    - TimeoutError
+  # Path to cacert needed to use SSL. 
+  # Defaults ("") to that of http://curl.haxx.se/ca/cacert.pem being stored next to this file
+  cacert: ""
+
+# Defaults to a file @ RAILS_PATH/config/RAILS_ENV.yml
+# Rhack is intended to use a database only for caching,
+# so I've came to a conclusion that Redis is the most suitable choice
+# redis.conf can be generated using `rake redis:config`
+# Delete this block to disable Redis connection at all 
+db:
+  redis:
+    configfilename: config/redis.conf
+    socket: tmp/sockets/redis.socket
+    port: 0
+    hostname: localhost
+    dbfilename: db/redis-dump.rdb
+    db: 0
+    save:
+    # "run BGSAVE every minute if anything have changed"
+    - 60 1
+    # You can change it to whatever you want and just use Redis#bgsave before exit Ruby
+  

data/ext/curb-original/curb_config.h

@@ -231,5 +231,8 @@
 #define  HAVE_CURLOPT_NEW_DIRECTORY_PERMS 1
 #define  HAVE_CURLOPT_TELNETOPTIONS 1
 #define  HAVE_CURLE_OBSOLETE 1
+#define  HAVE_RUBY19_HASH 1
+#define  HAVE_RUBY19_ST_H 1
 #define  HAVE_CURL_EASY_ESCAPE 1
+#define HAVE_RB_THREAD_BLOCKING_REGION 1
 #endif

data/ext/curb-original/curb_easy.c

@@ -211,10 +211,6 @@ static void ruby_curl_easy_free(ruby_curl_easy *rbce) {
   if (rbce->curl) {
     curl_easy_cleanup(rbce->curl);
   }
-  
-  if (rbce->first) {
-    curl_formfree(rbce->first);
-  }
 }
 
 void curl_easy_free(ruby_curl_easy *rbce) {
@@ -705,15 +701,11 @@ static VALUE ruby_curl_easy_post_body_set(VALUE self, VALUE post_body) {
   Data_Get_Struct(self, ruby_curl_easy, rbce);
   
   curl = rbce->curl;
-  if (rbce->first) {
-    curl_formfree(rbce->first);
-    rbce->first = NULL;
-    rbce->last = NULL;
-  }
-  rbce->multipart_form_post = 0;
+  
   if ( post_body == Qnil ) {
     //rbce->postdata_buffer = Qnil;
-    rb_easy_del("postdata_buffer");    
+    rb_easy_del("postdata_buffer");
+    
   } else {  
     if (rb_type(post_body) == T_STRING) {
       data = StringValuePtr(post_body);
@@ -742,42 +734,6 @@ static VALUE ruby_curl_easy_post_body_set(VALUE self, VALUE post_body) {
   
   return Qnil;
 }
-static VALUE ruby_curl_easy_multipart_post_body_set(VALUE self, VALUE ary) {
-  ruby_curl_easy *rbce;
-  CURL *curl;
-  int i;
-  
-  Data_Get_Struct(self, ruby_curl_easy, rbce);
-  curl = rbce->curl;
-  if (rbce->first) {
-    curl_formfree(rbce->first);
-    rbce->first = NULL;
-    rbce->last = NULL;
-  }
-  
-  // Make the multipart form
-  rbce->multipart_form_post = 1;
-  for (i=0; i<RARRAY_LEN(ary); i++) {
-    if (rb_obj_is_instance_of(RARRAY_PTR(ary)[i], cCurlPostField)) {
-      append_to_form(RARRAY_PTR(ary)[i], &(rbce->first), &(rbce->last));
-    } 
-    else {
-      if (rbce->first) {
-        curl_formfree(rbce->first);
-        rbce->first = NULL;
-        rbce->last = NULL;
-      }
-      rb_raise(eCurlErrInvalidPostField, "You must use only PostFields with multipart form posts");
-      return Qnil;
-    }
-  }
-  //rbce->postdata_buffer = ary;
-  rb_easy_set("postdata_buffer", post_body);
-
-  curl_easy_setopt(curl, CURLOPT_POST, 0);
-  curl_easy_setopt(curl, CURLOPT_HTTPPOST, rbce->first);
-  return Qnil;
-}
 
 /*
  * call-seq:
@@ -2401,12 +2357,6 @@ static VALUE ruby_curl_easy_perform_post(int argc, VALUE *argv, VALUE self) {
     }
   }
 }
-static VALUE ruby_curl_easy_set_method_get(VALUE self, VALUE onoff) {
-  ruby_curl_easy *rbce;
-  Data_Get_Struct(self, ruby_curl_easy, rbce);
-  curl_easy_setopt(rbce->curl, CURLOPT_HTTPGET, RTEST(onoff));
-  return onoff;
-}
 
 /*
  * call-seq:
@@ -3315,7 +3265,6 @@ void init_curb_easy() {
   rb_define_method(cCurlEasy, "encoding", ruby_curl_easy_encoding_get, 0);
   rb_define_method(cCurlEasy, "useragent=", ruby_curl_easy_useragent_set, 1);
   rb_define_method(cCurlEasy, "useragent", ruby_curl_easy_useragent_get, 0);
-  rb_define_method(cCurlEasy, "multipart_post_body=", ruby_curl_easy_multipart_post_body_set, 1);
   rb_define_method(cCurlEasy, "post_body=", ruby_curl_easy_post_body_set, 1);
   rb_define_method(cCurlEasy, "post_body", ruby_curl_easy_post_body_get, 0);
   rb_define_method(cCurlEasy, "put_data=", ruby_curl_easy_put_data_set, 1);

data/ext/curb-original/curb_multi.c

@@ -31,14 +31,6 @@ static VALUE idCall;
 #endif
 
 VALUE cCurlMulti;
-static void ruby_log(const char* str)
-{
-  rb_funcall(rb_const_get(mCurl, rb_intern("L")), rb_intern("debug"), 1, rb_str_new2(str));
-}
-static void ruby_log_obj(VALUE obj)
-{
-  rb_funcall(rb_const_get(mCurl, rb_intern("L")), rb_intern("debug"), 1, obj);
-}
 
 static long cCurlMutiDefaulttimeout = 100; /* milliseconds */
 
@@ -159,13 +151,6 @@ static VALUE ruby_curl_multi_requests(VALUE self) {
   
   return result_array;
 }
-static VALUE ruby_curl_multi_running(VALUE self) {
-  ruby_curl_multi *rbcm;
-
-  Data_Get_Struct(self, ruby_curl_multi, rbcm);
-
-  return INT2FIX(rbcm->running);
-}
 
 /*
  * call-seq:
@@ -455,8 +440,6 @@ static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_runnin
     raise_curl_multi_error_exception(mcode);
   }
   
-  rb_curl_multi_read_info( self, multi_handle );
-  if (rb_block_given_p()) rb_yield(self);
 }
 
 #ifdef _WIN32
@@ -517,78 +500,7 @@ static VALUE curb_select(void *args) {
  *
  * Run multi handles, looping selecting when data can be transfered
  */
-static VALUE ruby_curl_multi_perform(int argc, VALUE *argv, VALUE self) {
-  ruby_curl_multi *rbcm;
-  VALUE idle;
-  
-  Data_Get_Struct(self, ruby_curl_multi, rbcm);
-  
-  if (!(rbcm->active || rbcm->running)) {
-    rb_scan_args(argc, argv, "01", &idle);
-    if (idle == Qtrue) {
-      
-      if (rb_gv_get("$CarierThreadIsJoined") == Qtrue) {
-        ruby_log("Nothing to perform; recalling...");
-        return Qfalse;
-      }
-      ruby_log("Nothing to perform; idling...");
-      rb_curl_multi_idle_perform(self, rbcm);
-      
-    }
-    else {
-      rb_raise(rb_eRuntimeError, "Nothing to perform");
-    }
-    return Qtrue;
-  }
-  
-  rb_curl_multi_perform(self, rbcm);
-  return Qtrue;
-}
-
-static void rb_curl_multi_idle_perform(VALUE self, ruby_curl_multi *rbcm) {
-  struct timeval tv = {1, 0}; /* sleep time must not be lesser 1 second, otherwise multi thread will always "run" */
-  int rc, maxfd;
-#ifdef _WIN32
-  fd_set crt_fdread, crt_fdwrite, crt_fdexcep;
-#endif
-#ifdef HAVE_RB_THREAD_BLOCKING_REGION
-  struct _select_set fdset_args;
-#endif
-  fd_set fdread, fdwrite, fdexcep;
-  FD_ZERO(&fdread);
-  FD_ZERO(&fdwrite);
-  FD_ZERO(&fdexcep);
-#ifdef _WIN32
-  create_crt_fd(&fdread, &crt_fdread);
-  create_crt_fd(&fdwrite, &crt_fdwrite);
-  create_crt_fd(&fdexcep, &crt_fdexcep);
-#endif
-  
-  do {
-#ifdef HAVE_RB_THREAD_BLOCKING_REGION
-    fdset_args.maxfd = 0;
-    fdset_args.fdread = &fdread;
-    fdset_args.fdwrite = &fdwrite;
-    fdset_args.fdexcep = &fdexcep;
-    fdset_args.tv = &tv;
-    rc = rb_thread_blocking_region(curb_select, &fdset_args, RUBY_UBF_IO, 0);
-#else
-    rc = rb_thread_select(0, &fdread, &fdwrite, &fdexcep, &tv);
-#endif
-    if (rc == -1)
-      rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno));
-    
-  } while (!(RHASH_TBL(rbcm->requests)->num_entries));
-  
-#ifdef _WIN32
-  cleanup_crt_fd(&fdread, &crt_fdread);
-  cleanup_crt_fd(&fdwrite, &crt_fdwrite);
-  cleanup_crt_fd(&fdexcep, &crt_fdexcep);
-#endif
-  rb_curl_multi_perform(self, rbcm);
-}
-
-void rb_curl_multi_perform(VALUE self, ruby_curl_multi *rbcm) {
+VALUE ruby_curl_multi_perform(int argc, VALUE *argv, VALUE self) {
   CURLMcode mcode;
   ruby_curl_multi *rbcm;
   int maxfd, rc;
@@ -598,85 +510,102 @@ void rb_curl_multi_perform(VALUE self, ruby_curl_multi *rbcm) {
 #endif
   long timeout_milliseconds;
   struct timeval tv = {0, 0};
+  VALUE block = Qnil;
 #ifdef HAVE_RB_THREAD_BLOCKING_REGION
   struct _select_set fdset_args;
 #endif
+
+  rb_scan_args(argc, argv, "0&", &block);
+
+  Data_Get_Struct(self, ruby_curl_multi, rbcm);
+
   timeout_milliseconds = cCurlMutiDefaulttimeout;
 
   rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
-  
-  while (rbcm->running) {
+  rb_curl_multi_read_info( self, rbcm->handle );
+  if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
+ 
+  do {
+    while (rbcm->running) {
+
 #ifdef HAVE_CURL_MULTI_TIMEOUT
-    /* get the curl suggested time out */
-    mcode = curl_multi_timeout(rbcm->handle, &timeout_milliseconds);
-    if (mcode != CURLM_OK) {
-      raise_curl_multi_error_exception(mcode);
-    }
+      /* get the curl suggested time out */
+      mcode = curl_multi_timeout(rbcm->handle, &timeout_milliseconds);
+      if (mcode != CURLM_OK) {
+        raise_curl_multi_error_exception(mcode);
+      }
 #else
-    /* libcurl doesn't have a timeout method defined, initialize to -1 we'll pick up the default later */
-    timeout_milliseconds = -1;
+      /* libcurl doesn't have a timeout method defined, initialize to -1 we'll pick up the default later */
+      timeout_milliseconds = -1;
 #endif
 
-    if (timeout_milliseconds == 0) { /* no delay */
-      rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
-      continue;
-    }
+      if (timeout_milliseconds == 0) { /* no delay */
+        rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
+        rb_curl_multi_read_info( self, rbcm->handle );
+        if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
+        continue;
+      }
 
-    if (timeout_milliseconds < 0 || timeout_milliseconds > cCurlMutiDefaulttimeout) {
-      timeout_milliseconds = cCurlMutiDefaulttimeout; /* libcurl doesn't know how long to wait, use a default timeout */
-                                                      /* or buggy versions libcurl sometimes reports huge timeouts... let's cap it */
-    }
+      if (timeout_milliseconds < 0 || timeout_milliseconds > cCurlMutiDefaulttimeout) {
+        timeout_milliseconds = cCurlMutiDefaulttimeout; /* libcurl doesn't know how long to wait, use a default timeout */
+                                                        /* or buggy versions libcurl sometimes reports huge timeouts... let's cap it */
+      }
 
-    tv.tv_sec  = 0; /* never wait longer than 1 second */
-    tv.tv_usec = (int)(timeout_milliseconds * 1000); /* XXX: int is the right type for OSX, what about linux? */
+      tv.tv_sec  = 0; /* never wait longer than 1 second */
+      tv.tv_usec = (int)(timeout_milliseconds * 1000); /* XXX: int is the right type for OSX, what about linux? */
 
-    FD_ZERO(&fdread);
-    FD_ZERO(&fdwrite);
-    FD_ZERO(&fdexcep);
+      FD_ZERO(&fdread);
+      FD_ZERO(&fdwrite);
+      FD_ZERO(&fdexcep);
 
-    /* load the fd sets from the multi handle */
-    mcode = curl_multi_fdset(rbcm->handle, &fdread, &fdwrite, &fdexcep, &maxfd);
-    if (mcode != CURLM_OK) {
-      raise_curl_multi_error_exception(mcode);
-    }
-    ruby_log_obj(INT2FIX(maxfd));
+      /* load the fd sets from the multi handle */
+      mcode = curl_multi_fdset(rbcm->handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+      if (mcode != CURLM_OK) {
+        raise_curl_multi_error_exception(mcode);
+      }
 
 #ifdef _WIN32
-    create_crt_fd(&fdread, &crt_fdread);
-    create_crt_fd(&fdwrite, &crt_fdwrite);
-    create_crt_fd(&fdexcep, &crt_fdexcep);
+      create_crt_fd(&fdread, &crt_fdread);
+      create_crt_fd(&fdwrite, &crt_fdwrite);
+      create_crt_fd(&fdexcep, &crt_fdexcep);
 #endif
 
 #ifdef HAVE_RB_THREAD_BLOCKING_REGION
-    fdset_args.maxfd = maxfd+1;
-    fdset_args.fdread = &fdread;
-    fdset_args.fdwrite = &fdwrite;
-    fdset_args.fdexcep = &fdexcep;
-    fdset_args.tv = &tv;
-    rc = rb_thread_blocking_region(curb_select, &fdset_args, RUBY_UBF_IO, 0);
+      fdset_args.maxfd = maxfd+1;
+      fdset_args.fdread = &fdread;
+      fdset_args.fdwrite = &fdwrite;
+      fdset_args.fdexcep = &fdexcep;
+      fdset_args.tv = &tv;
+      rc = rb_thread_blocking_region(curb_select, &fdset_args, RUBY_UBF_IO, 0);
 #else
-    rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
+      rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
 #endif
 
 #ifdef _WIN32
-    cleanup_crt_fd(&fdread, &crt_fdread);
-    cleanup_crt_fd(&fdwrite, &crt_fdwrite);
-    cleanup_crt_fd(&fdexcep, &crt_fdexcep);
+      cleanup_crt_fd(&fdread, &crt_fdread);
+      cleanup_crt_fd(&fdwrite, &crt_fdwrite);
+      cleanup_crt_fd(&fdexcep, &crt_fdexcep);
 #endif
 
-    switch(rc) {
-    case -1:
-      rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno));
-      break;
-    case 0: /* timeout */
-    default: /* action */
-      rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
-      break;
+      switch(rc) {
+      case -1:
+        rb_raise(rb_eRuntimeError, "select(): %s", strerror(errno));
+        break;
+      case 0: /* timeout */
+      default: /* action */
+        rb_curl_multi_run( self, rbcm->handle, &(rbcm->running) );
+        rb_curl_multi_read_info( self, rbcm->handle );
+        if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
+        break;
+      }
     }
-  }
+
+  } while( rbcm->running );
 
   rb_curl_multi_read_info( self, rbcm->handle );
-  if (rb_block_given_p()) {rb_yield(self);}
+  if (block != Qnil) { rb_funcall(block, rb_intern("call"), 1, self);  }
+    
+  return Qtrue;
 }
 
 /* =================== INIT LIB =====================*/

data/ext/curb/curb_multi.c

@@ -661,7 +661,7 @@ static VALUE ruby_curl_multi_perform(int argc, VALUE *argv, VALUE self) {
     rb_scan_args(argc, argv, "01", &idle);
     if (idle == Qtrue) {
       
-      if (rb_gv_get("$CarierThreadIsJoined") == Qtrue) {
+      if (rb_funcall(mCurl, rb_intern("joined"), 0) == Qtrue) {
         ruby_log("Nothing to perform; recalling...");
         return Qfalse;
       }

data/lib/rhack.rb

@@ -1,17 +1,87 @@
 # encoding: utf-8
 $KCODE	= 'UTF-8' if RUBY_VERSION < "1.9"
-require 'rmtools_dev' unless defined? RMTools
-requrie 'cgi'
-here = File.expand_path File.dirname __FILE__
+require 'rmtools'
+here = File.expand_path '..', __FILE__
+require "rhack/version"
+
 require File.join(here, 'curb_core.so')
-require 'active_record'
-Dir.glob("#{here}/extensions/**.rb") { |f| require f }
+require "rhack/js/johnson"
+
+module RHACK
+  mattr_reader :config, :redis, :useragents
+  
+  Dir.chdir ENV['APP_ROOT'] if ENV['APP_ROOT']
+  cfgfile = Dir['{config/,}rhack.yml'].first
+  @@config = cfgfile ? YAML.load(IO.read(cfgfile)) : {}
+  
+  L = RMTools::RMLogger.new(config.logger || {})
+  
+  db = config.db || {}
+  @@redis = nil
+  if rcfg = db.redis
+    begin
+      require 'redis'
+      @@redis = ::Redis.new(path: rcfg.socket, db: rcfg.db)
+      redis.client.connect
+    rescue Errno::ENOENT => errno
+      rfcgfile = rcfg.configfilename || File.expand_path('config/redis.conf')
+      cmd = %{redis-server "#{rcfgfile}"}
+      msg = "#{errno.message}. Trying to run redis-server: `#{cmd}`"
+      $stderr.puts msg
+      L.log msg
+      res = `#{cmd}`
+      begin
+        redis.client.connect
+      rescue => err
+        err.message = "Can't connect to redis using config @ #{rcfgfile}. #{err.message}"
+        raise err
+      end
+    rescue LoadError => loaderr
+      msg = "#{loaderr.message}. RHACK.redis will stay turned off"
+      $stderr.puts msg
+      L.log msg
+    end
+  end
+  
+  uas = config.useragents || {}
+  if uas = uas.desktop.to_s and File.file? uas
+    @@useragents = IO.read(uas)/"\n"
+  else
+    @@useragents = ['Mozilla/5.0 (Windows NT 6.1; WOW64; rv:14.0) Gecko/20100101 Firefox/14.0.1']
+  end
+  
+  class Scout
+    mattr_reader :cacert
+    mattr_accessor :retry, :timeout
+    
+    scout = RHACK.config.scout || {}
+    @@retry     = scout.retry.b || {}
+    @@timeout = scout.timeout.b || 60
+    @@cacert  = scout.cacert.b ? File.expand_path(scout.cacert) : File.expand_path('../../config/cacert.pem', __FILE__)
+  end
+  
+end
+
+module Curl
+  # $Carier, $CarierThread, $CarierThreadIsJoined respectively
+  mattr_reader :carier, :carier_thread, :joined
+  L = RHACK::L
+  
+  @@carier = Curl::Multi.new
+  carier.pipeline = true
+  $Carier	||= carier
+end
+
+class Johnson::Runtime
+  L = RHACK::L
+end
 
-require "#{here}/init" unless defined? RHACK
-RHACK::VERSION = IO.read(File.join here, '..', 'Rakefile').match(/RHACK_VERSION = '(.+?)'/)[1]
+# uhm, backward compatibility? more like historicity
+HTTPAccessKit = RHACK
 
-require "#{here}/curl-global"
-require "#{here}/scout"
-require "#{here}/frame"
-require "#{here}/words"
-require "#{here}/cache" if defined? RHACK::CacheDir
+require "rhack/curl"
+require "rhack/cookie"
+require "rhack/scout"
+require "rhack/scout_squad"
+require "rhack/frame"
+require "rhack/page"