scrubber-scrubyt 0.4.11 → 0.4.12

data/Rakefile

@@ -17,7 +17,7 @@ task "cleanup_readme" => ["rdoc"]
 
 gem_spec = Gem::Specification.new do |s|
   s.name = 'scrubyt'
-  s.version = '0.4.1'
+  s.version = '0.4.12'
   s.summary = 'A powerful Web-scraping framework built on Mechanize and Hpricot (and FireWatir)'
   s.description = %{scRUBYt! is an easy to learn and use, yet powerful and effective web scraping framework. It's most interesting part is a Web-scraping DSL built on HPricot and WWW::Mechanize, which allows to navigate to the page of interest, then extract and query data records with a few lines of code. It is hard to describe scRUBYt! in a few sentences - you have to see it for yourself!}
   # Files containing Test::Unit test cases.

data/lib/scrubyt/core/navigation/agents/mechanize.rb

@@ -29,13 +29,14 @@ module Scrubyt
           #_doc_url_ - the url or file name to fetch
           def self.fetch(doc_url, *args)
             #Refactor this crap!!! with option_accessor stuff
-
             if args.size > 0
               mechanize_doc = args[0][:mechanize_doc]
               html = args[0][:html]
               resolve = args[0][:resolve]
               basic_auth = args[0][:basic_auth]
               parse_and_set_basic_auth(basic_auth) if basic_auth
+              proxy = args[0][:proxy]
+              parse_and_set_proxy(proxy) if proxy
               if html
                 @@current_doc_protocol = 'string'
                 mechanize_doc = page = WWW::Mechanize::Page.new(nil, {'content-type' => 'text/html'}, html) 
@@ -122,6 +123,38 @@ module Scrubyt
             @@original_host_name ||= @@host_name
           end #end of method store_host_name
 
+          def self.parse_and_set_proxy(proxy)
+            proxy = proxy[:proxy]    
+            if proxy.downcase == 'localhost'
+              @@host = 'localhost'
+              @@port = proxy.split(':').last
+            else
+              parts = proxy.split(':')
+              if (parts.size > 2)
+               user_pass = parts[0].split('@')
+               if (user.pass.size > 1)
+                 @@proxy_user = user_pass[0]
+                 @@proxy_pass = user_pass[1]
+               else
+                 @@proxy_user = user_pass
+               end
+               @@host = parts[1]
+               @@port = parts[2]
+              else
+                @@host = parts[0]
+                @@port = parts[1]
+              end
+
+              if (@@host == nil || @@port == nil)# !@@host =~ /^http/)
+                puts "Invalid proxy specification..."
+                puts "neither host nor port can be nil!"
+                exit
+              end
+            end
+            Scrubyt.log :ACTION, "[ACTION] Setting proxy: host=<#{@@host}>, port=<#{@@port}>, username=<#{@@proxy_user}, password=<#{@@proxy_pass}>"
+            @@agent.set_proxy(@@host, @@port)
+          end
+
           def self.determine_protocol
             old_protocol = @@current_doc_protocol
             new_protocol = case @@current_doc_url

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: scrubber-scrubyt
 version: !ruby/object:Gem::Version 
-  version: 0.4.11
+  version: 0.4.12
 platform: ruby
 authors: 
 - Peter Szinek