scrubyt 0.3.4 → 0.4.1

data/CHANGELOG

@@ -1,5 +1,36 @@
 = scRUBYt! Changelog
 
+== 0.4.05
+== 20th October
+
+=<tt>changes:</tt>
+- [NEW] possibility to use FireWatir as the agent for scraping (credit: Glenn Gillen, Glen Gillen and... did I mention Glenn already?)
+- [FIX] navigation doesn't crash if a 404/500 is returned (credit: Glen Gillen)
+- [NEW] navigation action: click_by_xpath to click arbitrary elements
+- [MOD] dropped dependencies: RubyInline, ParseTree, Ruby2Ruby (hooray for win32 users)
+- [NEW] scraping through frames (e.g. google analytics)
+- [MOD] exporting temporarily doesn't work - for now, generated XPaths are printed to the screen
+- [MOD] possibility to wait after clicking link/filling textfield (to be able to scrape inserted AJAX stuff)
+- [NEW] possibility to fetch from a string, by specifying nil as the url and the html string with the :html option 
+- [FIX] firewatir slowness (credit: jak4)
+- [FIX] lot of bugfixes and stability fixes
+- 
+
+== 0.4.0 (unofficial)
+=== 31st October, 2007
+
+=<tt>changes:</tt>
+- [NEW] possibility to define a default value for patterns
+- [MOD] rewrite of to_flat_xml to a more robust algorithm
+- [NEW] find_string method in text pattern; return the string if it's present in the input
+
+== 0.3.4
+=== 26th September, 2007
+
+=<tt>changes:</tt>
+It seems I have been too busy to update the changelog ;)
+
+
 == 0.3.1
 === 29th May, 2007
 

data/README

@@ -1,4 +1,4 @@
-= scRUBYt! - Hpricot and Mechanize on steroids
+= scRUBYt! - Hpricot and Mechanize (or FireWatir)  on steroids
 
 A simple to learn and use, yet very powerful web extraction framework written in Ruby. Navigate through the Web, Extract, query, transform and save relevant data from the Web page of your interest by the concise and easy to use DSL.
 

data/Rakefile

@@ -17,8 +17,8 @@ task "cleanup_readme" => ["rdoc"]
 
 gem_spec = Gem::Specification.new do |s|
   s.name = 'scrubyt'
-  s.version = '0.3.4'
-  s.summary = 'A powerful Web-scraping framework built on Mechanize and Hpricot'
+  s.version = '0.4.1'
+  s.summary = 'A powerful Web-scraping framework built on Mechanize and Hpricot (and FireWatir)'
   s.description = %{scRUBYt! is an easy to learn and use, yet powerful and effective web scraping framework. It's most interesting part is a Web-scraping DSL built on HPricot and WWW::Mechanize, which allows to navigate to the page of interest, then extract and query data records with a few lines of code. It is hard to describe scRUBYt! in a few sentences - you have to see it for yourself!}
   # Files containing Test::Unit test cases.
   s.test_files = FileList['test/unittests/**/*']
@@ -29,12 +29,7 @@ gem_spec = Gem::Specification.new do |s|
   s.homepage = 'http://www.scrubyt.org'
   s.add_dependency('hpricot', '>= 0.5')
   s.add_dependency('mechanize', '>= 0.6.3')
-  s.add_dependency('ParseTreeReloaded')
-  s.add_dependency('RubyInlineAcceleration')
-  s.add_dependency('RubyInline', '= 3.6.3')
-  s.add_dependency('ParseTree', '= 1.7.1')
-  s.add_dependency('ruby2ruby', '= 1.1.6')
-  #s.has_rdoc = 'true'
+  s.has_rdoc = 'true'
 end
 
 ###################################################
@@ -99,7 +94,7 @@ Rake::GemPackageTask.new(gem_spec) do |pkg|
   pkg.need_tar = false
 end
 
-#Rake::PackageTask.new('scrubyt-examples', '0.3.0') do |pkg|
+#Rake::PackageTask.new('scrubyt-examples', '0.4.03') do |pkg|
 #  pkg.need_zip = true
 #  pkg.need_tar = true
 #  pkg.package_files.include("examples/**/*")

data/lib/scrubyt.rb

@@ -1,62 +1,43 @@
-$KCODE = 'u'
-require 'jcode'
+$KCODE = "u"
+require "jcode"
 
 #ruby core
-require 'open-uri'
-require 'erb'
+require "open-uri"
+require "erb"
 
 #gems
-require 'rubygems'
-require 'mechanize'
-require 'hpricot'
-require 'parse_tree_reloaded'
-require 'rexml/text'
-
-#little hack to avoid that ruby2ruby tries to load the original parse_tree
-if Gem
-  module Gem
-    class << self
-      alias_method :activate_orig, :activate
-      def activate(gem, autorequire, *version_requirements)
-        activate_orig(gem, autorequire, *version_requirements) unless gem.is_a?(Gem::Dependency) && gem.name == 'ParseTree'
-      end
-    end
-  end
-end
-module Kernel
-  alias_method :require_orig, :require
-  def require(path)
-    require_orig(path) unless path == 'parse_tree'
-  end
-end
-require 'ruby2ruby'
+require "rexml/text"
+require "rubygems"
+require "mechanize"
+require "hpricot"
 
 #scrubyt
-require 'scrubyt/logging'
-require 'scrubyt/utils/ruby_extensions.rb'
-require 'scrubyt/utils/xpathutils.rb'
-require 'scrubyt/utils/shared_utils.rb'
-require 'scrubyt/utils/simple_example_lookup.rb'
-require 'scrubyt/utils/compound_example_lookup.rb'
-require 'scrubyt/core/scraping/constraint_adder.rb'
-require 'scrubyt/core/scraping/constraint.rb'
-require 'scrubyt/core/scraping/result_indexer.rb'
-require 'scrubyt/core/scraping/pre_filter_document.rb'
-require 'scrubyt/core/scraping/compound_example.rb'
-require 'scrubyt/output/result_node.rb'
-require 'scrubyt/output/scrubyt_result.rb'
-require 'scrubyt/output/export.rb'
-require 'scrubyt/core/navigation/navigation_actions.rb'
-require 'scrubyt/core/navigation/fetch_action.rb'
-require 'scrubyt/core/shared/extractor.rb'
-require 'scrubyt/core/scraping/filters/base_filter.rb'
-require 'scrubyt/core/scraping/filters/attribute_filter.rb'
-require 'scrubyt/core/scraping/filters/constant_filter.rb'
-require 'scrubyt/core/scraping/filters/script_filter.rb'
-require 'scrubyt/core/scraping/filters/text_filter.rb'
-require 'scrubyt/core/scraping/filters/detail_page_filter.rb'
-require 'scrubyt/core/scraping/filters/download_filter.rb'
-require 'scrubyt/core/scraping/filters/html_subtree_filter.rb'
-require 'scrubyt/core/scraping/filters/regexp_filter.rb'
-require 'scrubyt/core/scraping/filters/tree_filter.rb'
-require 'scrubyt/core/scraping/pattern.rb'
+require "#{File.dirname(__FILE__)}/scrubyt/logging"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/ruby_extensions.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/xpathutils.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/shared_utils.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/simple_example_lookup.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/utils/compound_example_lookup.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/constraint_adder.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/constraint.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/result_indexer.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/pre_filter_document.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/compound_example.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/output/result_node.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/output/scrubyt_result.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/navigation/agents/mechanize.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/navigation/agents/firewatir.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/navigation/navigation_actions.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/navigation/fetch_action.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/shared/extractor.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/base_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/attribute_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/constant_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/script_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/text_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/detail_page_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/download_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/html_subtree_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/regexp_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/filters/tree_filter.rb"
+require "#{File.dirname(__FILE__)}/scrubyt/core/scraping/pattern.rb"

data/lib/scrubyt/core/navigation/agents/firewatir.rb

@@ -0,0 +1,249 @@
+require 'rubygems'
+require 'firewatir'
+module Scrubyt
+  ##
+  #=<tt>Fetching pages (and related functionality)</tt>
+  #
+  #Since lot of things are happening during (and before)
+  #the fetching of a document, I decided to move out fetching related
+  #functionality to a separate class - so if you are looking for anything
+  #which is loading a document (even by submitting a form or clicking a link)
+  #and related things like setting a proxy etc. you should find it here.
+  module Navigation
+    module Firewatir
+
+      def self.included(base)
+        base.module_eval do 
+          @@agent = FireWatir::Firefox.new
+          @@current_doc_url = nil
+          @@current_doc_protocol = nil
+          @@base_dir = nil
+          @@host_name = nil
+          @@history = []
+          @@current_form = nil
+          @@current_frame = nil
+
+          ##
+          #Action to fetch a document (either a file or a http address)
+          #
+          #*parameters*
+          #
+          #_doc_url_ - the url or file name to fetch
+          def self.fetch(doc_url, *args)
+            #Refactor this crap!!! with option_accessor stuff
+            if args.size > 0
+              mechanize_doc = args[0][:mechanize_doc]
+              resolve = args[0][:resolve]
+              basic_auth = args[0][:basic_auth]
+              #Refactor this whole stuff as well!!! It looks awful...
+              parse_and_set_basic_auth(basic_auth) if basic_auth
+            else
+              mechanize_doc = nil
+              resolve = :full
+            end
+
+            @@current_doc_url = doc_url
+            @@current_doc_protocol = determine_protocol
+            if mechanize_doc.nil?
+              handle_relative_path(doc_url) unless @@current_doc_protocol == 'xpath'
+              handle_relative_url(doc_url, resolve)
+              Scrubyt.log :ACTION, "fetching document: #{@@current_doc_url}"
+              case @@current_doc_protocol
+                when 'file': @@agent.goto("file://"+ @@current_doc_url)
+                else @@agent.goto(@@current_doc_url)
+              end
+              @@mechanize_doc = "<html>#{@@agent.html}</html>"
+            else
+              @@mechanize_doc = mechanize_doc
+            end
+            @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc))
+            store_host_name(@@agent.url)   # in case we're on a new host
+          end
+          
+          def self.frame(attribute, value)
+            if @@current_frame
+              @@current_frame.frame(attribute, value)
+            else
+              @@current_frame = @@agent.frame(attribute, value)
+            end
+          end
+
+          ##
+          #Submit the last form;
+          def self.submit(current_form, sleep_time=nil, button=nil, type=nil)
+            if @@current_frame
+              #BRUTAL hax but FW is such a shitty piece of software
+              #this sucks FAIL omg            
+              @@current_frame.locate
+              form = Document.new(@@current_frame).all.find{|t| t.tagName=="FORM"}
+              form.submit
+            else
+              @@agent.element_by_xpath(@@current_form).submit
+            end            
+            
+            if sleep_time
+              sleep sleep_time
+              @@agent.wait
+            end
+
+            @@current_doc_url = @@agent.url
+            @@mechanize_doc = "<html>#{@@agent.html}</html>"
+            @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc))
+          end
+
+          ##
+          #Click the link specified by the text
+          def self.click_link(link_spec,index = 0,wait_secs=0)
+            Scrubyt.log :ACTION, "Clicking link specified by: %p" % link_spec
+            if link_spec.is_a?(Hash)
+              elem = XPathUtils.generate_XPath(CompoundExampleLookup.find_node_from_compund_example(@@hpricot_doc, link_spec, false, index), nil, true) 
+              result_page = @@agent.element_by_xpath(elem).click
+            else
+              @@agent.link(:innerHTML, Regexp.escape(link_spec)).click
+            end            
+            sleep(wait_secs) if wait_secs > 0
+            @@agent.wait
+            @@current_doc_url = @@agent.url
+            @@mechanize_doc = "<html>#{@@agent.html}</html>"
+            @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc))
+            Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}"
+          end          
+
+          def self.click_by_xpath(xpath)
+            Scrubyt.log :ACTION, "Clicking by XPath : %p" % xpath        
+            @@agent.element_by_xpath(xpath).click
+            @@agent.wait
+            @@current_doc_url = @@agent.url
+            @@mechanize_doc = "<html>#{@@agent.html}</html>"
+            @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc))
+            Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}"            
+          end
+          
+          def self.click_image_map(index = 0)
+            Scrubyt.log :ACTION, "Clicking image map at index: %p" % index
+            uri = @@mechanize_doc.search("//area")[index]['href']
+            result_page = @@agent.get(uri)
+            @@current_doc_url = result_page.uri.to_s
+            Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}"
+            fetch(@@current_doc_url, :mechanize_doc => result_page)
+          end
+
+          def self.store_host_name(doc_url)
+            @@host_name = doc_url.match(/.*\..*?\//)[0] if doc_url.match(/.*\..*?\//)
+            @@original_host_name ||= @@host_name
+          end #end of method store_host_name
+
+          def self.determine_protocol
+            old_protocol = @@current_doc_protocol
+            new_protocol = case @@current_doc_url
+              when /^\/\//
+                'xpath'
+              when /^https/
+                'https'
+              when /^http/
+                'http'
+              when /^www\./
+                'http'
+              else
+                'file'
+              end
+            return 'http' if ((old_protocol == 'http') && new_protocol == 'file')
+            return 'https' if ((old_protocol == 'https') && new_protocol == 'file')
+            new_protocol
+          end
+
+          def self.parse_and_set_basic_auth(basic_auth)
+            login, pass = basic_auth.split('@')
+            Scrubyt.log :ACTION, "Basic authentication: login=<#{login}>, pass=<#{pass}>"
+            @@agent.basic_auth(login, pass)
+          end
+
+          def self.handle_relative_path(doc_url)
+            if @@base_dir == nil || doc_url[0..0] == "/"
+              @@base_dir = doc_url.scan(/.+\//)[0] if @@current_doc_protocol == 'file'
+            else
+              @@current_doc_url = ((@@base_dir + doc_url) if doc_url !~ /#{@@base_dir}/)
+            end
+          end
+
+          def self.handle_relative_url(doc_url, resolve)
+            return if doc_url =~ /^(http:|javascript:)/
+            if doc_url !~ /^\//
+              first_char = doc_url[0..0]
+              doc_url = ( first_char == '?'  ? '' : '/'  ) + doc_url
+              if first_char == '?' #This is an ugly hack... really have to throw this shit out and go with mechanize's
+                current_uri = @@mechanize_doc.uri.to_s
+                current_uri = @@agent.history.first.uri.to_s if current_uri =~ /\/popup\//
+                if (current_uri.include? '?')
+                  current_uri = current_uri.scan(/.+\//)[0]
+                else
+                  current_uri += '/' unless current_uri[-1..-1] == '/'
+                end
+                @@current_doc_url = current_uri + doc_url
+                return
+              end
+            end
+            case resolve
+              when :full
+                @@current_doc_url = (@@host_name + doc_url) if ( @@host_name != nil && (doc_url !~ /#{@@host_name}/))
+                @@current_doc_url = @@current_doc_url.split('/').uniq.join('/')
+              when :host
+                base_host_name = (@@host_name.count("/") == 2 ? @@host_name : @@host_name.scan(/(http.+?\/\/.+?)\//)[0][0])
+                @@current_doc_url = base_host_name + doc_url
+              else
+                #custom resilving
+                @@current_doc_url = resolve + doc_url
+            end
+          end
+        
+          def self.fill_textfield(textfield_name, query_string, wait_secs, useValue)
+            @@current_form = "//input[@name='#{textfield_name}']/ancestor::form"
+            target = @@current_frame || @@agent
+            if useValue
+              target.text_field(:name,textfield_name).value = query_string              
+            else
+              target.text_field(:name,textfield_name).set(query_string)
+            end
+            sleep(wait_secs) if wait_secs > 0
+            @@mechanize_doc = "<html>#{@@agent.html}</html>"
+            @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc))
+            
+          end
+
+          ##
+          #Action to fill a textarea with text
+          def self.fill_textarea(textarea_name, text)
+            @@current_form = "//input[@name='#{textarea_name}']/ancestor::form"
+            @@agent.text_field(:name,textarea_name).set(text)
+          end
+
+          ##
+          #Action for selecting an option from a dropdown box
+          def self.select_option(selectlist_name, option)
+            @@current_form = "//select[@name='#{selectlist_name}']/ancestor::form"
+            @@agent.select_list(:name,selectlist_name).select(option)
+          end
+
+          def self.check_checkbox(checkbox_name)
+            @@current_form = "//input[@name='#{checkbox_name}']/ancestor::form"
+            @@agent.checkbox(:name,checkbox_name).set(true)
+          end
+
+          def self.check_radiobutton(checkbox_name, index=0)
+            @@current_form = "//input[@name='#{checkbox_name}']/ancestor::form"
+            @@agent.elements_by_xpath("//input[@name='#{checkbox_name}']")[index].set
+          end
+
+          def self.click_image_map(index=0)
+            raise 'NotImplemented'
+          end
+          
+          def self.wait(time=1)
+            sleep(time)
+            @@agent.wait
+          end
+        end
+      end   
+    end
+  end
+end

data/lib/scrubyt/core/navigation/agents/mechanize.rb

@@ -0,0 +1,253 @@
+require 'rubygems'
+require 'mechanize'
+module Scrubyt
+  ##
+  #=<tt>Fetching pages (and related functionality)</tt>
+  #
+  #Since lot of things are happening during (and before)
+  #the fetching of a document, I decided to move out fetching related
+  #functionality to a separate class - so if you are looking for anything
+  #which is loading a document (even by submitting a form or clicking a link)
+  #and related things like setting a proxy etc. you should find it here.
+  module Navigation
+    module Mechanize
+
+      def self.included(base)
+        base.module_eval do 
+          @@agent = WWW::Mechanize.new
+          @@current_doc_url = nil
+          @@current_doc_protocol = nil
+          @@base_dir = nil
+          @@host_name = nil
+          @@history = []
+
+          ##
+          #Action to fetch a document (either a file or a http address)
+          #
+          #*parameters*
+          #
+          #_doc_url_ - the url or file name to fetch
+          def self.fetch(doc_url, *args)
+            #Refactor this crap!!! with option_accessor stuff
+
+            if args.size > 0
+              mechanize_doc = args[0][:mechanize_doc]
+              html = args[0][:html]
+              resolve = args[0][:resolve]
+              basic_auth = args[0][:basic_auth]
+              parse_and_set_basic_auth(basic_auth) if basic_auth
+              if html
+                @@current_doc_protocol = 'string'
+                mechanize_doc = page = WWW::Mechanize::Page.new(nil, {'content-type' => 'text/html'}, html) 
+              end
+            else
+              mechanize_doc = nil
+              resolve = :full
+            end
+
+            @@current_doc_url = doc_url
+            @@current_doc_protocol ||= determine_protocol
+
+            if mechanize_doc.nil? && @@current_doc_protocol != 'file'
+              handle_relative_path(doc_url)
+              handle_relative_url(doc_url, resolve)
+              Scrubyt.log :ACTION, "fetching document: #{@@current_doc_url}"
+
+              unless 'file' == @@current_doc_protocol
+                @@mechanize_doc = @@agent.get(@@current_doc_url)
+              end
+            else
+              @@mechanize_doc = mechanize_doc
+            end
+
+            if @@current_doc_protocol == 'file'
+              @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(open(@@current_doc_url).read))
+            else
+              @@hpricot_doc = Hpricot(PreFilterDocument.br_to_newline(@@mechanize_doc.body))
+              store_host_name(self.get_current_doc_url) if self.get_current_doc_url   # in case we're on a new host
+            end
+          end
+
+          ##
+          #Submit the last form;
+          def self.submit(index=nil, sleep_time=nil, type=nil)
+            Scrubyt.log :ACTION, 'Submitting form...'
+            if index == nil
+              result_page = @@agent.submit(@@current_form)
+              process_submit(@@current_form)
+              #----- added by nickmerwin@gmail.com -----
+            elsif index.class == String && !type.nil?
+              button = @@current_form.buttons.detect{|b| b.name == index}
+              result_page = @@current_form.submit(button)
+              process_submit(@@current_form, button,type)
+              #-----------------------------------------
+            else
+              result_page = @@agent.submit(@@current_form, @@current_form.buttons[index])
+            end
+            @@current_doc_url = result_page.uri.to_s
+            Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}"
+            fetch(@@current_doc_url, :mechanize_doc => result_page)
+          end
+
+          ##
+          #Click the link specified by the text
+          def self.click_link(link_spec,index = 0,wait_secs=0)
+            Scrubyt.log :ACTION, "Clicking link specified by: %p" % link_spec
+            if link_spec.is_a? Hash
+              clicked_elem = CompoundExampleLookup.find_node_from_compund_example(@@hpricot_doc, link_spec, false, index)
+            else
+              clicked_elem = SimpleExampleLookup.find_node_from_text(@@hpricot_doc, link_spec, false, index)
+            end
+            clicked_elem = XPathUtils.find_nearest_node_with_attribute(clicked_elem, 'href')
+            result_page = @@agent.click(clicked_elem)
+            @@current_doc_url = result_page.uri.to_s
+            Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}"
+            fetch(@@current_doc_url, :mechanize_doc => result_page)
+          end
+
+          def self.click_image_map(index = 0)
+            Scrubyt.log :ACTION, "Clicking image map at index: %p" % index
+            uri = @@mechanize_doc.search("//area")[index]['href']
+            result_page = @@agent.get(uri)
+            @@current_doc_url = result_page.uri.to_s
+            Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}"
+            fetch(@@current_doc_url, :mechanize_doc => result_page)
+          end
+
+          def self.store_host_name(doc_url)
+            @@host_name = 'http://' + @@mechanize_doc.uri.to_s.match(%r{http://(.+?)/+})[0] if @@current_doc_protocol == 'http'
+            @@host_name = 'https://' + @@mechanize_doc.uri.to_s.match(%r{https://(.+?)/+})[0] if @@current_doc_protocol == 'https'
+            @@host_name = doc_url if @@host_name == nil
+            @@host_name = @@host_name[0..-2] if @@host_name[-1].chr == '/'
+            @@original_host_name ||= @@host_name
+          end #end of method store_host_name
+
+          def self.determine_protocol
+            old_protocol = @@current_doc_protocol
+            new_protocol = case @@current_doc_url
+              when /^https/
+                'https'
+              when /^http/
+                'http'
+              when /^www/
+                'http'
+              else
+                'file'
+              end
+            return 'http' if ((old_protocol == 'http') && new_protocol == 'file')
+            return 'https' if ((old_protocol == 'https') && new_protocol == 'file')
+            new_protocol
+          end
+
+          def self.handle_relative_path(doc_url)
+            if @@base_dir == nil
+              @@base_dir = doc_url.scan(/.+\//)[0] if @@current_doc_protocol == 'file'
+            else
+              @@current_doc_url = ((@@base_dir + doc_url) if doc_url !~ /#{@@base_dir}/)
+            end
+          end
+
+          def self.handle_relative_url(doc_url, resolve)
+            return if doc_url =~ /^http/
+            if doc_url !~ /^\//
+              first_char = doc_url[0..0]
+              doc_url = ( first_char == '?'  ? '' : '/'  ) + doc_url
+              if first_char == '?' #This is an ugly hack... really have to throw this shit out and go with mechanize's
+                current_uri = @@mechanize_doc.uri.to_s
+                current_uri = @@agent.history.first.uri.to_s if current_uri =~ /\/popup\//
+                if (current_uri.include? '?')
+                  current_uri = current_uri.scan(/.+\//)[0]
+                else
+                  current_uri += '/' unless current_uri[-1..-1] == '/'
+                end
+                @@current_doc_url = current_uri + doc_url
+                return
+              end
+            end
+            case resolve
+              when :full
+                @@current_doc_url = (@@host_name + doc_url) if ( @@host_name != nil && (doc_url !~ /#{@@host_name}/))
+                @@current_doc_url = @@current_doc_url.split('/').uniq.join('/')
+              when :host
+                base_host_name = (@@host_name.count("/") == 2 ? @@host_name : @@host_name.scan(/(http.+?\/\/.+?)\//)[0][0])
+                @@current_doc_url = base_host_name + doc_url
+              else
+                #custom resilving
+                @@current_doc_url = resolve + doc_url
+            end
+          end
+        
+          def self.fill_textfield(textfield_name, query_string, *unused)
+            lookup_form_for_tag('input','textfield',textfield_name,query_string)
+            eval("@@current_form['#{textfield_name}'] = '#{query_string}'")
+          end
+
+          ##
+          #Action to fill a textarea with text
+          def self.fill_textarea(textarea_name, text)
+            lookup_form_for_tag('textarea','textarea',textarea_name,text)
+            eval("@@current_form['#{textarea_name}'] = '#{text}'")
+          end
+
+          ##
+          #Action for selecting an option from a dropdown box
+          def self.select_option(selectlist_name, option)
+            lookup_form_for_tag('select','select list',selectlist_name,option)
+            select_list = @@current_form.fields.find {|f| f.name == selectlist_name}
+            searched_option = select_list.options.find{|f| f.text.strip == option}
+            searched_option.click
+          end
+
+          def self.check_checkbox(checkbox_name)
+            lookup_form_for_tag('input','checkbox',checkbox_name, '')
+            @@current_form.checkboxes.name(checkbox_name).check
+          end
+
+          def self.check_radiobutton(checkbox_name, index=0)
+            lookup_form_for_tag('input','radiobutton',checkbox_name, '',index)
+            @@current_form.radiobuttons.name(checkbox_name)[index].check
+          end
+
+          #private
+          def self.process_submit(current_form, button=nil, type=nil)
+            if button == nil
+              result_page = @@agent.submit(current_form)
+            elsif type
+              result_page = current_form.submit(button)
+            else
+              result_page = @@agent.submit(current_form, button)
+            end
+            @@current_doc_url = result_page.uri.to_s
+            Scrubyt.log :ACTION, "Fetching #{@@current_doc_url}"
+            fetch(@@current_doc_url, :mechanize_doc => result_page)
+          end
+          
+          def self.lookup_form_for_tag(tag, widget_name, name_attribute, query_string, index=0)
+            Scrubyt.log :ACTION, "typing #{query_string} into the #{widget_name} named '#{name_attribute}'"
+            widget = (FetchAction.get_hpricot_doc/"#{tag}[@name=#{name_attribute}]").map()[index]
+            form_tag = Scrubyt::XPathUtils.traverse_up_until_name(widget, 'form')
+            find_form_based_on_tag(form_tag, ['name', 'id', 'action'])
+          end
+
+          def self.find_form_based_on_tag(tag, possible_attrs)
+            lookup_attribute_name = nil
+            lookup_attribute_value = nil
+
+            possible_attrs.each { |a|
+              lookup_attribute_name = a
+              lookup_attribute_value = tag.attributes[a]
+              break if lookup_attribute_value != nil
+            }
+            i = 0
+            loop do
+              @@current_form = FetchAction.get_mechanize_doc.forms[i]
+              return nil if @@current_form == nil
+              break if @@current_form.form_node.attributes[lookup_attribute_name] == lookup_attribute_value
+              i+= 1
+            end
+          end
+        end
+      end   
+    end
+  end
+end