watir_crawler 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format progress
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in watir_crawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# WatirCrawler
+
+A watir based web crawler
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'watir_crawler'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install watir_crawler
+
+## Usage
+
+See examples
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+

data/examples/example.rb

@@ -0,0 +1,115 @@
+require 'bundler/setup'
+require 'watir_crawler'
+
+#WatirCrawler.logger.level = Logger::INFO
+#WatirCrawler.debug = true
+
+class CrawlerExample < WatirCrawler::Base
+  def yandex_news
+    browser_session do
+      goto 'http://yandex.ru'
+
+      news_list = "//ul[@class='b-news-list']"
+      wait(news_list)
+
+      pull(news_list) do
+        titles = pull(:all, "./li").map{|li| li.text }
+        links  = pull(:all, "./li/a").map{|a| a.uri }
+
+        Hash[ titles.zip(links) ]
+      end
+    end
+  end
+
+  def goto_login
+    browser_session do
+      goto 'https://myaccount.alagasco.com/ccsuces/public/frameset_top_html.jsp'
+    end
+  end
+
+  def google_news
+    browser_session do
+      goto 'http://news.google.com/'
+
+      sections = "//div[@class='section-stream-content']//div[@class='section-list-content']/div"
+      wait(sections)
+
+      pull(:all, sections).reduce({}) do |result, section|
+        pull(section.node_xpath) do
+          section_name   = pull(".//span[@class='section-name']").text
+          article_titles = pull(:all, ".//span[@class='titletext']").map{|element| element.text }
+
+          result[section_name] = article_titles
+        end
+
+        result
+      end
+    end
+  end
+
+  def get_proxy(proxy_port = 3128)
+    browser_session do
+      goto 'http://hideme.ru/proxy-list/'
+
+      checkbox = "//input[@id='c_all']"
+      wait(checkbox).clear
+      sleep 1
+
+      wait("//select[@id='country']").select_value('JP')
+      wait("//input[@id='t_h']").set # set http proxy
+      wait("//input[@id='maxtime']").set 1400 # set proxy timeout
+      wait("//input[@id='ports']").set proxy_port # set proxy port
+      wait("//a[contains(@href,'search()')]").click # search !
+
+      # get 1th proxy ip from the list
+      proxy_list = "//table[@class='pl']"
+      wait(proxy_list)
+
+      proxy_ip = pull(proxy_list) { pull(".//tr[2]/td[1]") }
+      raise 'No proxy found' unless proxy_ip
+
+      [proxy_ip.text, proxy_port]
+    end
+  end
+
+  def via_proxy proxy_ip, proxy_port
+    browser_profile do |profile|
+      profile['network.proxy.type'] = 1
+      profile['network.proxy.http'] = proxy_ip
+      profile['network.proxy.http_port'] = proxy_port.to_i
+    end
+
+    browser_session do
+      goto 'http://www.whatsmyip.org/'
+      wait("//span[@id='ip']").text
+    end
+  end
+
+end
+
+# ----------------------------------------------------------------------------------------------------------------------
+
+timeouts = {
+  :page_load => 150,
+  :wait_timeout => 100 # wait for element on the page
+}
+
+begin
+  crawler = CrawlerExample.new(timeouts)
+
+  puts 'Last google news'
+  p crawler.google_news
+
+  puts 'Last yandex news'
+  p crawler.yandex_news
+
+  proxy = crawler.get_proxy
+  puts "Found proxy: #{proxy.join(':')}"
+
+  current_proxy = crawler.via_proxy(*proxy)
+  puts "Current proxy: #{current_proxy}"
+rescue WatirCrawler::SiteTooSlow
+  puts
+  puts 'ERROR: Site too slow'
+end
+

data/lib/watir_crawler/abstract.rb

@@ -0,0 +1,5 @@
+module WatirCrawler
+  class Abstract
+    include Loggable
+  end
+end

data/lib/watir_crawler/base.rb

@@ -0,0 +1,216 @@
+require 'timeout' # fix error 'uninitialized constant WatirCrawler::Base::Timeout'
+
+module WatirCrawler
+  class Base < Abstract
+    attr_reader :timeouts
+
+    def initialize(timeouts = {})
+      @elements_path = []
+      @timeouts = { :wait_timeout => 150 }.merge(timeouts)
+      @browser = WatirCrawler::Browser.new(@timeouts)
+    end
+
+    def browser_profile
+      @browser.profile do |profile|
+        yield profile if block_given?
+      end
+    end
+
+    def browser
+      @browser.browser
+    end
+
+    def browser_start
+      @browser.start
+    end
+
+    def browser_stop
+      @browser.stop
+    end
+
+    def browser_session
+      timer do
+        catch_error do
+          browser_start
+          yield
+        end
+      end
+    ensure
+      browser_stop
+    end
+
+    def timer
+      log.info "Session start"
+      start_time = Time.now
+      yield
+    ensure
+      log.info "Session end, elapsed time: #{Time.now - start_time}"
+    end
+
+    def catch_error
+      yield
+    rescue Timeout::Error, # http connection with driver
+           Selenium::WebDriver::Error::TimeOutError, # browser.driver.manage.timeouts.page_load
+           Selenium::WebDriver::Error::ScriptTimeOutError # browser.driver.manage.timeouts.script_timeout
+
+      log.error "Site is too slow at page: '#{browser.url}'"
+      raise SiteTooSlow
+
+    rescue SystemCallError, # 'Unknown error - Connection reset by peer'
+           Errno::ECONNREFUSED, # 'Connection refused - Connection refused'
+           Selenium::WebDriver::Error::WebDriverError => e # 'unable to obtain stable firefox connection in 60 seconds (127.0.0.1:7055)'
+                                                           # 'unable to bind to locking port 7054 within 45 seconds'
+      messages = [
+          /Connection reset by peer/, # SystemCallError
+          /Connection refused/, # Errno::ECONNREFUSED
+          /unable to obtain stable firefox connection/, # Selenium::WebDriver::Error::WebDriverError
+          /unable to bind to locking port/ # Selenium::WebDriver::Error::WebDriverError
+      ]
+
+      log "#{e.class}: #{e.message} \n#{e.backtrace.join("\n")}"
+
+      klass = messages.select{|msg| msg =~ e.message }.any? ? WebdriverError : SiteChanged
+      raise klass
+    end
+
+    # --- commands
+
+    def goto url
+      browser.goto url if url != browser.url
+    end
+
+    def exec script
+      browser.execute_script(script)
+    end
+
+  #  #####################################################################################################
+
+    # --------------------------------------------------------------------------------------------------------------------
+    def pull *args, &block
+      opts, xpaths = args.flatten.partition{|a| a.is_a?(Symbol) }
+      opt_mode  =   opts.delete(:exist?) ||  opts.delete(:present?) || :present? # default is :present?
+      opt_first = !!opts.delete(:first)  || !opts.delete(:all) # default is true, return 1th element
+      raise "Unknown options: '#{opts.inspect}'" if opts.any?
+
+      elements = xpaths.select do |xpath|
+        node_for(xpath).send(opt_mode) # detect element on the page by opt_mode
+      end.map do |xpath|
+        nodes_for(xpath) # get all elements
+      end.flatten.select do |node|
+        node.send(opt_mode) # select elements by mode
+      end
+
+      # flash result nodes
+      elements = elements.take(1) if opt_first
+      elements.each{|node| node.flash unless node.is_a?(Watir::Frame) }
+
+      first_element = elements.first
+
+      if block
+        raise SiteChanged, "Not found elements for xpath: #{xpaths.inspect}" if first_element.nil?
+        nodes_path << first_element.node_xpath
+        yield
+      else
+        if opt_first
+          first_element && first_element.to_subtype
+        else
+          elements.map{|element| element.to_subtype }
+        end
+      end
+    rescue Selenium::WebDriver::Error::StaleElementReferenceError,
+           Selenium::WebDriver::Error::ObsoleteElementError
+      sleep 1
+      retry
+    ensure
+      nodes_path.pop if block
+    end
+
+    # --------------------------------------------------------------------------------------------------------------------
+    # :first - get FIRST element of FIRST founded xpath, DEFAULT OPTION
+    # :all   - get ALL elements of FIRST founded xpath
+    def wait *xpaths, &block
+      #todo 3 raise_if_site_too_slow if respond_to?(:raise_if_site_too_slow)
+      common_wait *xpaths, &block
+    end
+
+    # --------------------------------------------------------------------------------------------------------------------
+    def common_wait *args, &block
+      browser.wait_until(@timeouts[:wait_timeout]) do
+        #todo 1 raise_if_firefox_error if respond_to?(:raise_if_firefox_error)
+        #todo 2 raise_if_service_unavailable if respond_to?(:raise_if_service_unavailable) # see class method :raise_service_unavailable_if
+
+        if args.any? || block
+          pull(args) || (block && instance_eval(&block))
+        else
+          return nil # running raise_if 1 times and exit if no args & block
+        end
+      end
+    rescue Selenium::WebDriver::Error::StaleElementReferenceError,
+           Selenium::WebDriver::Error::ObsoleteElementError
+      sleep 1
+      retry
+    rescue Watir::Wait::TimeoutError
+      raise SiteChanged
+    end
+    # --------------------------------------------------------------------------------------------------------------------
+
+    def exist? xpath
+      !!pull(xpath)
+    end
+
+    # --------------------------------------------------------------------------------------------------------------------
+
+    private
+
+    def nodes_path
+      @nodes_path ||= []
+    end
+
+    def node_for xpath
+      get_nodes(xpath, :get_all => false).first
+    end
+
+    def nodes_for xpath
+      get_nodes(xpath, :get_all => true)
+    end
+
+    def get_nodes xpath, opts
+      element_path = xpath_relative?(xpath) ? nodes_path.map { |node_xpath| element_name_for(node_xpath) } : nil
+      element_name = element_name_for(xpath, opts[:get_all])
+
+      eval_string = [element_path, element_name].flatten.compact.join('.')
+
+      log "#{File.basename(__FILE__)}:#{__LINE__}, eval_string: " + eval_string.inspect
+
+      elements = browser.instance_eval(eval_string)
+      elements = elements.to_a if elements.is_a? Watir::ElementCollection
+      elements = [elements].flatten
+
+      elements.map.with_index do |element, index|
+        class << element
+          attr_accessor :node_xpath
+        end
+
+        element.node_xpath = xpath + "[#{index + 1}]"
+        element
+      end
+    end
+
+    def element_name_for xpath, plural = false
+      [
+          xpath_with_frame?(xpath) ? 'frame' : 'element',
+          plural ? 's' : '',
+          '(:xpath, "' + xpath + '")'
+      ].join
+    end
+
+    def xpath_relative? xpath
+      xpath =~ /^\.\/.*/ # "./"
+    end
+
+    def xpath_with_frame? xpath
+      xpath =~ /^[\.]?\/[\/]?[i]?frame.*/ # "//frame", "//iframe", ".//frame", ".//iframe"
+    end
+
+  end
+end

data/lib/watir_crawler/browser.rb

@@ -0,0 +1,41 @@
+module WatirCrawler
+  class Browser < Abstract
+    def initialize(timeouts)
+      @browser = nil
+
+      @timeouts = {
+          :http_client_timeout => 120,
+          :implicit_wait => 0,
+          :page_load => 100,
+          :script_timeout => 10
+      }.merge(timeouts)
+    end
+
+    def profile
+      @browser_profile ||= Selenium::WebDriver::Firefox::Profile.new
+      yield @browser_profile if block_given?
+      @browser_profile
+    end
+
+    def browser
+      @browser
+    end
+
+    def start
+      return if @browser && @browser.exist?
+
+      # See http://code.google.com/p/selenium/wiki/RubyBindings#Timeouts
+      http_client = Selenium::WebDriver::Remote::Http::Default.new
+      http_client.timeout = @timeouts[:http_client_timeout]
+
+      @browser = Watir::Browser.new :firefox, :profile => profile, :http_client => http_client
+      @browser.driver.manage.timeouts.implicit_wait = @timeouts[:implicit_wait]
+      @browser.driver.manage.timeouts.page_load = @timeouts[:page_load]
+      @browser.driver.manage.timeouts.script_timeout = @timeouts[:script_timeout]
+    end
+
+    def stop
+      @browser.close if @browser
+    end
+  end
+end

data/lib/watir_crawler/errors.rb

@@ -0,0 +1,14 @@
+require 'nestegg'
+
+module WatirCrawler
+  class Error < StandardError
+    include Nestegg::NestingException
+  end
+
+  class WebdriverError < Error; end
+  class ServiceUnavailable < Error; end
+  class SiteTooSlow < Error; end
+  class SiteChanged < Error; end
+
+  class UnknownError < Error; end
+end

data/lib/watir_crawler/loggable.rb

@@ -0,0 +1,49 @@
+require 'logger'
+
+module WatirCrawler
+  module Loggable
+    module Logger
+      def logger
+        @@logger ||= ::Logger.new(STDOUT) # Ruby's logger by default
+      end
+
+      def logger=(logger)
+        @@logger = logger
+      end
+
+      def debug
+        @@debug ||= false
+      end
+
+      def debug=(debug)
+        @@debug = debug
+      end
+    end
+
+    extend Logger
+
+    module Log
+      def log msg = nil
+        if msg
+          Loggable.logger.debug(msg) if Loggable.debug
+        else
+          Loggable.logger
+        end
+      end
+    end
+
+    # for extending of module
+    def self.extended(base)
+      base.extend Logger
+      base.extend Log
+    end
+
+    # for including to class
+    def self.included(base)
+      base.extend Log
+      base.send :include, Log
+    end
+  end
+
+  extend Loggable
+end

data/lib/watir_crawler/version.rb

@@ -0,0 +1,3 @@
+module WatirCrawler
+  VERSION = '0.0.1'
+end

data/lib/watir_crawler/watir-webdriver.rb

@@ -0,0 +1,19 @@
+module Watir
+  class Element
+    def uri
+      url = self.attribute_value(:src) || self.attribute_value(:href)
+      URI.join(self.browser.url, url).to_s if url
+    end
+  end
+
+  class Image
+    def save_to_file filepath
+      File.open(filepath, 'wb') do |f|
+        f.write open(self.uri).read
+      end
+
+      filepath
+    end
+  end
+end
+

data/lib/watir_crawler.rb

@@ -0,0 +1,9 @@
+require 'watir-webdriver'
+require 'watir_crawler/watir-webdriver'
+
+require 'watir_crawler/version'
+require 'watir_crawler/errors'
+require 'watir_crawler/loggable'
+require 'watir_crawler/abstract'
+require 'watir_crawler/browser'
+require 'watir_crawler/base'

data/spec/base_spec.rb

@@ -0,0 +1,234 @@
+describe 'WatirCrawler::Base : ' do
+  before(:all) do
+    #log_file = File.join('/tmp', 'watir_crawler.log')
+    #puts "\nSaving debug log to: '#{log_file}'"
+    #WatirCrawler.logger = ::Logger.new(log_file)
+    #WatirCrawler.debug = true
+
+    @htmlfile = Htmlfile.new
+
+    @crawler = WatirCrawler::Base.new(:wait_timeout => 3)
+    @crawler.browser_start
+    @crawler.goto @htmlfile.url
+  end
+
+  after(:all) do
+    @crawler.browser_stop
+    @htmlfile.delete
+  end
+
+  # --------------------------------------------------------------------------------------------------------------------
+
+  it 'pull' do
+    # unknown option
+    begin
+      @crawler.pull(:unknown_option)
+    rescue => e
+      e.should be_a(RuntimeError)
+      e.message.should =~ /Unknown option/i
+    end
+
+    # not exist tag
+    @crawler.pull('//not_exist_tag').should be_nil
+    @crawler.pull(:first, '//not_exist_tag').should be_nil
+    @crawler.pull(:all, '//not_exist_tag').should =~ []
+
+    # exist tag, :present? mode by default
+    @crawler.pull('//div').should be_a(Watir::HTMLElement)
+    @crawler.pull(:first, '//div').should be_a(Watir::HTMLElement)
+
+    result = @crawler.pull(:all, '//div')
+    result.should be_a(Array)
+    result.size.should eq(3)
+
+    # exist tag, :present? mode
+    @crawler.pull(:present?, '//div').should be_a(Watir::HTMLElement)
+    @crawler.pull(:present?, :first, '//div').should be_a(Watir::HTMLElement)
+
+    result = @crawler.pull(:present?, :all, '//div')
+    result.should be_a(Array)
+    result.size.should eq(3)
+
+    # exist tag, :exist? mode
+    @crawler.pull(:exist?, '//div').should be_a(Watir::HTMLElement)
+    @crawler.pull(:exist?, :first, '//div').should be_a(Watir::HTMLElement)
+
+    result = @crawler.pull(:exist?, :all, '//div')
+    result.should be_a(Array)
+    result.size.should eq(4)
+
+    # hidden tag, :present? mode
+    @crawler.pull(:present?, "//div[@id='4']").should be_nil
+
+    result = @crawler.pull(:present?, :all, "//div[@id='4']")
+    result.should be_a(Array)
+    result.should be_empty
+
+    # hidden tag, :exist? mode
+    @crawler.pull(:exist?, "//div[@id='4']").should be_a(Watir::HTMLElement)
+
+    result = @crawler.pull(:exist?, :all, "//div[@id='4']")
+    result.should be_a(Array)
+    result.size.should eq(1)
+  end
+
+  # --------------------------------------------------------------------------------------------------------------------
+
+  it 'wait' do
+    @crawler.wait.should be_nil
+
+    # wait without params but with block
+    @crawler.wait{ true }.should be_true
+
+    begin
+      @crawler.wait{ false }
+    rescue => e
+      e.should be_a(WatirCrawler::SiteChanged)
+    end
+
+    # unknown option
+    begin
+      @crawler.wait(:unknown_option)
+    rescue => e
+      e.should be_a(RuntimeError)
+      e.message.should =~ /Unknown option/i
+    end
+
+    # not exist tag
+    begin
+      @crawler.wait('//not_exist_tag')
+    rescue => e
+      e.should be_a(WatirCrawler::SiteChanged)
+    end
+
+    # exist tag, :present? mode by default
+    @crawler.wait('//div').should be_a(Watir::HTMLElement)
+
+    # exist tag, :present? mode
+    @crawler.wait(:present?, '//div').should be_a(Watir::HTMLElement)
+
+    # hidden tag, :present? mode
+    begin
+      @crawler.wait(:present?, "//div[@id='4']")
+    rescue => e
+      e.should be_a(WatirCrawler::SiteChanged)
+    end
+
+    # exist tag, :exist? mode
+    @crawler.wait(:exist?, "//div[@id='1']").should be_a(Watir::HTMLElement)
+
+    # hidden tag, :exist? mode
+    @crawler.wait(:exist?, "//div[@id='4']").should be_a(Watir::HTMLElement)
+  end
+
+  # --------------------------------------------------------------------------------------------------------------------
+
+  it 'exist?' do
+    @crawler.exist?("//div[@id='1']").should be_true
+    @crawler.exist?('//not_exist_tag').should be_false
+  end
+
+  # --------------------------------------------------------------------------------------------------------------------
+
+  it 'pull for nested elements' do
+    @crawler.pull("//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+    @crawler.pull(".//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+
+    result = @crawler.pull(:all, '//span')
+    result.should be_a(Array)
+    result.size.should eq(2)
+
+    result = @crawler.pull(:all, './/span')
+    result.should be_a(Array)
+    result.size.should eq(2)
+
+    @crawler.pull("//p[@id='level_1']") do
+      @crawler.pull("//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+      @crawler.pull(".//span[text()='Test span 1']").should be_nil
+
+      result = @crawler.pull(:all, '//span')
+      result.should be_a(Array)
+      result.size.should eq(2)
+
+      result = @crawler.pull(:all, './/span')
+      result.should be_a(Array)
+      result.should be_empty
+
+      @crawler.pull("//p[@id='level_2']") do
+        @crawler.pull("//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+        @crawler.pull(".//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+
+        result = @crawler.pull(:all, '//span')
+        result.should be_a(Array)
+        result.size.should eq(2)
+
+        result = @crawler.pull(:all, './/span')
+        result.should be_a(Array)
+        result.size.should eq(2)
+      end
+    end
+
+  end
+
+  it 'wait for nested elements' do
+    @crawler.wait("//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+    @crawler.wait(".//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+
+    result = @crawler.wait(:all, '//span')
+    result.should be_a(Array)
+    result.size.should eq(2)
+
+    result = @crawler.wait(:all, './/span')
+    result.should be_a(Array)
+    result.size.should eq(2)
+
+    @crawler.pull("//p[@id='level_1']") do
+      @crawler.wait("//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+
+      begin
+        @crawler.wait(".//span[text()='Test span 1']")
+      rescue => e
+        e.should be_a(WatirCrawler::SiteChanged)
+      end
+
+      result = @crawler.wait(:all, '//span')
+      result.should be_a(Array)
+      result.size.should eq(2)
+
+      begin
+        @crawler.wait('.//span')
+      rescue => e
+        e.should be_a(WatirCrawler::SiteChanged)
+      end
+
+      @crawler.pull("//p[@id='level_2']") do
+        @crawler.wait("//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+        @crawler.wait(".//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+
+        result = @crawler.wait(:all, '//span')
+        result.should be_a(Array)
+        result.size.should eq(2)
+
+        result = @crawler.wait(:all, './/span')
+        result.should be_a(Array)
+        result.size.should eq(2)
+      end
+    end
+  end
+
+  it 'frame', :frame => true do
+    @crawler.pull('//iframe') do
+      @crawler.pull(".//span[text()='Test span 11']").should be_a(Watir::HTMLElement)
+
+      @crawler.pull(".//span[text()='Test span 21']").should be_nil
+      @crawler.pull('.//iframe') do
+        @crawler.pull("//span[text()='Test span 1']").should be_a(Watir::HTMLElement)
+        @crawler.pull("//span[text()='Test span 11']").should be_nil
+        @crawler.pull("//span[text()='Test span 21']").should be_nil
+        @crawler.pull(".//span[text()='Test span 11']").should be_nil
+        @crawler.pull(".//span[text()='Test span 21']").should be_a(Watir::HTMLElement)
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,189 @@
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+end
+
+require 'watir_crawler'
+WORKING_DIRECTORY = '/tmp'
+$debug = true
+# ----------------------------------------------------------------------------------------------------------------------
+
+require 'tempfile'
+
+class Htmlfile
+  def initialize
+    @frameset = Tempfile.new('frameset.html')
+    @frameset.write(frameset_content)
+    @frameset.close
+
+    @frame2 = Tempfile.new('frame2.html')
+    @frame2.write(frame2_content)
+    @frame2.close
+
+    @frame1 = Tempfile.new('frame1.html')
+    @frame1.write(frame1_content)
+    @frame1.close
+
+    @file = Tempfile.new('test.html')
+    @file.write(content)
+    @file.close
+    @file
+  end
+
+  def content
+    # todo     #{frameset}
+    cnt = <<-EOF
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Test</title>
+</head>
+<body>
+    <div id="1">div 1</div>
+    <div id="2">div 2</div>
+    <div id="3">div 3</div>
+    <div id="4" style="display:none;">div 4</div>
+    <p id="level_1">
+      <p id="level_2">
+        <span>Test span 1</span>
+        <span>Test span 2</span>
+      </p>
+    </p>
+    #{frame1}
+</body>
+</html>
+    EOF
+
+    cnt
+  end
+
+  def frame1
+    cnt = <<-EOF
+    <iframe id="loginframe1" src="#{file_url @frame1}" style=""></iframe>
+    EOF
+
+    cnt
+  end
+
+  def frame2
+    cnt = <<-EOF
+    <iframe id="loginframe2" src="#{file_url @frame2}" style=""></iframe>
+    EOF
+
+    cnt
+  end
+
+  def frame1_content
+    cnt = <<-EOF
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Frame 1</title>
+</head>
+<body>
+    <h2>Frame 1</h2>
+    <div id="11">div 1</div>
+    <div id="12">div 2</div>
+    <div id="13">div 3</div>
+    <div id="14" style="display:none;">div 4</div>
+    <p id="level_11">
+      <p id="level_12">
+        <span>Test span 11</span>
+        <span>Test span 12</span>
+      </p>
+    </p>
+#{frame2}
+</body>
+</html>
+    EOF
+
+    cnt
+  end
+
+  def frame2_content
+    cnt = <<-EOF
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Frame 2</title>
+</head>
+<body>
+    <h2>Frame 2</h2>
+    <div id="21">div 1</div>
+    <div id="22">div 2</div>
+    <div id="23">div 3</div>
+    <div id="24" style="display:none;">div 4</div>
+    <p id="level_21">
+      <p id="level_22">
+        <span>Test span 21</span>
+        <span>Test span 22</span>
+      </p>
+    </p>
+</body>
+</html>
+    EOF
+
+    cnt
+  end
+
+  def frameset
+    # TODO
+    cnt = <<-EOF
+<frameset rows="80,*" cols="*">
+  <frame src="#{file_url @frameset}" name="topFrame">
+  <frameset cols="80,*">
+    <frame src="#{file_url @frameset}" name="leftFrame">
+    <frame src="#{file_url @frameset}" name="mainFrame">
+  </frameset>
+</frameset>
+    EOF
+
+    cnt
+  end
+
+  def frameset_content
+    cnt = <<-EOF
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Frameset Frame</title>
+</head>
+<body>
+    <h2>Frameset Frame</h2>
+    <div id="111">div 1</div>
+    <div id="112">div 2</div>
+    <div id="113">div 3</div>
+    <div id="114" style="display:none;">div 4</div>
+    <p id="level_111">
+      <p id="level_112">
+        <span>Test span 111</span>
+        <span>Test span 112</span>
+      </p>
+    </p>
+</body>
+</html>
+    EOF
+
+    cnt
+  end
+
+  def url
+    file_url @file
+  end
+
+  def file_url file
+    "file://#{file.path}"
+  end
+
+  def delete
+    [
+      @frameset,
+      @frame1,
+      @frame2,
+      @file
+    ].each{|file| file.unlink }
+  end
+end
+
+

data/watir_crawler.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'watir_crawler/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'watir_crawler'
+  spec.version       = WatirCrawler::VERSION
+  spec.authors       = ['Dmitry T']
+  spec.email         = ['atlancer@gmail.com']
+  spec.description   = %q{A watir based web crawler}
+  spec.summary       = %q{A watir based web crawler}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+  spec.add_dependency 'nestegg'
+  spec.add_dependency 'settingslogic'
+  spec.add_dependency 'watir-webdriver', '~> 0.6.4'
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rspec', '~> 2.6'
+  spec.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: watir_crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Dmitry T
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nestegg
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: settingslogic
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: watir-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.6.4
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A watir based web crawler
+email:
+- atlancer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/example.rb
+- lib/watir_crawler.rb
+- lib/watir_crawler/abstract.rb
+- lib/watir_crawler/base.rb
+- lib/watir_crawler/browser.rb
+- lib/watir_crawler/errors.rb
+- lib/watir_crawler/loggable.rb
+- lib/watir_crawler/version.rb
+- lib/watir_crawler/watir-webdriver.rb
+- spec/base_spec.rb
+- spec/spec_helper.rb
+- watir_crawler.gemspec
+homepage: ''
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -715382381
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -715382381
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A watir based web crawler
+test_files:
+- spec/base_spec.rb
+- spec/spec_helper.rb