crabfarm 0.5.3 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89af60fe404e36fbc112ab5325ad3bbc618e25f1
-  data.tar.gz: 2930de41a6c3c9725533d1a9f84fc6d0b89b0ab9
+  metadata.gz: 425cdb9cbc7e43b16e7ab9a7d8ef5f187f95d07d
+  data.tar.gz: a8f7360685f9febdb8e9076586ab1e856f6c0323
 SHA512:
-  metadata.gz: 31c2d828b6c7351f2efa1a7b79ba0762a21770b12f66560cc310a7ce1e0c91a574adfc1a3c8193937fed2d63a7f7d35d891c4b4c419e9b87a8e73d192c61d167
-  data.tar.gz: de65bef60361d0c7fa3365d8c1e05a82ed702bfbce0224c54f2ea5d94fd4a32d572e775d9321b388b0b46460fbba5c691a7c472f19ca213f45d64f1b4cf4f8ff
+  metadata.gz: 203b29582a08881d693923620e7cb601433be93edf3551f8daebc414fff2aca12e3bfcc34b3e81b0d045921cfbb5ad74a47e7d3858925c31bf300e995107b33d
+  data.tar.gz: 8b2215150d4976599385b2da16b1b5cb89f169ef6b0ba41b3e719cd56fc8d68b2c061d015cece7523ad72ec95378c9c49ad0faa292259a5f7645c17cc4af273d

data/lib/crabfarm/adapters/driver_wrapper/pincers.rb

@@ -0,0 +1,17 @@
+class Pincers::Core::SearchContext
+  def webdriver_elements
+    elements
+  end
+end
+
+module Crabfarm
+  module Adapters
+    module DriverWrapper
+      class Pincers
+        def self.wrap(_driver)
+          ::Pincers.for_webdriver _driver
+        end
+      end
+    end
+  end
+end

data/lib/crabfarm/adapters/parser/pincers.rb

@@ -0,0 +1,23 @@
+module Crabfarm
+  module Adapters
+    module Parser
+      class Pincers
+        def self.format
+          'html'
+        end
+
+        def self.parse(_raw)
+          ::Pincers.for_nokogiri ::Nokogiri::HTML _raw
+        end
+
+        def self.preprocess_parsing_target(_target)
+          if _target.respond_to? :to_html
+            _target.to_html
+          else
+            _target
+          end
+        end
+      end
+    end
+  end
+end

data/lib/crabfarm/base_navigator.rb

@@ -21,6 +21,12 @@ module Crabfarm
       @params = _params
     end
 
+    def navigate(_name, _params={})
+      TransitionService.transition(@context, _name, params.merge(_params)).navigator
+    end
+
+    alias :nav :navigate
+
     def browser(_name=nil)
       @context.pool.driver(_name)
     end

data/lib/crabfarm/configuration.rb

@@ -15,10 +15,10 @@ module Crabfarm
       [:webdriver_host, :string, 'Remote host, only available in driver: remote'],
       [:webdriver_port, :integer, 'Remote port, only available in driver: remote'],
       [:webdriver_capabilities, :mixed, 'Driver capabilities, depends on selected driver.'],
-      [:webdriver_remote_timeout, :float, 'Request timeout in seconds, only available for remote or phatomjs driver.'],
+      [:webdriver_remote_timeout, :float, 'Request timeout in seconds, only available for remote or phantomjs driver.'],
       [:webdriver_window_width, :integer, 'Initial browser window width.'],
       [:webdriver_window_height, :integer, 'Initial browser window height.'],
-      [:webdriver_dsl, :string, 'Webdriver wrapper to use, built in options are watir and surfer'],
+      [:webdriver_dsl, :string, 'Webdriver wrapper to use, built in options are pincers and watir'],
 
       # Phantom launcher configuration
       [:phantom_load_images, :boolean, 'Phantomjs image loading, only for phantomjs driver.'],
@@ -53,7 +53,7 @@ module Crabfarm
     def reset
       @values = {
         browser: 'phantomjs',
-        parser: :nokogiri,
+        parser: :pincers,
         driver_factory: nil,
         log_path: nil,
         proxy: nil,
@@ -63,7 +63,7 @@ module Crabfarm
         webdriver_remote_timeout: 120,
         webdriver_window_width: 1280,
         webdriver_window_height: 800,
-        webdriver_dsl: :watir,
+        webdriver_dsl: :pincers,
         phantom_load_images: false,
         phantom_ssl: 'any',
         phantom_bin_path: 'phantomjs',

data/lib/crabfarm/modes/publisher.rb

@@ -88,7 +88,7 @@ module Crabfarm
       end
 
       def validate_remote(_url)
-        return true if /^\w+\/\w+$/i === _url
+        return true if /^[\w\-]+\/[\w\-]+$/i === _url
         puts "Invalid remote syntax: #{_url}".color :red
         return false
       end

data/lib/crabfarm/templates/Crabfile.erb

@@ -3,8 +3,8 @@
 set_browser :phantomjs
 
 # The default parser engine for reducers that do not specify one.
-# Available options are :nokogiri and :pdf_parser. :pdf_parser requires an additional gem to be added to Gemfile
-set_parser :nokogiri
+# Available options are :pincers, :nokogiri and :pdf_parser. :pdf_parser requires an additional gem to be added to Gemfile
+set_parser :pincers
 
 # The path where every crawler log is stored.
 set_log_path 'logs'
@@ -17,9 +17,9 @@ set_log_path 'logs'
 
 # The following parameters only apply if using a webdriver based driver
 
-# Selects the webdriver wrapper library to be used, options are :surfer, :watir and :capybara.
+# Selects the webdriver wrapper library to be used, options are :pincers, :watir and :capybara.
 # Both watir and capybara require an additional gem to be added to Gemfile
-set_webdriver_dsl :watir
+set_webdriver_dsl :pincers
 
 # Set the selected webdriver capabilities (check the driver documentation for more details)
 # set_webdriver_capabilities

data/lib/crabfarm/templates/Gemfile.erb

@@ -1,16 +1,14 @@
 source 'https://rubygems.org'
 
 gem "crabfarm", '<%= version %>'
+gem "pincers", '~> 0.2.0'
 
-# Comment this is not using the nokogiri default HTML parser
+# Comment this if not using a nokogiri based parser or browser
 gem 'nokogiri', "~> 1.6.6"
 
-# Comment this is not using a selenium webdriver based driver
+# Comment this if not using a selenium webdriver based driver
 gem "selenium-webdriver", "~> 2.45"
 
-# Comment this if using other webdriver dsl
-gem "watir-webdriver"
-
 group :test do
   gem "rspec", "~> 3.2.0"
   gem "rspec-nc"

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.5.3"
+  VERSION = "0.6.0"
 end

data/lib/crabfarm.rb

@@ -101,11 +101,12 @@ module Crabfarm
     register :browser, :noop, 'Crabfarm::Adapters::Browser::Noop'
 
     # bundled webdriver dsl adapters
-    register :webdriver_dsl, :surfer, 'Crabfarm::Adapters::DriverWrapper::Surfer'
+    register :webdriver_dsl, :pincers, 'Crabfarm::Adapters::DriverWrapper::Pincers', dependencies: ['pincers']
     register :webdriver_dsl, :watir, 'Crabfarm::Adapters::DriverWrapper::Watir', dependencies: ['watir-webdriver']
     register :webdriver_dsl, :capybara, 'Crabfarm::Adapters::DriverWrapper::Capybara', dependencies: ['capybara']
 
     # bundled parsers dsl adapters
+    register :parser, :pincers, 'Crabfarm::Adapters::Parser::Pincers', dependencies: ['pincers', 'nokogiri']
     register :parser, :nokogiri, 'Crabfarm::Adapters::Parser::Nokogiri', dependencies: ['nokogiri']
     register :parser, :pdf_reader, 'Crabfarm::Adapters::Parser::PdfReader', dependencies: ['pdf-reader']
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.6.0
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-08 00:00:00.000000000 Z
+date: 2015-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -184,6 +184,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.6.6
+- !ruby/object:Gem::Dependency
+  name: pincers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -409,10 +423,11 @@ files:
 - lib/crabfarm/adapters/browser/phantom_js.rb
 - lib/crabfarm/adapters/browser/remote_webdriver.rb
 - lib/crabfarm/adapters/driver_wrapper/capybara.rb
-- lib/crabfarm/adapters/driver_wrapper/surfer.rb
+- lib/crabfarm/adapters/driver_wrapper/pincers.rb
 - lib/crabfarm/adapters/driver_wrapper/watir.rb
 - lib/crabfarm/adapters/parser/nokogiri.rb
 - lib/crabfarm/adapters/parser/pdf_reader.rb
+- lib/crabfarm/adapters/parser/pincers.rb
 - lib/crabfarm/assertion/context.rb
 - lib/crabfarm/assertion/fields.rb
 - lib/crabfarm/assertion/parsers.rb
@@ -428,9 +443,6 @@ files:
 - lib/crabfarm/crabtrap_context.rb
 - lib/crabfarm/crabtrap_runner.rb
 - lib/crabfarm/driver_pool.rb
-- lib/crabfarm/dsl/surfer/search_context.rb
-- lib/crabfarm/dsl/surfer/surf_context.rb
-- lib/crabfarm/dsl/surfer.rb
 - lib/crabfarm/engines/async_state_manager.rb
 - lib/crabfarm/engines/sync_state_manager.rb
 - lib/crabfarm/errors.rb
@@ -528,3 +540,4 @@ signing_key:
 specification_version: 4
 summary: Crabfarm crawler creation framework
 test_files: []
+has_rdoc: 

data/lib/crabfarm/adapters/driver_wrapper/surfer.rb

@@ -1,13 +0,0 @@
-require 'crabfarm/dsl/surfer'
-
-module Crabfarm
-  module Adapters
-    module DriverWrapper
-      class Surfer
-        def self.wrap(_driver)
-          Crabfarm::Dsl::Surfer::SurfContext.new _driver
-        end
-      end
-    end
-  end
-end

data/lib/crabfarm/dsl/surfer/search_context.rb

@@ -1,152 +0,0 @@
-module Crabfarm
-  module Dsl
-    module Surfer
-      class SearchContext
-        include Enumerable
-        extend Forwardable
-
-        TIMEOUT = 10.0 # Default timeout for waiting operations
-
-        attr_accessor :elements, :parent
-
-        def_delegators :elements, :length, :count, :empty?
-
-        def initialize(_elements, _parent)
-          @elements = _elements
-          @parent = _parent
-        end
-
-        def webdriver_elements
-          @elements
-        end
-
-        def root
-          @parent.root
-        end
-
-        def each
-          elements.each { |el| yield child_context [el] }
-        end
-
-        def [](*args)
-          if args[0].is_a? String or args[0].is_a? Symbol
-            attribute args[0]
-          else
-            child_context Array(elements.send(:[],*args))
-          end
-        end
-
-        def first
-          if elements.first.nil? then nil else child_context [elements.first] end
-        end
-
-        def last
-          if elements.last.nil? then nil else child_context [elements.last] end
-        end
-
-        def element!
-          raise EmptySetError.new("This set is empty", self) if empty?
-          elements.first
-        end
-
-        def classes
-          wrap_errors { (element!['class'] || '').split(' ') }
-        end
-
-        def search(_selector=nil, _options={})
-          _options[:css] = _selector if _selector
-
-          wait_mode = _options.delete :wait
-          if wait_mode
-
-            # retrieve timeout
-            timeout = _options.delete :timeout
-            timeout = TIMEOUT if timeout.nil?
-
-            # use a selenium timeout
-            wrap_errors do
-              wait = Selenium::WebDriver::Wait.new(timeout: timeout)
-              wait.until do
-                new_elements = search_elements _options
-
-                # test wait condition
-                ok = case wait_mode
-                when :present then (new_elements.length > 0)
-                when :visible then (new_elements.length > 0 and new_elements.first.displayed?)
-                when :enabled then (new_elements.length > 0 and new_elements.first.displayed? and new_elements.first.enabled?)
-                when :not_present then (new_elements.length == 0)
-                when :not_visible then (not new_elements.any? { |e| e.displayed? })
-                else
-                  raise SetupError.new "Invalid wait mode '#{wait_mode}'"
-                end
-
-                child_context new_elements if ok
-              end
-            end
-          else
-            child_context search_elements(_options)
-          end
-        end
-
-        def fill(_value)
-          wrap_errors do
-            element!.clear
-            element!.send_keys _value
-          end
-        end
-
-        def to_html
-          elements.map { |e| e['outerHTML'] }.join
-        end
-
-        # Any methods missing are forwarded to the main element (first).
-        def method_missing(_method, *_args, &_block)
-          wrap_errors do
-            m = /^(.*)_all$/.match _method.to_s
-            if m then
-              return [] if empty?
-              elements.map { |e| e.send(m[1], *_args, &_block) }
-            else
-              element!.send(_method, *_args, &_block)
-            end
-          end
-        end
-
-        def respond_to?(_method, _include_all=false)
-          return true if super
-          m = /^.*_all$/.match _method.to_s
-          if m then
-            return true if empty?
-            elements.first.respond_to? m[1], _include_all
-          else
-            return true if empty?
-            elements.first.respond_to? _method, _include_all
-          end
-        end
-
-      private
-
-        def child_context(_elements)
-          SearchContext.new _elements, self
-        end
-
-        def wrap_errors
-          begin
-            yield
-          rescue Selenium::WebDriver::Error::WebDriverError => e
-            raise WebdriverError.new e, self
-          end
-        end
-
-        def search_elements(_options)
-          wrap_errors do
-            elements.inject([]) do |r, element|
-              r + element.find_elements(_options)
-            end
-          end
-        end
-
-      end
-    end
-  end
-end

data/lib/crabfarm/dsl/surfer/surf_context.rb

@@ -1,42 +0,0 @@
-module Crabfarm
-  module Dsl
-    module Surfer
-      class SurfContext < SearchContext
-
-        attr_reader :driver
-
-        def_delegators 'driver.navigate', :back, :forward, :refresh
-
-        def initialize(_driver)
-          super nil, self
-          @driver = _driver
-        end
-
-        def root
-          self
-        end
-
-        def elements
-          [driver]
-        end
-
-        def to_html
-          driver.page_source
-        end
-
-        def current_uri
-          URI.parse driver.current_url
-        end
-
-        def cookies
-          driver.manage.all_cookies
-        end
-
-        def goto(_url, _params=nil)
-          _url += "?#{_params.to_query}" if _params
-          driver.get(_url)
-        end
-      end
-    end
-  end
-end

data/lib/crabfarm/dsl/surfer.rb

@@ -1,22 +0,0 @@
-require 'crabfarm/dsl/surfer/search_context'
-require 'crabfarm/dsl/surfer/surf_context'
-
-module Crabfarm
-  module Dsl
-    module Surfer
-
-      class Error < StandardError
-        attr_reader :source
-
-        def initialize(_message, _ctx)
-          super _message
-          @ctx = _ctx
-          @source = _ctx.root.page_source rescue nil # cache page source for future reference
-        end
-      end
-
-      class EmptySetError < Error; end
-      class WebdriverError < Error; end
-    end
-  end
-end