crabfarm 0.2.5 → 0.3.0

data/lib/crabfarm/templates/Gemfile.erb

@@ -2,6 +2,12 @@ source 'https://rubygems.org'
 
 gem "crabfarm", '<%= version %>'
 
+# Comment this is not using the nokogiri default HTML parser
+gem 'nokogiri', "~> 1.6.6"
+
+# Comment this is not using a selenium webdriver based driver
+gem "selenium-webdriver", "~> 2.45"
+
 group :test do
   gem "rspec-nc"
 end

data/lib/crabfarm/templates/navigator.rb.erb

@@ -0,0 +1,20 @@
+class <%= navigator_class %> < Crabfarm::BaseNavigator
+
+  def run
+  	<% if navigator_url.nil? %>
+  	# replace the following by your navigation code:
+	raise NotImplementedError.new 'You must provide some navigation code for <%= navigator_class %>'
+  	<% else %>
+  	browser.goto '<%= navigator_url %>'
+  	<% end %>
+
+  	# Call the homonymous reducer over the entire document and output it by default
+  	# You can call other reducers by using the `reduce` method, like this:
+  	#
+  	#     reduce browser.search('td').first, using: MyOtherReducer
+  	#
+  	reduce_with_defaults
+  end
+
+end
+

data/lib/crabfarm/templates/{state_spec.rb.erb → navigator_spec.rb.erb}

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe <%= state_class %> do
+describe <%= navigator_class %> do
 
   pending "should ensure output has the right structure"
 

data/lib/crabfarm/templates/{parser.rb.erb → reducer.rb.erb}

@@ -1,9 +1,9 @@
-class <%= parser_class %> < Crabfarm::BaseParser
+class <%= reducer_class %> < Crabfarm::BaseReducer
 
-  def parse
-    # You can replace the following line after running the owner state specs once.
+  def run
+    # You can replace the following line after running the owner navigator specs once.
     # Take a look at the 'Testing' section of the README.md for more information!
-    take_snapshot
+    take_snapshot_and_fail
   end
 
 end

data/lib/crabfarm/templates/{parser_spec.rb.erb → reducer_spec.rb.erb}

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe <%= parser_class %> do
+describe <%= reducer_class %> do
 
   pending "should extract values from snapshot"
 

data/lib/crabfarm/templates/struct.rb.erb

@@ -0,0 +1,12 @@
+class <%= struct_class %> < Crabfarm::BaseStruct
+
+  # add some fields, some examples:
+  #
+  # has_string :a_string
+  # has_integer :an_integer_greater_than_8, greated_than: 8
+  # has_array :an_array
+  # has_field :misc_field
+  #
+
+end
+

data/lib/crabfarm/transition_service.rb

@@ -1,20 +1,33 @@
 module Crabfarm
   class TransitionService
 
-    def self.apply_state(_context, _name, _params={})
-      state_class = if _name.is_a? String or _name.is_a? Symbol
+    def self.transition(_context, _name, _params={})
+      self.new(_context).transition(_name, _params)
+    end
+
+    attr_reader :document, :navigator
+
+    def initialize(_context)
+      @context = _context
+    end
+
+    def transition(_name, _params={})
+      navigator_class = if _name.is_a? String or _name.is_a? Symbol
         load_class_from_uri _name
       else _name end
 
-      _context.prepare
-      state = state_class.new _context, _params
-      state.crawl
-      state
+      @context.prepare
+      @navigator = navigator_class.new @context, _params
+
+      @document = @navigator.run
+      @document = @document.as_json if @document.respond_to? :as_json
+
+      self
     end
 
   private
 
-    def self.load_class_from_uri(_uri)
+    def load_class_from_uri(_uri)
       class_name = Utils::Naming.decode_crabfarm_uri _uri
       class_name.constantize
     end

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.2.5"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,43 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-27 00:00:00.000000000 Z
+date: 2015-04-15 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: selenium-webdriver
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.45'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.45'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.6.6
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.6.6
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
@@ -156,6 +128,34 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.5.5
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.45'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.45'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.6
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -374,45 +374,46 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/crabfarm/adapters/browser/capybara.rb
-- lib/crabfarm/adapters/browser/surfer.rb
-- lib/crabfarm/adapters/browser/watir.rb
-- lib/crabfarm/adapters/output/hash.rb
-- lib/crabfarm/adapters/output/jbuilder.rb
-- lib/crabfarm/adapters/output/ostruct.rb
+- lib/crabfarm/adapters/browser/abstract_webdriver.rb
+- lib/crabfarm/adapters/browser/chrome.rb
+- lib/crabfarm/adapters/browser/firefox.rb
+- lib/crabfarm/adapters/browser/noop.rb
+- lib/crabfarm/adapters/browser/phantom_js.rb
+- lib/crabfarm/adapters/browser/remote_webdriver.rb
+- lib/crabfarm/adapters/driver_wrapper/capybara.rb
+- lib/crabfarm/adapters/driver_wrapper/surfer.rb
+- lib/crabfarm/adapters/driver_wrapper/watir.rb
 - lib/crabfarm/adapters/parser/nokogiri.rb
 - lib/crabfarm/adapters/parser/pdf_reader.rb
 - lib/crabfarm/assertion/context.rb
+- lib/crabfarm/assertion/fields.rb
 - lib/crabfarm/assertion/parsers.rb
 - lib/crabfarm/assertion/validations.rb
 - lib/crabfarm/assertion/wrapper.rb
-- lib/crabfarm/base_parser.rb
-- lib/crabfarm/base_state.rb
+- lib/crabfarm/base_navigator.rb
+- lib/crabfarm/base_reducer.rb
+- lib/crabfarm/base_struct.rb
 - lib/crabfarm/cli.rb
 - lib/crabfarm/configuration.rb
 - lib/crabfarm/context.rb
 - lib/crabfarm/context_factory.rb
 - lib/crabfarm/crabtrap_context.rb
 - lib/crabfarm/crabtrap_runner.rb
-- lib/crabfarm/default_driver_factory.rb
-- lib/crabfarm/driver_bucket.rb
-- lib/crabfarm/driver_bucket_pool.rb
+- lib/crabfarm/driver_pool.rb
 - lib/crabfarm/dsl/surfer/search_context.rb
 - lib/crabfarm/dsl/surfer/surf_context.rb
 - lib/crabfarm/dsl/surfer.rb
 - lib/crabfarm/engines/async_state_manager.rb
 - lib/crabfarm/engines/sync_state_manager.rb
 - lib/crabfarm/errors.rb
-- lib/crabfarm/forked_state.rb
+- lib/crabfarm/forked_navigator.rb
 - lib/crabfarm/global_state.rb
 - lib/crabfarm/http_client.rb
-- lib/crabfarm/mocks/noop_driver.rb
 - lib/crabfarm/modes/console.rb
 - lib/crabfarm/modes/generator.rb
 - lib/crabfarm/modes/publisher.rb
 - lib/crabfarm/modes/recorder.rb
 - lib/crabfarm/modes/server.rb
-- lib/crabfarm/phantom_driver_factory.rb
 - lib/crabfarm/phantom_runner.rb
 - lib/crabfarm/rspec.rb
 - lib/crabfarm/state_store.rb
@@ -427,11 +428,12 @@ files:
 - lib/crabfarm/templates/dot_gitkeep.erb
 - lib/crabfarm/templates/dot_rspec.erb
 - lib/crabfarm/templates/Gemfile.erb
-- lib/crabfarm/templates/parser.rb.erb
-- lib/crabfarm/templates/parser_spec.rb.erb
+- lib/crabfarm/templates/navigator.rb.erb
+- lib/crabfarm/templates/navigator_spec.rb.erb
+- lib/crabfarm/templates/reducer.rb.erb
+- lib/crabfarm/templates/reducer_spec.rb.erb
 - lib/crabfarm/templates/spec_helper.rb.erb
-- lib/crabfarm/templates/state.rb.erb
-- lib/crabfarm/templates/state_spec.rb.erb
+- lib/crabfarm/templates/struct.rb.erb
 - lib/crabfarm/transition_service.rb
 - lib/crabfarm/utils/naming.rb
 - lib/crabfarm/utils/port_discovery.rb

data/lib/crabfarm/adapters/browser/capybara.rb

@@ -1,7 +0,0 @@
-module Crabfarm
-  class CapybaraBrowserDsl
-    def self.wrap(_bucket)
-      raise NotImplementedError.new "Capybara adapter is incompleted"
-    end
-  end
-end

data/lib/crabfarm/adapters/browser/surfer.rb

@@ -1,9 +0,0 @@
-require 'crabfarm/dsl/surfer'
-
-module Crabfarm
-  class SurferBrowserDsl
-    def self.wrap(_bucket)
-      Crabfarm::Dsl::Surfer::SurfContext.new _bucket
-    end
-  end
-end

data/lib/crabfarm/adapters/output/hash.rb

@@ -1,11 +0,0 @@
-module Crabfarm
-  class HashOutputBuilder
-    def self.prepare
-      Hash.new
-    end
-
-    def self.serialize(_output)
-      _output
-    end
-  end
-end

data/lib/crabfarm/adapters/output/jbuilder.rb

@@ -1,11 +0,0 @@
-module Crabfarm
-  class JbuilderOutputBuilder
-    def self.prepare
-      Jbuilder.new
-    end
-
-    def self.serialize(_output)
-      _output.attributes!
-    end
-  end
-end

data/lib/crabfarm/adapters/output/ostruct.rb

@@ -1,14 +0,0 @@
-require 'ostruct'
-
-module Crabfarm
-  class OStructOutputBuilder
-    def self.prepare
-      # TODO: maybe wrap open struct in a class that automatically generate other openstruct when nested properties are accessed
-      OpenStruct.new
-    end
-
-    def self.serialize(_output)
-      _output.to_h
-    end
-  end
-end

data/lib/crabfarm/base_parser.rb

@@ -1,59 +0,0 @@
-require "crabfarm/assertion/context"
-
-module Crabfarm
-  class BaseParser < Delegator
-    include Assertion::Context
-
-    attr_reader :params, :document
-
-    def self.parser_engine(_engine=nil)
-      @engine_name = _engine
-    end
-
-    def self.engine
-      @engine ||= Strategies.load(:parser_engine, @engine_name || Crabfarm.config.parser_engine)
-    end
-
-    def self.snapshot_path(_name=nil)
-      _name = self.to_s.underscore if _name.nil?
-      File.join(GlobalState.snapshots_path, _name + '.' + engine.format)
-    end
-
-    def engine
-      self.class.engine
-    end
-
-    def initialize(_target, _params)
-      @parsed_data = engine.preprocess_parsing_target _target
-      @document = engine.parse @parsed_data
-      @params = _params
-
-      super @document
-    end
-
-    def parse
-      raise NotImplementedError.new
-    end
-
-    def take_snapshot(_name=nil)
-      file_path = self.class.snapshot_path _name
-
-      raise ArgumentError.new "Snapshot already exists '#{file_path}', make sure to implement the #{self.class.to_s} parse method." if File.exist? file_path
-
-      dir_path = file_path.split(File::SEPARATOR)[0...-1]
-      FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
-
-      File.write file_path, @parsed_data
-      nil
-    end
-
-    def __getobj__
-      @document
-    end
-
-    def __setobj__(obj)
-      @document = obj
-    end
-
-  end
-end

data/lib/crabfarm/base_state.rb

@@ -1,112 +0,0 @@
-require 'thwait'
-require 'crabfarm/forked_state'
-require "crabfarm/assertion/context"
-
-module Crabfarm
-  class BaseState
-    include Assertion::Context
-    extend Forwardable
-
-    PARSE_METHOD_RX = /^parse_(.*)$/
-
-    attr_reader :params, :output
-
-    def_delegators '@context', :http
-    def_delegators '@context.pool', :driver
-    def_delegators '@context.store', :get, :fetch
-
-    def self.browser_dsl(_dsl)
-      @class_browser_dsl = _dsl
-    end
-
-    def self.output_builder(_builder)
-      @class_output_builder = _builder
-    end
-
-    def initialize(_context, _params)
-      @context = _context
-      @params = _params
-
-      @dsl = Strategies.load(:browser_dsl, class_browser_dsl || Crabfarm.config.browser_dsl)
-      @builder = Strategies.load(:output_builder, class_output_builder || Crabfarm.config.output_builder)
-      @output = @builder.prepare
-    end
-
-    def browser(_name=nil)
-      @dsl.wrap driver(_name)
-    end
-
-    def download(_url)
-      @context.http.get(_url).body
-    end
-
-    def output
-      @output
-    end
-
-    def output_as_json
-      @builder.serialize @output
-    end
-
-    def crawl
-      raise NotImplementedError.new
-    end
-
-    def parse(_target=nil, _options={})
-      parser_class = _options.delete :using
-
-      if parser_class.nil?
-        parser_class = (self.class.name + 'Parser').constantize
-      end
-
-      parser = parser_class.new _target, @params.merge(_options)
-      parser.parse
-      return parser
-    end
-
-    def fork_each(_enumerator, &_block)
-      session_id = 0
-      mutex = Mutex.new
-      ths = _enumerator.map do |value|
-        session_id += 1
-        start_forked_state("th_session_#{session_id}", value, _block, mutex)
-      end
-      ThreadsWait.all_waits(*ths)
-    end
-
-    def method_missing(_method, *_args, &_block)
-      m = PARSE_METHOD_RX.match(_method)
-      if m
-        options = _args[1] || {}
-        options[:using] = (m[1].camelize + 'Parser').constantize
-        parse _args[0], options
-      else super end
-    end
-
-    def respond_to?(_method, _include_all=false)
-      return true if PARSE_METHOD_RX === _method
-      super
-    end
-
-  private
-
-    def class_browser_dsl
-      self.class.instance_variable_get :@class_browser_dsl
-    end
-
-    def class_output_builder
-      self.class.instance_variable_get :@class_output_builder
-    end
-
-    def start_forked_state(_name, _value, _block, _mutex)
-      Thread.new {
-        sub_state = ForkedState.new self, _name, _mutex
-        begin
-          sub_state.instance_exec _value, &_block
-        ensure
-          sub_state.driver.reset
-        end
-      }
-    end
-  end
-end