crabfarm 0.4.2 → 0.5.1

data/lib/crabfarm/live/context.rb

@@ -0,0 +1,21 @@
+require 'crabfarm/utils/webdriver'
+
+module Crabfarm
+  module Live
+    class Context < Crabfarm::Context
+
+      def initialize(_manager)
+        @manager = _manager
+      end
+
+      def proxy
+        "127.0.0.1:#{@manager.proxy_port}"
+      end
+
+      def viewer
+        @manager
+      end
+
+    end
+  end
+end

data/lib/crabfarm/live/controller.rb

@@ -1,35 +1,25 @@
-require 'benchmark'
-require 'rainbow'
-require 'rainbow/ext/string'
-require 'crabfarm/modes/console'
-
-require 'crabfarm/support/webdriver_factory'
-require 'crabfarm/crabtrap_runner'
-
-require 'crabfarm/live/helpers.rb'
-require 'crabfarm/live/navigator_runner.rb'
-require 'crabfarm/live/reducer_runner.rb'
+require 'crabfarm/utils/console'
+require 'crabfarm/live/navigator_runner'
+require 'crabfarm/live/reducer_runner'
 
 module Crabfarm
   module Live
     class Controller
 
-      Colors = Crabfarm::Modes::Console::Colors
-
-      class LiveWarning < StandardError; end
-
       def initialize(_manager)
         @manager = _manager
       end
 
+      def display_external_error(_exc)
+        @manager.reset_driver_status
+        display_error_feedback _exc
+      end
+
       def execute_live(_class)
         begin
           runner = build_runner_for _class
           prepare_session_for runner
-          elapsed = Benchmark.measure { runner.execute }
-          display_feedback runner, elapsed
-        rescue LiveWarning => exc
-          display_warning_feedback exc
+          runner.execute
         rescue Exception => exc
           display_error_feedback exc
         ensure
@@ -40,9 +30,9 @@ module Crabfarm
     private
 
       def build_runner_for(_class)
-        raise LiveWarning.new "'#{_class.to_s} is not Interactable" unless _class < Interactable
+        raise ArgumentError.new "'#{_class.to_s} is not Interactable" unless _class < Interactable
 
-        puts "Launching #{_class.to_s}".color Colors::NOTICE
+        Utils::Console.operation "Launching #{_class.to_s}"
 
         runner = if _class.live_delegate
           build_runner_for _class.live_delegate
@@ -52,8 +42,8 @@ module Crabfarm
           elsif _class < BaseReducer
             ReducerRunner
           else
-            raise LiveWarning.new "Don't know how to run #{_class.to_s}, you should provide a navigator or reducer as delegate."
-          end.new _class
+            raise ConfigurationError.new "Don't know how to run #{_class.to_s}, you should provide a navigator or reducer as delegate."
+          end.new @manager, _class
         end
 
         runner.dsl.instance_eval(&_class.live_setup) if _class.live_setup
@@ -61,65 +51,23 @@ module Crabfarm
       end
 
       def prepare_session_for(_runner)
-        @manager.stop_crabtrap
-        if _runner.memento
-          @manager.start_crabtrap :replay, memento_path(_runner.memento)
-        else
-          @manager.start_crabtrap :pass
-        end
+        @manager.reset_driver_status
       end
 
       def clean_up_session
         # leave crabtrap running for debugging purposes.
       end
 
-      def display_feedback(_runner, _elapsed)
-        load_web_ui
-
-        @manager.primary_driver.execute_script(
-          "window.crabfarm.showResults(arguments[0], arguments[1]);",
-          _runner.output,
-          _elapsed.real
-        );
-
-        puts _runner.output.to_s.color Colors::RESULT
-        puts "Completed in #{_elapsed.real} s".color Colors::NOTICE
-      end
-
       def display_error_feedback(_exc)
-        load_web_ui
-
-        @manager.primary_driver.execute_script(
-          "window.crabfarm.showError(arguments[0], arguments[1]);",
+        @manager.inject_web_tools
+        @manager.show_dialog(
+          :error,
+          'Crawler error!',
           "#{_exc.class.to_s}: #{_exc.to_s}",
           _exc.backtrace.join("\n")
-        );
-
-        puts "#{_exc.class.to_s}: #{_exc.to_s}".color Colors::ERROR
-        puts _exc.backtrace
-      end
-
-      def display_warning_feedback(_exc)
-        load_web_ui
-        puts "Warning: #{_exc.to_s}".color Colors::WARNING
-      end
-
-      def load_web_ui
-        Helpers.inject_style @manager.primary_driver, 'https://www.crabtrap.io/selectorgadget_combined.css'
-        Helpers.inject_style @manager.primary_driver, 'https://www.crabtrap.io/tools.css'
-        Helpers.inject_script @manager.primary_driver, 'https://www.crabtrap.io/selectorgadget_combined.js'
-        Helpers.inject_script @manager.primary_driver, 'https://www.crabtrap.io/tools.js'
-        wait_for_injection
-      end
-
-      def wait_for_injection
-        while @manager.primary_driver.execute_script "return (typeof window.crabfarm === 'undefined');"
-          sleep 1.0
-        end
-      end
+        )
 
-      def memento_path(_name)
-        File.join(GlobalState.mementos_path, _name + '.json.gz')
+        Utils::Console.exception _exc
       end
 
     end

data/lib/crabfarm/live/interactable.rb

@@ -1,4 +1,4 @@
-require 'crabfarm/live/helpers'
+require 'crabfarm/utils/webdriver'
 
 module Crabfarm
   module Live
@@ -15,6 +15,10 @@ module Crabfarm
           @setup = _setup
         end
 
+        def live_rspec?
+          @setup.nil?
+        end
+
         def live_delegate
           @delegate
         end
@@ -37,7 +41,7 @@ module Crabfarm
             _elements = Crabfarm.live.primary_driver.find_elements(css: _elements)
           end
 
-          Helpers.set_style _elements, "border: 3px solid yellow;"
+          Utils::Webdriver.set_style _elements, "border: 3px solid yellow;"
         end
       end
 

data/lib/crabfarm/live/manager.rb

@@ -1,3 +1,6 @@
+require 'timeout'
+require 'crabfarm/utils/console'
+require 'crabfarm/utils/webdriver'
 require 'crabfarm/support/webdriver_factory'
 require 'crabfarm/crabtrap_runner'
 
@@ -5,9 +8,11 @@ module Crabfarm
   module Live
     class Manager
 
+      INJECTION_TM = 5 # seconds
+
       def initialize
         @port = Utils::PortDiscovery.find_available_port
-        @driver = Crabfarm.config.recorder_driver
+        @driver_name = Crabfarm.config.recorder_driver
       end
 
       def proxy_port
@@ -15,6 +20,7 @@ module Crabfarm
       end
 
       def start
+        set_memento
         load_primary_driver
       end
 
@@ -27,26 +33,39 @@ module Crabfarm
         @driver
       end
 
-      def generate_support_driver
-        # TODO: improve on this mechanics, maybe use a frame in the same driver for this
-        build_driver
+      def reset_driver_status
+        # TODO: manage driver handles and recreate driver if needed
+        primary_driver.get('https://www.crabtrap.io/instructions.html')
       end
 
-      def start_crabtrap(_mode, _memento_path=nil)
-
-        raise ConfigurationError.new "No memento found at #{_memento_path}" unless File.exists? _memento_path
+      def block_requests
+        begin
+          stop_crabtrap
+          return yield
+        ensure
+          set_memento nil
+        end
+      end
 
-        if @crabtrap.nil?
-          options = {
-            mode: _mode,
-            bucket_path: _memento_path,
-            port: @port,
-            virtual: File.expand_path('./assets/live-tools', Crabfarm.root)
-          }
+      def set_memento(_memento=nil)
 
-          @crabtrap = CrabtrapRunner.new config.crabtrap_config.merge(options)
-          @crabtrap.start
+        options = if _memento
+          path = Utils::Resolve.memento_path _memento
+          raise ConfigurationError.new "No memento found at #{path}" unless File.exists? path
+          { mode: :replay, bucket_path: path }
+        else
+          { mode: :pass }
         end
+
+        options.merge!({
+          port: @port,
+          virtual: File.expand_path('./assets/live-tools', Crabfarm.root)
+        })
+
+        stop_crabtrap
+        @crabtrap = CrabtrapRunner.new config.crabtrap_config.merge(options)
+        @crabtrap.start
+
       end
 
       def stop_crabtrap
@@ -56,18 +75,65 @@ module Crabfarm
         else nil end
       end
 
+      def inject_web_tools
+        Utils::Console.trap_errors 'injecting web tools' do
+          Utils::Webdriver.inject_style primary_driver, 'https://www.crabtrap.io/selectorgadget_combined.css'
+          Utils::Webdriver.inject_style primary_driver, 'https://www.crabtrap.io/tools.css'
+          Utils::Webdriver.inject_script primary_driver, 'https://www.crabtrap.io/selectorgadget_combined.js'
+          Utils::Webdriver.inject_script primary_driver, 'https://www.crabtrap.io/tools.js'
+          Timeout::timeout(INJECTION_TM) { wait_for_injection }
+        end
+      end
+
+      def show_dialog(_status, _title, _subtitle, _content=nil, _content_type=:text)
+        Utils::Console.trap_errors 'loading web dialog' do
+          primary_driver.execute_script(
+            "window.crabfarm.showDialog.apply(null, arguments);",
+            _status.to_s,
+            _title,
+            _subtitle,
+            _content,
+            _content_type.to_s
+          );
+        end
+      end
+
+      def show_selector_gadget()
+        Utils::Console.trap_errors 'loading selector gadget' do
+          primary_driver.execute_script(
+            'window.crabfarm.showSelectorGadget();'
+          )
+        end
+      end
+
+      # Viewer implementation
+
+      def attach(_primary=true)
+        if _primary then primary_driver else build_driver end
+      end
+
+      def detach(_driver)
+        if _driver != primary_driver
+          _driver.quit rescue nil
+        end
+      end
+
     private
 
       def load_primary_driver
         @driver = build_driver
+        @driver.get('https://www.crabtrap.io/welcome.html')
       end
 
       def release_primary_driver
-        @driver.quit rescue nil unless @driver.nil?
+        unless @driver.nil?
+          @driver.quit rescue nil
+          @driver = nil
+        end
       end
 
       def build_driver
-        case @driver
+        case @driver_name
         when :firefox
           Crabfarm::Support::WebdriverFactory.build_firefox_driver driver_config
         when :chrome
@@ -85,6 +151,12 @@ module Crabfarm
         Crabfarm.config
       end
 
+      def wait_for_injection
+        while primary_driver.execute_script "return (typeof window.crabfarm === 'undefined');"
+          sleep 1.0
+        end
+      end
+
     end
   end
 end

data/lib/crabfarm/live/navigator_runner.rb

@@ -1,4 +1,6 @@
-require 'json'
+require 'crabfarm/live/context'
+require 'crabfarm/live/navigator_runner_direct'
+require 'crabfarm/live/navigator_runner_rspec'
 
 module Crabfarm
   module Live
@@ -11,11 +13,13 @@ module Crabfarm
           @runner = _runner
         end
 
-        def_delegators :@runner, :use_memento, :use_params, :clear_params, :navigate_to
+        def_delegators :@runner, :use_memento, :use_params, :clear_params, :use_rspec, :navigate_to
       end
 
-      def initialize(_target)
+      def initialize(_manager, _target)
+        @manager = _manager
         @target = _target
+        @rspec = true
         @params = {}
       end
 
@@ -29,32 +33,44 @@ module Crabfarm
 
       def use_memento(_memento)
         @memento = _memento
+        @rspec = false
       end
 
       def use_params(_params={})
         @params = @params.merge _params
+        @rspec = false
       end
 
       def clear_params
         @params = {}
+        @rspec = false
+      end
+
+      def use_rspec
+        @rspec = true
       end
 
       def navigate_to(_navigator, _params={})
+        # TODO.
+      end
+
+      def prepare(_memento) # decorator
+        @manager.set_memento _memento
+        Context.new @manager
       end
 
       def execute
-        context = Crabfarm::Context.new
-        begin
-          # TODO: execute prerequisites
-          @transition = TransitionService.transition context, @target, @params
-        ensure
-          context.release
+        strategy = if @rspec
+          NavigatorRunnerRSpec.new @manager, @target
+        else
+          NavigatorRunnerDirect.new @manager, memento, @target, @params
         end
-      end
 
-      def output
-        return nil if @transition.document.nil?
-        JSON.pretty_generate(@transition.document).gsub(/(^|\\n)/, '  ')
+        Factories::Context.with_decorator self do
+          strategy.execute
+        end
+
+        strategy.show_results
       end
 
     private
@@ -63,6 +79,20 @@ module Crabfarm
         Utils::Naming.route_from_constant(_class.to_s).join(File::SEPARATOR)
       end
 
+      def show_result
+        @manager.inject_web_tools
+        @manager.show_dialog(
+          :neutral,
+          'Navigation completed!',
+          "The page was scrapped in #{@elapsed.real} seconds",
+          @transition.document.to_json,
+          :json
+        )
+
+        Utils::Console.json_result @transition.document
+        Utils::Console.info "Completed in #{@elapsed.real} s"
+      end
+
     end
   end
 end

data/lib/crabfarm/live/navigator_runner_direct.rb

@@ -0,0 +1,39 @@
+require 'benchmark'
+require 'crabfarm/utils/console'
+
+module Crabfarm
+  module Live
+    class NavigatorRunnerDirect
+
+      def initialize(_manager, _memento, _target, _params)
+        @manager = _manager
+        @memento = _memento
+        @target = _target
+        @params = _params
+      end
+
+      def execute
+        Crabfarm.with_context @memento do |ctx|
+          @elapsed = Benchmark.measure do
+            @transition = TransitionService.transition ctx, @target, (@params || {})
+          end
+        end
+      end
+
+      def show_results
+        @manager.inject_web_tools
+        @manager.show_dialog(
+          :neutral,
+          'Navigation completed!',
+          "The page was scrapped in #{@elapsed.real} seconds",
+          @transition.document.to_json,
+          :json
+        )
+
+        Utils::Console.json_result @transition.document
+        Utils::Console.info "Completed in #{@elapsed.real} s"
+      end
+
+    end
+  end
+end

data/lib/crabfarm/live/navigator_runner_rspec.rb

@@ -0,0 +1,111 @@
+require 'crabfarm/utils/console'
+require 'crabfarm/utils/rspec_runner'
+
+module Crabfarm
+  module Live
+    class NavigatorRunnerRSpec
+
+      def initialize(_manager, _target)
+        @manager = _manager
+        @target = _target
+      end
+
+      def execute
+        @examples = Factories::Context.with_decorator self do
+          Utils::RSpecRunner.run_spec_for spec_for(@target), live: true
+        end
+      end
+
+      def show_results
+        @manager.inject_web_tools
+        if @examples.count == 0
+          show_empty_warning
+        elsif @examples.count == 1
+          show_example_output @examples.first
+        else
+          show_example_summary @examples
+        end
+      end
+
+    private
+
+      def spec_for(_class)
+        route = Utils::Naming.route_from_constant(_class.to_s)
+        route = route.join(File::SEPARATOR)
+        route = route + '_spec.rb'
+        File.join('spec','navigators', route)
+      end
+
+      def show_empty_warning
+        @manager.show_dialog(
+          :warning,
+          'No examples were found!',
+          'Make sure you have tagged some specs with live: true'
+        )
+
+        Utils::Console.warning 'No examples were found!'
+      end
+
+      def show_example_summary(_examples)
+        total = _examples.count
+        error = _examples.select { |e| !e.exception.nil? }.count
+        errored = (error > 0)
+
+        if error > 0
+          @manager.show_dialog(
+            :error,
+            'FAILED',
+            "#{error} of #{total} tests failed"
+          )
+
+          Utils::Console.error "#{total} examples, #{error} failures"
+        else
+          @manager.show_dialog(
+            :success,
+            'SUCCESS',
+            "All #{total} tests passed!"
+          )
+
+          Utils::Console.result "#{total} examples, 0 failures"
+        end
+      end
+
+      def show_example_output(_example)
+
+        handle_standard_errors _example
+
+        if _example.exception
+          @manager.show_dialog(
+            :error,
+            'FAILED',
+            _example.exception.to_s,
+            _example.metadata[:result].to_json,
+            :json
+          )
+
+          Utils::Console.error "1 example, 1 failure"
+          Utils::Console.error _example.exception.to_s
+          Utils::Console.json_result _example.metadata[:result]
+        else
+          @manager.show_dialog(
+            :success,
+            'SUCCESS',
+            "\"#{_example.full_description}\"",
+            _example.metadata[:result].to_json,
+            :json
+          )
+
+          Utils::Console.result "1 example, 0 failures"
+          Utils::Console.json_result _example.metadata[:result]
+        end
+      end
+
+      def handle_standard_errors(_example)
+        if _example.exception and not _example.exception.is_a? ::RSpec::Expectations::ExpectationNotMetError
+          raise _example.exception
+        end
+      end
+
+    end
+  end
+end

data/lib/crabfarm/live/reducer_runner.rb

@@ -1,20 +1,71 @@
+require 'crabfarm/live/reducer_runner_direct'
+require 'crabfarm/live/reducer_runner_rspec'
+
 module Crabfarm
   module Live
-    class ReducerRunner < NavigatorRunner
+    class ReducerRunner
+
+      class Dsl
+        extend Forwardable
+
+        def initialize(_runner)
+          @runner = _runner
+        end
+
+        def_delegators :@runner, :use_snapshot, :use_params, :clear_params, :use_rspec
+      end
+
+      def initialize(_manager, _target)
+        @manager = _manager
+        @target = _target
+        @rspec = true
+        @params = {}
+      end
+
+      def dsl
+        @dsl ||= Dsl.new self
+      end
+
+      def snapshot
+        if @snapshot.nil? then snapshot_for(@target) else @snapshot end
+      end
+
+      def use_snapshot(_snapshot)
+        @snapshot = _snapshot
+        @rspec = false
+      end
 
-      def initialize(_target)
-        # use navigator runner for now.
-        super navigator_from_reducer _target
+      def use_params(_params={})
+        @params = @params.merge _params
+        @rspec = false
       end
 
-      def navigator_from_reducer _reducer
-        m = _reducer.to_s.match(/^(.*?)Reducer$/)
-        if m
-          navigator = m[1].constantize rescue nil
-          return navigator if navigator and navigator < BaseNavigator
+      def clear_params
+        @params = {}
+        @rspec = false
+      end
+
+      def use_rspec
+        @rspec = true
+      end
+
+      def prepare(_class, _path, _params) # decorator
+        @manager.primary_driver.get("file://#{_path}")
+        nil
+      end
+
+      def execute
+        strategy = if @rspec
+          ReducerRunnerRSpec.new @manager, @target
+        else
+          ReducerRunnerDirect.new @manager, snapshot, @target, @params
+        end
+
+        Factories::SnapshotReducer.with_decorator self do
+          @manager.block_requests { strategy.execute }
         end
 
-        raise Controller::LiveWarning.new "Could not find a matching navigator for reducer #{_reducer.to_s}"
+        strategy.show_results
       end
 
     end