crabfarm 0.0.13 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c7539be24f102d8e0e7eb4961b2bb10678cfda2
-  data.tar.gz: 7c5fc40e8a2fa4a5b2474b1b8b34bc5287b88d2f
+  metadata.gz: 5ffa69cbe5f984ef8f3d348bcec410f1676f8409
+  data.tar.gz: 9f496ef1a4bd75caa56fac8c26ef24b69a284acc
 SHA512:
-  metadata.gz: bc1848de3ad1f401834f17d3c7691e6cfa90f17939ec5cde588fc1fc933f25e4ecd6af3b1be43a26d42182bb4f008fbd6d42b68ffe23fcd195b396540bfb72db
-  data.tar.gz: 6b5a780eddbf1ca4cdf53b9d5ababfcd10825badda27f170e51f1e07ea22a12ff5aabea661f35166cf4f153955b9f13a589bbafe556efa845630a77cd2ce7ca1
+  metadata.gz: 976296a79fa1281b7415ec823b9e4c1e8c01d8f9280d13f382c507cba69f64a26c508f0bd9fccae7eb4d377101066cc66b33673939eb04e6133d33591658d468
+  data.tar.gz: 6d0cca01d8dc9bef52d888c72f6af1a3e623f14df75913b16c946db9fa86d67f9a3d883fec7c85eef30ff80f10e529e1a20f819f4bad76ba3383db99cd47bdff

data/lib/crabfarm/base_state.rb

@@ -22,6 +22,7 @@ module Crabfarm
       @pool = _pool
       @store = _store
       @params = _params
+      @events = []
 
       @dsl = Strategies.load(:browser_dsl, class_browser_dsl || Crabfarm.config.browser_dsl)
       @builder = Strategies.load(:output_builder, class_output_builder || Crabfarm.config.output_builder)
@@ -43,6 +44,18 @@ module Crabfarm
       raise NotImplementedError.new
     end
 
+    def event(_type, _msg)
+      @events << { created_at: Time.current, type: _type, msg: _msg }
+    end
+
+    def alert(_msg)
+      event(:alert, _msg)
+    end
+
+    def info(_msg)
+      event(:info, _msg)
+    end
+
     def fork_each(_enumerator, &_block)
       session_id = 0
       mutex = Mutex.new

data/lib/crabfarm/context.rb

@@ -12,11 +12,9 @@ module Crabfarm
     end
 
     def load
-      unless @loaded
-        init_phantom_if_required
-        @pool = DriverBucketPool.new build_driver_factory
-        @loaded = true
-      end
+      init_phantom_if_required
+      init_driver_pool
+      @loaded = true
     end
 
     def run_state(_name, _params={})
@@ -27,28 +25,44 @@ module Crabfarm
     end
 
     def reset
-      load
       @store.reset
-      @pool.reset
+      @pool.reset unless @pool.nil?
     end
 
     def release
-      if @loaded
-        @pool.release
-        @phantom.stop unless @phantom.nil?
-        @loaded = false
-      end
+      release_driver_pool
+      release_phantom
+      @loaded = false
     end
 
   private
 
+    def init_driver_pool
+      @pool = DriverBucketPool.new build_driver_factory if @pool.nil?
+    end
+
+    def release_driver_pool
+      @pool.release unless @pool.nil?
+      @pool = nil
+    end
+
     def init_phantom_if_required
-      if config.phantom_mode_enabled?
-        @phantom = PhantomRunner.new phantom_config
-        @phantom.start
+      if config.phantom_mode_enabled? and @phantom.nil?
+        @phantom = load_and_start_phantom
       end
     end
 
+    def load_and_start_phantom
+      new_phantom = PhantomRunner.new phantom_config
+      new_phantom.start
+      return new_phantom
+    end
+
+    def release_phantom
+      @phantom.stop unless @phantom.nil?
+      @phantom = nil
+    end
+
     def build_driver_factory
       if @phantom
         PhantomDriverFactory.new @phantom, driver_config

data/lib/crabfarm/engines/safe_state_loop.rb

@@ -8,6 +8,7 @@ module Crabfarm
       def initialize
         @running = true
         @working = false
+        @fatal = nil
         @lock = Mutex.new
         @thread = Thread.new { crawl_loop }
       end
@@ -19,7 +20,9 @@ module Crabfarm
 
       def change_state(_name, _params={}, _wait=nil)
         @lock.synchronize {
-          if @working
+          if @fatal
+            raise CrawlerError.new @fatal
+          elsif @working
             raise StillWorkingError.new unless matches_next_state? _name, _params
             wait_and_load_struct _wait
           elsif matches_current_state? _name, _params
@@ -77,6 +80,7 @@ module Crabfarm
       end
 
       def state_as_struct
+        raise CrawlerError.new @fatal if @fatal
         raise CrawlerError.new @error if @error
 
         OpenStruct.new({
@@ -118,6 +122,11 @@ module Crabfarm
         rescue Exception => e
           logger.fatal "StateLoop: unhandled exception!"
           logger.fatal e
+
+          @lock.synchronize {
+            @working = false
+            @fatal = e
+          }
         ensure
           context.release
         end

data/lib/crabfarm/phantom_runner.rb

@@ -1,8 +1,11 @@
 require 'net/http'
+require 'timeout'
 
 module Crabfarm
   class PhantomRunner
 
+    PHANTOM_START_TM = 5 # seconds
+
     attr_reader :port
 
     def initialize(_config={})
@@ -13,8 +16,7 @@ module Crabfarm
     def start
       find_available_port
       Crabfarm.logger.info "Starting phantomjs in port #{@port}"
-      @pid = Process.spawn({}, phantomjs_cmd)
-      wait_for_server
+      @pid = spawn_phantomjs
       Crabfarm.logger.info "Phantomjs started (PID: #{@pid})"
     end
 
@@ -30,6 +32,18 @@ module Crabfarm
 
   private
 
+    def spawn_phantomjs
+      pid = Process.spawn({}, phantomjs_cmd)
+      begin
+        Timeout::timeout(PHANTOM_START_TM) { wait_for_server }
+      rescue Timeout::Error
+        Process.kill "INT", pid
+        Process.wait pid
+        raise
+      end
+      return pid
+    end
+
     def phantomjs_cmd
       cmd = [@config[:bin_path]]
       cmd << '--load-images=false' unless @config[:load_images]

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.0.13"
+  VERSION = "0.0.14"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.0.14
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-27 00:00:00.000000000 Z
+date: 2015-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jbuilder