cangrejo 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d9cb5f49f576488055b6f9e9164bbfb6af271b4
-  data.tar.gz: af63078d5ff257624f736e463ab96e8df5be5871
+  metadata.gz: 654c44f4ed8700ac8dbb214f8fd3006540ee31cb
+  data.tar.gz: cdf70051e44adbee2ca3291061ddd2ac71155571
 SHA512:
-  metadata.gz: 1efc55a238a31a5962e11a41122ec066f1955129a65cd44f4c840f6ce39e4b40325a59f16ab3810cbdf68da8ef382dad617dbfca4f8c3b25cb9163d7c8d2d793
-  data.tar.gz: eede15bb49b8efb34b51520e71fa83fea70819d0768e719a88a36b166626fbfda07fcb923ac41bf17202156037608ffb6a890f5fda5e5702731398824e34309b
+  metadata.gz: 847580c48eb860475720e652e7ee83dbb66f91aa7c88c2a15635296f0aa9471b7d63b67209a60b532a7ed421331aa8fc98b5c3721e68e6b913a2bc1edaf98b5d
+  data.tar.gz: e42eda249f094f9eca98b8a67f2f1bf77de681fe4c20e8538872ec46977a8cd5f255b9c96d2907cbc0dc72baddb71eb8d1664df1d30a92caa2d2238c66ff6984

data/lib/cangrejo/errors.rb

@@ -4,4 +4,23 @@ module Cangrejo
 
   class ConfigurationError < Error; end
 
+  class LaunchTimeout < Error
+    def initialize(_msg)
+      super "Timed out trying to start crawler"
+    end
+  end
+
+  class CrawlerError < Error
+
+    def initialize(_msg, _backtrace)
+      super _msg
+      @original_bt = _backtrace
+    end
+
+    def set_backtrace(_backtrace)
+      super @original_bt + _backtrace
+    end
+
+  end
+
 end

data/lib/cangrejo/modes/local.rb

@@ -1,41 +1,94 @@
 require "cangrejo/restclient/request_extensions"
 require "cangrejo/restclient/json_resource"
-require "cangrejo/support/launcher"
+require "childprocess"
 
 module Cangrejo
   module Modes
     class Local
 
+      attr_reader :process, :path
+
       def initialize(_path)
         @path = _path
       end
 
       def setup
-        init_launcher
+        select_socket_file
+        start_process
+        wait_for_socket
         init_rest_client
       end
 
       def release
-        @launcher.kill unless @launcher.nil?
+        process.stop unless process.nil?
       end
 
     private
 
-      def init_launcher
-        @launcher = Support::Launcher.new @path, timeout: launch_timeout, argv: launch_cmd_arguments
-        @launcher.launch
+      def cmd_enviroment
+        {
+          'BUNDLE_GEMFILE' => gem_path
+        }
+      end
+
+      def cmd_arguments
+        [
+          '--no-reload'
+        ]
       end
 
       def launch_timeout
         5.0
       end
 
-      def launch_cmd_arguments
-        ['--no-reload']
+      def start_process
+        @process = prepare_process
+        @process.start
+      end
+
+      def prepare_process
+        cmd = [ "bin/crabfarm", "s", "--host=#{host}" ]
+        cmd += cmd_arguments
+
+        puts cmd.join(' ')
+
+        cp = ChildProcess.build(*cmd)
+        cp.environment.merge! cmd_enviroment
+        cp.cwd = @path
+        cp.leader = true
+        cp.io.inherit!
+
+        return cp
+      end
+
+      def wait_for_socket
+        Timeout::timeout(launch_timeout, LaunchTimeout) do
+          # TODO: detect if the process crashes before timeout
+          sleep 0.1 while not File.exist? @socket_file
+        end
       end
 
       def init_rest_client
-        RestClient::JsonResource.new Net::SocketUri.new(@launcher.host, '/api/state')
+        RestClient::JsonResource.new Net::SocketUri.new(host, '/api/state')
+      end
+
+      def select_socket_file
+        @socket_file = random_filename while @socket_file.nil? or File.exist? @socket_file
+      end
+
+      def random_filename
+        File.join(Cangrejo.config.temp_path, "csocket-#{Random.rand(1000000)}.sock")
+      end
+
+      def gem_path
+        File.join(@path, 'Gemfile')
+      end
+
+      def host
+        # TODO: add posibility to use ports instead of unix sockets, it would also
+        # be nice to have a mechanism where the loaded process reports the port it
+        # binded to.
+        "unix://#{@socket_file}"
       end
 
     end

data/lib/cangrejo/session.rb

@@ -53,16 +53,22 @@ module Cangrejo
 
       params = add_timestamp(_params)
 
-      while_times_out do
-        @rest.put(name: _state, params: params, wait: WAIT_STEP)
-        @state_name = _state
-        @state_params = _params
+      begin
+        while_times_out do
+          @rest.put(name: _state, params: params, wait: WAIT_STEP)
+          @state_name = _state
+          @state_params = _params
+        end
+      rescue RestClient::InternalServerError => exc
+        raise unwrap_packed_exception exc
       end
 
       wrap_response_doc
       self
     end
 
+    alias :navigate :crawl
+
     def release
       @mode.release
       @rest = nil
@@ -107,5 +113,14 @@ module Cangrejo
       end
     end
 
+    def unwrap_packed_exception(_exc)
+      begin
+        data = JSON.parse _exc.http_body
+        Cangrejo::CrawlerError.new data['exception'], data['backtrace']
+      rescue
+        _exc
+      end
+    end
+
   end
 end

data/lib/cangrejo/version.rb

@@ -1,3 +1,3 @@
 module Cangrejo
-  VERSION = "0.1.5"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,92 +1,183 @@
 --- !ruby/object:Gem::Specification
 name: cangrejo
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-04 00:00:00.000000000 Z
+date: 2015-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.2
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.2
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: git
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: childprocess
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.4'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: rspec-nc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.11'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.5'
+- !ruby/object:Gem::Dependency
+  name: terminal-notifier-guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+- !ruby/object:Gem::Dependency
+  name: sys-proctable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: 
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: Cangrejo lets you consume crabfarm crawlers using a simple DSL
 email:
 - ignacio@platan.us
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/cangrejo.rb
 - lib/cangrejo/configurator.rb
 - lib/cangrejo/errors.rb
 - lib/cangrejo/modes/git.rb
@@ -97,10 +188,8 @@ files:
 - lib/cangrejo/restclient/json_resource.rb
 - lib/cangrejo/restclient/request_extensions.rb
 - lib/cangrejo/session.rb
-- lib/cangrejo/support/launcher.rb
 - lib/cangrejo/version.rb
-- lib/cangrejo.rb
-homepage: ''
+homepage: https://github.com/platanus/cangrejo-gem
 licenses:
 - MIT
 metadata: {}
@@ -110,19 +199,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Crabfarm client for ruby
 test_files: []
-has_rdoc: 

data/lib/cangrejo/support/launcher.rb

@@ -1,79 +0,0 @@
-require 'timeout'
-
-module Cangrejo
-  module Support
-    class Launcher
-
-      class LaunchTimeout < Cangrejo::Error
-        def initialize(_msg)
-          super "Timed out trying to start crawler"
-        end
-      end
-
-      SPAWN_TIMEOUT = 5
-      KILL_TIMEOUT = 5
-
-      def initialize(_path, _options={})
-        @path = _path
-        @timeout = _options.fetch(:timeout, SPAWN_TIMEOUT)
-        @argv = _options.fetch(:argv, [])
-        select_socket_file
-      end
-
-      def host
-        "unix://#{@socket_file}"
-      end
-
-      def launch
-        gem_path = File.join(@path, 'Gemfile')
-        # TODO: for some reason, the gemfile path must be specified here, maybe because of rbenv?
-        @pid = Process.spawn({ 'BUNDLE_GEMFILE' => gem_path }, "bin/crabfarm s --host=#{host} #{@argv.join(' ')}", chdir: @path, pgroup: true)
-        wait_for_socket
-      end
-
-      def kill
-        safe_kill @pid unless @pid.nil?
-      end
-
-    private
-
-      def select_socket_file
-        @socket_file = random_filename while @socket_file.nil? or File.exist? @socket_file
-      end
-
-      def random_filename
-        File.join(Cangrejo.config.temp_path, "csocket-#{Random.rand(1000000)}.sock")
-      end
-
-      def wait_for_socket
-        Timeout::timeout(@timeout, LaunchTimeout) do
-          # TODO: detect if the process crashes before timeout
-          sleep 0.1 while not File.exist? @socket_file
-        end
-      end
-
-      def safe_kill _pid
-        begin
-          Timeout.timeout(KILL_TIMEOUT) do
-            Process.kill "INT", _pid
-            Process.wait _pid
-          end
-        rescue Timeout::Error
-        ensure
-          ensure_dead _pid
-        end
-      end
-
-      def ensure_dead _pid
-        begin
-          # Kill the entire process group to make sure childs aren't left hanging around
-          Process.kill(-9, _pid)
-          Process.wait _pid
-        rescue
-          nil
-        end
-      end
-
-    end
-  end
-end