crabfarm 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2cd329fb207d26d0ce84b7d9a245c9555032f72d
-  data.tar.gz: a50a7c709070a8d1a1ff1528c28c03502b75ae9b
+  metadata.gz: 2df1fe8ddac13f58aebe2e0f99b2654e58ecc73e
+  data.tar.gz: 10243091f0a47c4af76cbd76ccf46a4d114c5d7c
 SHA512:
-  metadata.gz: 97697da5b22796d930e8db64269ad94e6205c729637f623bcd3d64331f7936200807b73873c93d5bbddc5c9c5b066dc5df85c8a191a0b4f82fa64c2cbb3ef19c
-  data.tar.gz: c9418e6a5ad1d64871bb88763c5980da97fc6a6d0ce91d99ec69ffed994c35d950240a52448c9be9b6ef4876980a4b65ea5bd9db9b24d2c12a5de780fda45e27
+  metadata.gz: 134313f90f076d08a2360cd0fca4f1920109bf871b49caa9843c222c860d42eeb6f8e463d4c8f0632370cf23f95546bbb65e9968fc19477ae225a15fe7eb98dd
+  data.tar.gz: 08db898418eb68af12a18658f21448fc92499ab1675b8092877d650ca039a81a2369a57330d760b8460d503029851660b25dc9ac18199489f7852c0141884e3d

data/lib/crabfarm.rb

@@ -1,13 +1,11 @@
 require "logger"
 require "forwardable"
-require "net/http"
 require "active_support/inflector"
 
 require "crabfarm/version"
 require "crabfarm/errors"
 require "crabfarm/configuration"
 require "crabfarm/driver_pool"
-require "crabfarm/http_client"
 require "crabfarm/state_store"
 require "crabfarm/context"
 require "crabfarm/transition_service"

data/lib/crabfarm/base_navigator.rb

@@ -13,7 +13,6 @@ module Crabfarm
 
     attr_reader :params
 
-    def_delegators '@context', :http
     def_delegators '@context.store', :get, :fetch
 
     def initialize(_context,  _params)
@@ -31,10 +30,6 @@ module Crabfarm
       @context.pool.driver(_name)
     end
 
-    def download(_url)
-      @context.http.get(_url).body
-    end
-
     def run
       raise NotImplementedError.new
     end

data/lib/crabfarm/context.rb

@@ -2,7 +2,7 @@ module Crabfarm
   class Context
     extend Forwardable
 
-    attr_accessor :pool, :store, :http
+    attr_accessor :pool, :store
 
     def initialize
       @store = StateStore.new
@@ -34,7 +34,6 @@ module Crabfarm
     def load_services
       init_browser_adapter
       init_driver_pool
-      init_http_client
     end
 
     def reset_services
@@ -43,7 +42,6 @@ module Crabfarm
     end
 
     def unload_services
-      release_http_client
       release_driver_pool
       release_browser_adapter
     end
@@ -69,22 +67,10 @@ module Crabfarm
       @pool = nil
     end
 
-    def init_http_client
-      @http = build_http_client proxy if @http.nil?
-    end
-
     def build_browser_adapter(_proxy)
       Strategies.load(:browser, config.browser).new _proxy
     end
 
-    def build_http_client(_proxy)
-      HttpClient.new _proxy
-    end
-
-    def release_http_client
-      @http = nil
-    end
-
     def proxy
       config.proxy
     end

data/lib/crabfarm/live/context.rb

@@ -21,8 +21,6 @@ module Crabfarm
         return BrowserAdapter.new @manager
       end
 
-      # TODO: override build_http_client, i would like to tap into the http requests and show downloaded data in viewer
-
       class BrowserAdapter < Crabfarm::Adapters::Browser::Base
 
         def initialize(_manager)

data/lib/crabfarm/modes/publisher.rb

@@ -8,6 +8,8 @@ require 'base64'
 require 'rainbow'
 require 'rainbow/ext/string'
 require 'digest/sha1'
+require 'net/http'
+require 'crabfarm/utils/console'
 
 module Crabfarm
   module Modes
@@ -22,12 +24,11 @@ module Crabfarm
         @options = _options
 
         load_config
-        return unless dry_run or check_credentials
-        detect_git_repo
+        return unless dry_run? or check_credentials
 
-        if inside_git_repo?
-          if not unsafe and is_tree_dirty?
-            puts "Aborting: Your working copy has uncommited changes! Use the --unsafe option to force.".color(:red)
+        if !unsafe? and detect_git_repo
+          if is_tree_dirty?
+            console.warning "Aborting: Your working copy has uncommited changes! Use the --unsafe option to force."
             return
           end
           load_files_from_git
@@ -39,16 +40,23 @@ module Crabfarm
         compress_package
         generate_signature
 
-        send_package if not dry_run and ensure_valid_remote
+        build_payload
+        send_package if not dry_run? and ensure_valid_remote
+
+        @payload
       end
 
     private
 
-      def dry_run
+      def verbose?
+        @options.fetch(:verbose, true)
+      end
+
+      def dry_run?
         @options.fetch(:dry, false)
       end
 
-      def unsafe
+      def unsafe?
         @options.fetch(:unsafe, false)
       end
 
@@ -77,7 +85,7 @@ module Crabfarm
 
       def ensure_valid_remote
         if @url.nil?
-          @url = Ask.input 'Enter default remote for crawler'
+          @url = console.question 'Enter default remote for crawler'
           return false unless validate_remote @url
           @local_config['remote'] = @url
           save_local_config
@@ -89,15 +97,15 @@ module Crabfarm
 
       def validate_remote(_url)
         return true if /^[\w\-]+\/[\w\-]+$/i === _url
-        puts "Invalid remote syntax: #{_url}".color :red
+        console.error "Invalid remote syntax: #{_url}"
         return false
       end
 
       def check_credentials
         if @token.nil?
-          puts 'No credential data found, please identify yourself'
-          email = Ask.input 'Enter your crabfarm.io email'
-          password = Ask.input 'Enter your crabfarm.io password'
+          console.info 'No credential data found, please identify yourself'
+          email = console.question 'Enter your crabfarm.io email'
+          password = console.question 'Enter your crabfarm.io password'
 
           resp = send_request Net::HTTP::Post, 'api/tokens', {
             'email' => email,
@@ -110,9 +118,9 @@ module Crabfarm
             @home_config['token'] = @token
             save_home_config
           when Net::HTTPUnauthorized
-            puts "The provided credentials are invalid!".color(:red)
+            console.error "The provided credentials are invalid!"
           else
-            puts "Unknown error when asking for token!".color(:red)
+            console.error "Unknown error when asking for token!"
           end
         end
 
@@ -148,7 +156,7 @@ module Crabfarm
           if File.exists? File.join(git_path, '.git')
             @git = Git.open git_path
             @rel_path = if path_to_git.count > 0 then File.join(*path_to_git.reverse!) else nil end
-            return
+            return true
           else
             path_to_git << File.basename(git_path)
             git_path = File.expand_path('..', git_path)
@@ -156,16 +164,13 @@ module Crabfarm
         end
 
         @git = nil
-      end
-
-      def inside_git_repo?
-        not @git.nil?
+        return false
       end
 
       def load_files_from_git
         @git.chdir do
           @ref = @git.log.first.sha
-          puts "Packaging files from current HEAD (#{@ref}):".color(:green)
+          console.result "Packaging files from current HEAD (#{@ref}):" if verbose?
           entries = @git.gtree(@ref).full_tree.map(&:split)
           entries = entries.select { |e| e[1] == 'blob' }
 
@@ -184,10 +189,12 @@ module Crabfarm
       end
 
       def load_files_from_fs
-        puts "Packaging files (no version control)".color(:green)
-        @file_list = Dir[*@include].map do |path|
-          full_path = File.join(@crawler_path, path)
-          [path, File.stat(full_path).mode, File.read(full_path)]
+        console.result "Packaging files (no version control)" if verbose?
+        Dir.chdir(@crawler_path) do
+          @file_list = Dir[*@include].map do |path|
+            full_path = File.join(@crawler_path, path)
+            [path, File.stat(full_path).mode, File.read(full_path)]
+          end
         end
         @ref = "filesystem"
       end
@@ -196,7 +203,7 @@ module Crabfarm
         @package = StringIO.new("")
         Gem::Package::TarWriter.new(@package) do |tar|
           @file_list.each do |f|
-            puts "+ #{f[0]} - #{f[1]}"
+            console.info "+ #{f[0]} - #{f[1]}" if verbose?
             path, mode, contents = f
             tar.add_file(path, mode) { |tf| tf.write contents }
           end
@@ -214,26 +221,30 @@ module Crabfarm
 
       def generate_signature
         @signature = Digest::SHA1.hexdigest @package.string
-        puts "Package SHA1: #{@signature}"
+        console.info "Package SHA1: #{@signature}" if verbose?
       end
 
-      def send_package
-        resp = send_request(Net::HTTP::Put, "api/bots/#{@url}", {
+      def build_payload
+        @payload = {
           "repo" => Base64.encode64(@cpackage.string),
           "sha" => @signature,
           "ref" => @ref
-        })
+        }
+      end
+
+      def send_package
+        resp = send_request(Net::HTTP::Put, "api/bots/#{@url}", @payload)
 
         case resp
         when Net::HTTPSuccess
           sha = JSON.parse(resp.body)['sha']
-          puts "#{@url} updated!"
+          console.result "#{@url} updated!"
         when Net::HTTPUnauthorized
-          puts "You are not authorized to update crawler: #{@url}".color(:red)
+          console.error "You are not authorized to update crawler: #{@url}"
         when Net::HTTPNotFound
-          puts "Crawler not found: #{@url}".color(:red)
+          console.error "Crawler not found: #{@url}"
         else
-          puts "Unknown error when updating crawler information!".color(:red)
+          console.error "Unknown error when updating crawler information!"
         end
       end
 
@@ -249,6 +260,10 @@ module Crabfarm
         end
       end
 
+      def console
+        Crabfarm::Utils::Console
+      end
+
     end
   end
 end

data/lib/crabfarm/support/phantom_runner.rb

@@ -1,4 +1,5 @@
 require 'timeout'
+require 'net/http'
 require 'crabfarm/utils/processes'
 
 module Crabfarm

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.7.2"
+  VERSION = "0.7.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-28 00:00:00.000000000 Z
+date: 2015-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -500,7 +500,6 @@ files:
 - lib/crabfarm/factories/reducer.rb
 - lib/crabfarm/factories/snapshot_reducer.rb
 - lib/crabfarm/forked_navigator.rb
-- lib/crabfarm/http_client.rb
 - lib/crabfarm/live/context.rb
 - lib/crabfarm/live/controller.rb
 - lib/crabfarm/live/interactable.rb

data/lib/crabfarm/http_client.rb

@@ -1,97 +0,0 @@
-require "uri"
-
-module Crabfarm
-  class HttpClient
-
-    class HttpRequestError < StandardError
-      extend Forwardable
-
-      def_delegators :@response, :code, :body
-
-      attr_reader :response
-
-      def initialize(_response)
-        @response = _response
-        super _response.message
-      end
-    end
-
-    class MaximumRedirectsError < StandardError
-      def initialize
-        super 'Redirection loop detected!'
-      end
-    end
-
-    attr_reader :proxy_addr, :proxy_port
-
-    def initialize(_proxy=nil)
-      if _proxy.nil?
-        @proxy_addr = nil
-        @proxy_port = nil
-      else
-        @proxy_addr, @proxy_port = _proxy.split ':'
-      end
-    end
-
-    def get(_url, _query={}, _headers={})
-      uri = URI _url
-      perform_request Net::HTTP::Get, uri, _headers
-    end
-
-    def post(_url, _data, _headers={})
-      perform_request Net::HTTP::Post, URI(_url), _headers do |req|
-        req.body = prepare_data(_data)
-      end
-    end
-
-    def put(_url, _data, _headers={})
-      perform_request Net::HTTP::Put, URI(_url), _headers do |req|
-        req.body = prepare_data(_data)
-      end
-    end
-
-    def delete(_url)
-      perform_request Net::HTTP::Delete, URI(_url), _headers
-    end
-
-  private
-
-    def perform_request(_req_type, _uri, _headers, _limit=10)
-
-      raise MaximumRedirectsError.new if _limit == 0
-
-      request = _req_type.new(_uri.request_uri.empty? ? '/' : _uri.request_uri)
-      _headers.keys.each { |k| request[k] = _headers[k] }
-      yield request if block_given?
-
-      response = build_client(_uri).request request
-
-      case response
-      when Net::HTTPSuccess then
-        response
-      when Net::HTTPRedirection then
-        location = response['location']
-        perform_request(_req_type, URI.parse(location), _headers, _limit - 1)
-      else
-        handle_error_response response
-      end
-    end
-
-    def build_client(uri)
-      client = Net::HTTP.new uri.host, uri.port || 80, proxy_addr, proxy_port
-      client.use_ssl = true if uri.scheme == 'https'
-      client.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      client
-    end
-
-    def handle_error_response(_response)
-      raise HttpRequestError.new _response
-    end
-
-    def prepare_data(_data)
-      if _data.is_a? Hash
-        _data.keys.map { |k| "#{k}=#{_data[k]}" }.join '&'
-      else _data end
-    end
-  end
-end