crabfarm 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7eb935482cb5663082aae4a3d9e24a722c1aea5d
-  data.tar.gz: 05b8038d530eb0d5f4be9325621405c90056b7be
+  metadata.gz: cc45d8d0e358193e3729c8ca8d38715680470c78
+  data.tar.gz: 30c698146ec3b2d4032cddf4d4b3011af4edaa75
 SHA512:
-  metadata.gz: ef524a63574fa86249a0f31f08af390c07be7cb738f6a1ac8063bc59ba1a426074bc5681867c234e341905b150c73f3e2f2bd056f1e71dd39b15b9eff8ea5e3c
-  data.tar.gz: 23496fa635c7baca3606c04693939a6b600a8898b6acbc2c55231985f0b5d8b9420316e52a54c743e26b31ced3e805cc7e938b1e14b21d234b171f4723d653f1
+  metadata.gz: 85c7b65ee73ea096635260495077002c74d4891ecab632814de81a474eed2cfc9b9eebbfe4f60a200d927a60e7cbbf00e9910b6988fc9cc73abbee8368bb7163
+  data.tar.gz: 9607b619d00a96ae83a20238492d47897976498bf48e63400e5845765e5585c2bf872d31b2d9803479ee4900ca23433ee6b58c391c05d8441298566716d2e1f4

data/lib/crabfarm/cli.rb

@@ -1,4 +1,5 @@
 require 'gli'
+require 'crabfarm/support/gli'
 
 module Crabfarm
   class CLI
@@ -6,16 +7,17 @@ module Crabfarm
 
     program_desc 'Crabfarm toolbelt'
 
-    pre do |global_options,command,options,args|
-      # Things to do before
-      true
-    end
-
     desc "Starts the crawler in console mode"
     command [:console, :c] do |c|
+
+      Support::GLI.generate_options c
+
       c.action do |global_options,options,args|
+        next puts "This command can only be run inside a crabfarm application" unless defined? CF_LOADER
+
         require "crabfarm/modes/console"
-        Crabfarm::Modes::Console.console_loop
+        CF_LOADER.load Support::GLI.parse_options options
+        Crabfarm::Modes::Console.start CF_LOADER
       end
     end
 
@@ -30,13 +32,19 @@ module Crabfarm
       c.desc "Set the server min and max threads, defaults to 0:16"
       c.flag [:t,:threads]
 
+      Support::GLI.generate_options c
+
       c.action do |global_options,options,args|
+        next puts "This command can only be run inside a crabfarm application" unless defined? CF_LOADER
+
         require "crabfarm/modes/server"
         server_options = {}
         server_options[:Host] = options[:host] unless options[:host].nil?
         server_options[:Port] = options[:port] || 3100
         server_options[:Threads] = options[:threads] unless options[:threads].nil?
-        Crabfarm::Modes::Server.start server_options
+
+        CF_LOADER.load Support::GLI.parse_options options
+        Crabfarm::Modes::Server.start CF_LOADER, server_options
       end
     end
 

data/lib/crabfarm/configuration.rb

@@ -2,81 +2,103 @@ module Crabfarm
 
   class Configuration
 
-    # TODO: improve DSL, it sucks
-
-    attr_accessor :default_dsl
-    attr_accessor :driver_factory
-
-    # Default driver configuration parameters
-    attr_accessor :driver_name
-    attr_accessor :driver_host
-    attr_accessor :driver_port
-    attr_accessor :driver_capabilities
-    attr_accessor :driver_remote_timeout
-    attr_accessor :driver_window_width
-    attr_accessor :driver_window_height
-
-    # Phantom launcher configuration
-    attr_accessor :phantom_enabled
-    attr_accessor :phantom_load_images
-    attr_accessor :phantom_proxy
-    attr_accessor :phantom_ssl
-    attr_accessor :phantom_bin_path
-    attr_accessor :phantom_lock_file
+    class Option < Struct.new(:name, :type, :text); end
+
+    OPTIONS = [
+      [:default_dsl, :string, 'Default dsl used by parsers and states'],
+      [:driver_factory, :mixed, 'Driver factory, disabled if phantom_mode is used'],
+      [:log_path, :string, 'Path where logs should be stored'],
+
+      # Default driver configuration parameters
+      [:driver, ['chrome', 'firefox', 'phantomjs', 'remote'], 'Webdriver to be user, common options: chrome, firefox, phantomjs, remote.'],
+      [:driver_host, :string, 'Remote host, only available in driver: remote'],
+      [:driver_port, :integer, 'Remote port, only available in driver: remote'],
+      [:driver_capabilities, :mixed, 'Driver capabilities, depends on selected driver.'],
+      [:driver_remote_timeout, :float, 'Request timeout in seconds, only available for remote or phatomjs driver.'],
+      [:driver_window_width, :integer, 'Initial browser window width.'],
+      [:driver_window_height, :integer, 'Initial browser window height.'],
+
+      # Phantom launcher configuration
+      [:phantom_load_images, :boolean, 'Phantomjs image loading, only for phantomjs driver.'],
+      [:phantom_proxy, :string, 'Phantonjs proxy address, only for phantomjs driver.'],
+      [:phantom_ssl, ['sslv3', 'sslv2', 'tlsv1', 'any'], 'Phantomjs ssl mode: sslv3, sslv2, tlsv1 or any, only for phantomjs driver.'],
+      [:phantom_bin_path, :string, 'Phantomjs binary path, only for phantomjs driver.'],
+      [:phantom_lock_file, :string, 'Phantomjs lock file path, only for phantomjs driver.']
+    ]
+    .map { |o| Option.new *o }
+
+    OPTIONS.each do |var|
+      define_method "set_#{var.name}" do |val|
+        @values[var.name] = val
+      end
+
+      define_method "#{var.name}" do
+        @values[var.name]
+      end
+    end
+
+    def initialize
+      @values = {
+        default_dsl: :surfer,
+        driver_factory: nil,
+        log_path: 'logs',
+
+        driver: 'phantomjs',
+        driver_capabilities: Selenium::WebDriver::Remote::Capabilities.firefox,
+        driver_host: 'localhost',
+        driver_port: '8080',
+        driver_remote_timeout: 120,
+        driver_window_width: 1280,
+        driver_window_height: 800,
+
+        phantom_load_images: false,
+        phantom_proxy: nil,
+        phantom_ssl: 'any',
+        phantom_bin_path: 'phantomjs',
+        phantom_lock_file: nil
+      }
+    end
+
+    def set(_options)
+      @values.merge! _options
+    end
+
+    def driver_remote_host
+      if driver_host then nil
+      elsif driver_port then "http://#{driver_host}"
+      else "http://#{driver_host}:#{driver_port}"
+      end
+    end
 
     def driver_config
       {
-        name: @driver_name,
-        capabilities: @driver_capabilities,
+        name: driver,
+        capabilities: driver_capabilities,
         remote_host: driver_remote_host,
-        remote_timeout: @driver_remote_timeout,
-        window_width: @driver_window_width,
-        window_height: @driver_window_height
+        remote_timeout: driver_remote_timeout,
+        window_width: driver_window_width,
+        window_height: driver_window_height
       }
     end
 
-    def phantom_enabled?
-      @phantom_enabled
+    def phantom_mode_enabled?
+      driver.to_s == 'phantomjs'
     end
 
     def phantom_config
       {
-        load_images: @phantom_load_images,
-        proxy: @phantom_proxy,
-        ssl: @phantom_ssl,
-        bin_path: @phantom_bin_path,
-        lock_file: @phantom_lock_file
+        load_images: phantom_load_images,
+        proxy: phantom_proxy,
+        ssl: phantom_ssl,
+        bin_path: phantom_bin_path,
+        lock_file: phantom_lock_file,
+        log_file: log_path ? File.join(log_path, 'phantom.log') : nil
       }
     end
 
-    def initialize
-      @default_dsl = :surfer
-      @driver_factory = nil
-
-      @driver_name = :chrome
-      @driver_capabilities = Selenium::WebDriver::Remote::Capabilities.firefox
-      @driver_host = 'localhost'
-      @driver_port = '8080'
-      @driver_remote_timeout = 120
-      @driver_window_width = 1280
-      @driver_window_height = 800
-
-      @phantom_enabled = false
-      @phantom_load_images = false
-      @phantom_proxy = nil
-      @phantom_ssl = 'any'
-      @phantom_bin_path = 'phantomjs'
-      @phantom_lock_file = nil
-    end
-
-  private
-
-    def driver_remote_host
-      if @driver_host.nil? then nil
-      elsif @driver_port.nil? then "http://#{@driver_host}"
-      else "http://#{@driver_host}:#{@driver_port}"
-      end
-    end
+     # Add enviroment support (like a Gemfile)
+     # group :test { set_driver :phantom }
+     # set_driver :phantom, group: :test
 
   end
 

data/lib/crabfarm/context.rb

@@ -7,7 +7,7 @@ module Crabfarm
     def_delegators :@pool, :driver
 
     def initialize(_module)
-      @module = ModuleHelper.new _module
+      @module = _module
       @pool = DriverBucketPool.new @module
       @store = StateStore.new @module
     end

data/lib/crabfarm/driver_bucket_pool.rb

@@ -29,14 +29,14 @@ module Crabfarm
   private
 
     def init_phantom_if_required
-      if @module.settings.phantom_enabled?
+      if @module.settings.phantom_mode_enabled?
         @phantom = PhantomRunner.new @module.settings.phantom_config
         @phantom.start
       end
     end
 
     def build_driver_factory
-      if @module.settings.phantom_enabled?
+      if @module.settings.phantom_mode_enabled?
         PhantomDriverFactory.new @phantom, @module.settings.driver_config
       else
         return @module.settings.driver_factory if @module.settings.driver_factory

data/lib/crabfarm/loader.rb

@@ -6,58 +6,75 @@ module Crabfarm
 
     attr_reader :module
 
-    def initialize(_base_path, _module_name=nil, &_config_block)
+    def initialize(_base_path, _module=Object)
       @path = _base_path
-      @name = _module_name
-      @config_block = _config_block
-      @module = if is_wrapped? then
-        "::#{@name}".constantize rescue nil
-      else nil end
+      @module = _module
+      @config = nil
+      @source_loaded = false
     end
 
-    def is_wrapped?
-      @name.present?
+    def load(_overrides=nil)
+      load_config _overrides
+      load_source
     end
 
-    def is_loaded?
-      not @module.nil?
+    def load_config(_overrides=nil)
+      raise ConfigurationError.new 'Source already loaded, call unload_source first' if @source_loaded
+      raise ConfigurationError.new 'Crabfile not found' unless File.exists? crafile_path
+
+      @config = read_crabfile crafile_path
+      @config.set _overrides unless _overrides.nil?
+    end
+
+    def load_source
+      raise ConfigurationError.new 'Crabfile must be loaded first' if @config.nil?
+      raise ConfigurationError.new 'Source already loaded, call reload_source instead' if @source_loaded
+
+      require_from_path source_path
+      @source_loaded = true
+    end
+
+    def unload_source
+      # TODO: unload every class in a list
+      @source_loaded = false
     end
 
-    def load
-      crabfile = load_crabfile(@path)
-      @module = load_module(@name, File.join(@path, 'app'))
-      @module.send(:remove_const, :CF_CONFIG) rescue nil
-      @module.const_set :CF_CONFIG, crabfile
+    def reload_source
+      unload_source if @source_loaded
+      load_source
     end
 
-    def load_context
-      load unless is_loaded?
-      Context.new @module
+    def is_loaded?
+      @source_loaded
     end
 
-    def unload
-      Object.send(:remove_const, @name) if is_wrapped?
-      @module = nil
+    def load_context(_overrides={})
+      raise ConfigurationError.new 'Must load source first' unless @source_loaded
+      Context.new ModuleHelper.new @module, @config
     end
 
   private
 
-    def load_crabfile(_path)
-      crabfile = File.read(File.join(_path, 'Crabfile'))
+    def crafile_path
+      File.join @path, 'Crabfile'
+    end
+
+    def source_path
+      File.join @path, 'app'
+    end
+
+    def read_crabfile(_path)
       config = Configuration.new
-      config.instance_eval crabfile
-      @config_block.call(config) unless @config_block.nil?
+      config.instance_eval File.read crafile_path
       return config
     end
 
-    def load_module(_name, _path)
-      require_all_as(_name, _path)
-      if is_wrapped? then "::#{_name}".constantize else Object end
-    end
+    def require_from_path(_src_path)
+      @module.module_eval do
+        # TODO: put every loaded class in a list, store last update or hash so next time is required
+        # it can be reloaded automatically.
 
-    def require_all_as(_name, _src_path)
-      loader_code = "
-        pending = Dir.glob('#{File.join(_src_path, '**/*')}').select { |f| f.end_with? '.rb' }.map { |f| f[0...-3] }
+        pending = Dir.glob(File.join(_src_path, '**/*')).select { |f| f.end_with? '.rb' }.map { |f| f[0...-3] }
 
         while pending.size > 0
           new_pending = []
@@ -72,10 +89,7 @@ module Crabfarm
           require new_pending.first if new_pending.size == pending.size
           pending = new_pending
         end
-      "
-
-      loader_code = "module ::#{_name}; #{loader_code}; end" if _name.present?
-      Object.instance_eval loader_code
+      end
     end
 
   end

data/lib/crabfarm/modes/console.rb

@@ -20,7 +20,7 @@ module Crabfarm
           unless @context.nil?
             puts "Reloading crawler source".color(:green)
             @context.release
-            @loader.unload
+            @loader.reload_source
           end
 
           @context = @loader.load_context
@@ -56,29 +56,23 @@ module Crabfarm
         alias :r :reset
       end
 
-      def self.console_loop
+      def self.start(_loader)
+        dsl = ConsoleDsl.new(_loader)
 
-        if defined? CF_LOADER
-          # TODO: generated app should load itself
-          dsl = ConsoleDsl.new(CF_LOADER)
-
-          loop do
-            begin
-              dsl.instance_eval Readline.readline("> ", true)
-            rescue SyntaxError => se
-              puts "Syntax error: #{se.message}".color(:red)
-            rescue SystemExit, Interrupt
-              break
-            rescue => e
-              puts "Unknown command".color(:red)
-            end
+        loop do
+          begin
+            dsl.instance_eval Readline.readline("> ", true)
+          rescue SyntaxError => se
+            puts "Syntax error: #{se.message}".color(:red)
+          rescue SystemExit, Interrupt
+            break
+          rescue => e
+            puts "Unknown command".color(:red)
           end
-
-          puts "Releasing crawling context".color(:green)
-          dsl.context.release
-        else
-          puts "This command can only be run inside a crabfarm application".color(:red)
         end
+
+        puts "Releasing crawling context".color(:green)
+        dsl.context.release
       end
 
     end

data/lib/crabfarm/modes/server.rb

@@ -8,7 +8,7 @@ module Crabfarm
 
       class API < Grape::API
 
-        MAX_WAIT = 60.0 * 5
+        DEFAULT_WAIT = 60.0 * 5
 
         format :json
         prefix :api
@@ -27,9 +27,7 @@ module Crabfarm
           end
 
           def wait
-            if params.has_key? :wait
-              [params[:wait].to_f, MAX_WAIT].min
-            else MAX_WAIT end
+            params.fetch(:wait, DEFAULT_WAIT)
           end
 
           def print_state(_state)
@@ -63,8 +61,8 @@ module Crabfarm
         @@evaluator
       end
 
-      def self.start(_options)
-        @@evaluator = Engines::SafeStateLoop.new CF_LOADER
+      def self.start(_loader, _options)
+        @@evaluator = Engines::SafeStateLoop.new _loader
         begin
           Support::CustomPuma.run API, _options
         ensure

data/lib/crabfarm/module_helper.rb

@@ -5,12 +5,13 @@ module Crabfarm
 
     attr_reader :dsl
 
-    def initialize(_module)
+    def initialize(_module, _config=nil)
       @module = _module
+      @config = _config || @module::CF_CONFIG
     end
 
     def settings
-      @module::CF_CONFIG
+      @config
     end
 
     def load_state(_name)

data/lib/crabfarm/phantom_runner.rb

@@ -33,24 +33,24 @@ module Crabfarm
       cmd << "--webdriver=#{@port}"
       cmd << "--ssl-protocol=#{@config[:ssl]}" unless @config[:ssl].nil?
       cmd << "--ignore-ssl-errors=true"
-      cmd << "--webdriver-loglevel=NONE" # TODO: remove when log path is choosen
-      # cmd << "--webdriver-logfile=/path/to/log/phantom.log"
+      cmd << "--webdriver-loglevel=WARN"
+      cmd << "--webdriver-logfile=#{@config[:log_file]}" unless @config[:log_file].nil?
       cmd.join(' ')
     end
 
     def find_available_port
       with_lock do
-        server = TCPServer.new('127.0.0.1', 0)
-        @port = server.addr[1]
-        server.close
+        socket = Socket.new(:INET, :STREAM, 0)
+        socket.bind(Addrinfo.tcp("127.0.0.1", 0))
+        @port = socket.local_address.ip_port
+        socket.close
       end
     end
 
     def wait_for_server
       loop do
         begin
-          # TODO: generate a valid request to prevent warnings
-          Net::HTTP.get_response(URI.parse("http://127.0.0.1:#{@port}"))
+          Net::HTTP.get_response(URI.parse("http://127.0.0.1:#{@port}/status"))
           break
         rescue
         end

data/lib/crabfarm/support/gli.rb

@@ -0,0 +1,34 @@
+module Crabfarm
+  module Support
+    module GLI
+      def self.generate_options(_cmd)
+        Configuration::OPTIONS.each do |opt|
+          if opt.type != :mixed
+            _cmd.desc opt.text
+            _cmd.flag "cf-#{opt.name}"
+          end
+        end
+      end
+
+      def self.parse_options(_options)
+        config_overrides = {}
+        Configuration::OPTIONS.each do |opt|
+          value = _options["cf-#{opt.name}"]
+          next if value.nil?
+
+          value = if opt.type.is_a? Array
+            opt.type.find { |t| t.to_s == value }
+          elsif opt.type == :integer then value.to_i
+          elsif opt.type == :float then value.to_f
+          elsif opt.type == :boolean then [true, false].find { |t| t.to_s == value }
+          elsif opt.type == :string then value
+          else nil end
+          next if value.nil?
+
+          config_overrides[opt.name] = value
+        end
+        config_overrides
+      end
+    end
+  end
+end

data/lib/crabfarm/templates/Crabfile.erb

@@ -1,3 +1,2 @@
-self.default_dsl = :surfer
-self.driver_name = :firefox
-# self.phantom_enabled = true
+set_default_dsl :surfer
+set_driver :phantomjs

data/lib/crabfarm/templates/boot.rb.erb

@@ -1,13 +1,11 @@
-# Load gems listed in the Gemfile.
-
 require 'bundler'
 
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __FILE__)
-require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+# Load gems listed in the Gemfile.
 
-Bundler.require
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __FILE__)
+Bundler.require :default
 
 # Run code loader
 
 CF_PATH = File.expand_path('../', __FILE__)
-CF_LOADER = Crabfarm::Loader.new CF_PATH, ENV['CF_NAMESPACE']
+CF_LOADER = Crabfarm::Loader.new CF_PATH

data/lib/crabfarm/templates/dot_gitignore.erb

@@ -1 +1,2 @@
 Gemfile.lock
+logs/*.*

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Ignacio Baixas
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-30 00:00:00.000000000 Z
+date: 2015-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jbuilder
@@ -317,6 +317,7 @@ files:
 - lib/crabfarm/rspec.rb
 - lib/crabfarm/state_store.rb
 - lib/crabfarm/support/custom_puma.rb
+- lib/crabfarm/support/gli.rb
 - lib/crabfarm/templates/boot.rb.erb
 - lib/crabfarm/templates/crabfarm_bin.erb
 - lib/crabfarm/templates/Crabfile.erb