crabfarm 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7eb935482cb5663082aae4a3d9e24a722c1aea5d
+  data.tar.gz: 05b8038d530eb0d5f4be9325621405c90056b7be
+SHA512:
+  metadata.gz: ef524a63574fa86249a0f31f08af390c07be7cb738f6a1ac8063bc59ba1a426074bc5681867c234e341905b150c73f3e2f2bd056f1e71dd39b15b9eff8ea5e3c
+  data.tar.gz: 23496fa635c7baca3606c04693939a6b600a8898b6acbc2c55231985f0b5d8b9420316e52a54c743e26b31ced3e805cc7e938b1e14b21d234b171f4723d653f1

data/bin/crabfarm

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+begin
+  require './boot'
+rescue LoadError
+  require 'crabfarm'
+end
+
+require 'crabfarm/cli'

data/lib/crabfarm.rb

@@ -0,0 +1,24 @@
+require "forwardable"
+require "jbuilder"
+require "selenium-webdriver"
+
+require "crabfarm/version"
+require "crabfarm/errors"
+require "crabfarm/configuration"
+require "crabfarm/module_helper"
+require "crabfarm/driver_bucket"
+require "crabfarm/driver_bucket_pool"
+require "crabfarm/default_driver_factory"
+require "crabfarm/phantom_driver_factory"
+require "crabfarm/phantom_runner"
+require "crabfarm/state_store"
+require "crabfarm/context"
+require "crabfarm/base_state"
+require "crabfarm/base_parser"
+require 'crabfarm/dsl/surfer'
+require "crabfarm/adapters"
+require "crabfarm/loader"
+
+module Crabfarm
+  # Your code goes here...
+end

data/lib/crabfarm/adapters.rb

@@ -0,0 +1,23 @@
+require 'crabfarm/adapters/capybara_adapter'
+require 'crabfarm/adapters/surfer_adapter'
+require 'crabfarm/adapters/watir_adapter'
+
+module Crabfarm
+  module Adapters
+    @@adapters = {}
+
+    def self.register_dsl(_name, _adapter)
+      @@adapters[_name.to_sym] = _adapter
+    end
+
+    def self.load_from_dsl_name _name
+      raise ConfigurationError.new "Invalid dsl name #{_name}" unless @@adapters.has_key? _name.to_sym
+      @@adapters[_name.to_sym]
+    end
+
+    # bundled adapters
+    register_dsl :watir, WatirAdapter
+    register_dsl :capybara, CapybaraAdapter
+    register_dsl :surfer, SurferAdapter
+  end
+end

data/lib/crabfarm/adapters/capybara_adapter.rb

@@ -0,0 +1,7 @@
+module Crabfarm::Adapters
+  class CapybaraAdapter
+    def self.wrap(_bucket)
+      raise "Capybara adapter is incompleted"
+    end
+  end
+end

data/lib/crabfarm/adapters/surfer_adapter.rb

@@ -0,0 +1,7 @@
+module Crabfarm::Adapters
+  class SurferAdapter
+    def self.wrap(_bucket)
+      Crabfarm::Dsl::Surfer::SurfContext.new _bucket
+    end
+  end
+end

data/lib/crabfarm/adapters/watir_adapter.rb

@@ -0,0 +1,7 @@
+module Crabfarm::Adapters
+  class WatirAdapter
+    def self.wrap(_bucket)
+      Watir::Browser.new _bucket.original
+    end
+  end
+end

data/lib/crabfarm/base_parser.rb

@@ -0,0 +1,26 @@
+module Crabfarm
+  class BaseParser
+
+    attr_reader :browser, :params
+
+    def self.browser_dsl(_dsl)
+      @dsl = _dsl
+    end
+
+    def initialize(_module, _driver, _params)
+      dsl_class = Adapters.load_from_dsl_name(class_dsl || _module.settings.default_dsl)
+      @browser = dsl_class.wrap _driver
+      @params = _params
+    end
+
+    def parse
+      raise NotImplementedError.new
+    end
+
+  private
+
+    def class_dsl
+      self.class.instance_variable_get :@dsl
+    end
+  end
+end

data/lib/crabfarm/base_state.rb

@@ -0,0 +1,41 @@
+module Crabfarm
+  class BaseState
+    extend Forwardable
+
+    attr_reader :params
+
+    def_delegators :@pool, :driver
+    def_delegators :@store, :get, :fetch
+
+    def self.browser_dsl(_dsl)
+      @class_dsl = _dsl
+    end
+
+    def initialize(_module, _pool, _store, _params)
+      @module = _module
+      @pool = _pool
+      @store = _store
+      @params = _params
+      @output = Jbuilder.new
+      @dsl = Adapters.load_from_dsl_name(class_dsl || @module.settings.default_dsl)
+    end
+
+    def browser(_name=nil)
+      @dsl.wrap driver(_name)
+    end
+
+    def output
+      @output ||= Jbuilder.new
+    end
+
+    def crawl
+      raise NotImplementedError.new
+    end
+
+  private
+
+    def class_dsl
+      self.class.instance_variable_get :@class_dsl
+    end
+  end
+end

data/lib/crabfarm/cli.rb

@@ -0,0 +1,79 @@
+require 'gli'
+
+module Crabfarm
+  class CLI
+    extend GLI::App
+
+    program_desc 'Crabfarm toolbelt'
+
+    pre do |global_options,command,options,args|
+      # Things to do before
+      true
+    end
+
+    desc "Starts the crawler in console mode"
+    command [:console, :c] do |c|
+      c.action do |global_options,options,args|
+        require "crabfarm/modes/console"
+        Crabfarm::Modes::Console.console_loop
+      end
+    end
+
+    desc "Starts the crawler in server mode"
+    command [:server, :s] do |c|
+      c.desc "Set the server host, defaults to 0.0.0.0"
+      c.flag [:h,:host]
+
+      c.desc "Set the server port, defaults to 3100"
+      c.flag [:p,:port]
+
+      c.desc "Set the server min and max threads, defaults to 0:16"
+      c.flag [:t,:threads]
+
+      c.action do |global_options,options,args|
+        require "crabfarm/modes/server"
+        server_options = {}
+        server_options[:Host] = options[:host] unless options[:host].nil?
+        server_options[:Port] = options[:port] || 3100
+        server_options[:Threads] = options[:threads] unless options[:threads].nil?
+        Crabfarm::Modes::Server.start server_options
+      end
+    end
+
+    desc "Generates crabfarm scaffolding"
+    command [:generate, :g] do |c|
+
+      c.desc "Generates a new crabfarm application"
+      c.command :app do |app|
+        app.action do |global_options,options,args|
+          require "crabfarm/modes/generator"
+          Crabfarm::Modes::Generator.new.generate_app(args[0], Dir.pwd)
+        end
+      end
+
+      c.desc "Generates a new crabfarm parser and parser spec"
+      c.command :parser do |parser|
+        parser.action do |global_options,options,args|
+          require "crabfarm/modes/generator"
+          Crabfarm::Modes::Generator.new.generate_parser(args[0])
+        end
+      end
+
+      c.desc "Generates a new crabfarm state and parser spec"
+      c.command :state do |parser|
+        parser.action do |global_options,options,args|
+          require "crabfarm/modes/generator"
+          Crabfarm::Modes::Generator.new.generate_state(args[0])
+        end
+      end
+    end
+
+    command :publish do |c|
+      c.action do |global_options,options,args|
+
+      end
+    end
+
+    exit run(ARGV)
+  end
+end

data/lib/crabfarm/configuration.rb

@@ -0,0 +1,83 @@
+module Crabfarm
+
+  class Configuration
+
+    # TODO: improve DSL, it sucks
+
+    attr_accessor :default_dsl
+    attr_accessor :driver_factory
+
+    # Default driver configuration parameters
+    attr_accessor :driver_name
+    attr_accessor :driver_host
+    attr_accessor :driver_port
+    attr_accessor :driver_capabilities
+    attr_accessor :driver_remote_timeout
+    attr_accessor :driver_window_width
+    attr_accessor :driver_window_height
+
+    # Phantom launcher configuration
+    attr_accessor :phantom_enabled
+    attr_accessor :phantom_load_images
+    attr_accessor :phantom_proxy
+    attr_accessor :phantom_ssl
+    attr_accessor :phantom_bin_path
+    attr_accessor :phantom_lock_file
+
+    def driver_config
+      {
+        name: @driver_name,
+        capabilities: @driver_capabilities,
+        remote_host: driver_remote_host,
+        remote_timeout: @driver_remote_timeout,
+        window_width: @driver_window_width,
+        window_height: @driver_window_height
+      }
+    end
+
+    def phantom_enabled?
+      @phantom_enabled
+    end
+
+    def phantom_config
+      {
+        load_images: @phantom_load_images,
+        proxy: @phantom_proxy,
+        ssl: @phantom_ssl,
+        bin_path: @phantom_bin_path,
+        lock_file: @phantom_lock_file
+      }
+    end
+
+    def initialize
+      @default_dsl = :surfer
+      @driver_factory = nil
+
+      @driver_name = :chrome
+      @driver_capabilities = Selenium::WebDriver::Remote::Capabilities.firefox
+      @driver_host = 'localhost'
+      @driver_port = '8080'
+      @driver_remote_timeout = 120
+      @driver_window_width = 1280
+      @driver_window_height = 800
+
+      @phantom_enabled = false
+      @phantom_load_images = false
+      @phantom_proxy = nil
+      @phantom_ssl = 'any'
+      @phantom_bin_path = 'phantomjs'
+      @phantom_lock_file = nil
+    end
+
+  private
+
+    def driver_remote_host
+      if @driver_host.nil? then nil
+      elsif @driver_port.nil? then "http://#{@driver_host}"
+      else "http://#{@driver_host}:#{@driver_port}"
+      end
+    end
+
+  end
+
+end

data/lib/crabfarm/context.rb

@@ -0,0 +1,32 @@
+require 'active_support'
+
+module Crabfarm
+  class Context
+    extend Forwardable
+
+    def_delegators :@pool, :driver
+
+    def initialize(_module)
+      @module = ModuleHelper.new _module
+      @pool = DriverBucketPool.new @module
+      @store = StateStore.new @module
+    end
+
+    def run_state(_name, _params={})
+      state = @module.load_state(_name).new @module, @pool, @store, _params
+      state.crawl
+      state
+    end
+
+    def reset
+      @store.reset
+      @pool.reset
+    end
+
+    def release
+      @pool.release
+    end
+
+  end
+
+end

data/lib/crabfarm/default_driver_factory.rb

@@ -0,0 +1,37 @@
+module Crabfarm
+  class DefaultDriverFactory
+
+    def initialize(_config={})
+      @config = _config
+    end
+
+    def build_driver(_session_id)
+
+      driver_name = @config[:name]
+      raise ConfigurationError.new 'must provide a webdriver type' if driver_name.nil?
+
+      case driver_name
+      when :remote
+        # setup a custom client to use longer timeouts
+        client = Selenium::WebDriver::Remote::Http::Default.new
+        client.timeout = @config[:remote_timeout]
+
+        driver = Selenium::WebDriver.for :remote, {
+          :url => @config[:remote_host],
+          :http_client => client,
+          :desired_capabilities => @config[:capabilities]
+        }
+
+        driver.send(:bridge).setWindowSize(@config[:window_width], @config[:window_height])
+      else
+        driver = Selenium::WebDriver.for driver_name.to_sym
+
+        # apply browser configuration to new driver
+        driver.manage.window.resize_to(@config[:window_width], @config[:window_height]) rescue nil
+      end
+
+      return driver
+    end
+
+  end
+end

data/lib/crabfarm/driver_bucket.rb

@@ -0,0 +1,50 @@
+module Crabfarm
+  class DriverBucket
+
+    attr_reader :session_id
+
+    def initialize(_module, _session_id, _factory)
+      @module = _module
+      @session_id = _session_id
+      @factory = _factory
+      @driver = nil
+    end
+
+    def setup(_factory)
+      reset
+      @factory = _factory
+    end
+
+    def parse(_parser_class, _options={})
+      _parser_class = @module.load_parser(_parser_class) if _parser_class.is_a? String or _parser_class.is_a? Symbol
+      parser = _parser_class.new @module, self, _options
+      parser.parse
+      return parser
+    end
+
+    def original
+      @driver ||= @factory.build_driver(@session_id)
+    end
+
+    def reset
+      if @driver
+        @driver.quit rescue nil
+        @driver = nil
+      end
+      self
+    end
+
+    # forward every missing method to actual driver
+
+    def respond_to?(symbol, include_priv=false)
+      original.respond_to?(symbol, include_priv)
+    end
+
+  private
+
+    def method_missing(method, *args, &block)
+      original.__send__(method, *args, &block)
+    end
+
+  end
+end