crabfarm 0.0.2

data/lib/crabfarm/modes/console.rb

@@ -0,0 +1,86 @@
+require 'readline'
+require 'rainbow'
+require 'rainbow/ext/string'
+require 'json'
+
+module Crabfarm
+  module Modes
+    class Console
+
+      class ConsoleDsl
+
+        attr_reader :context
+
+        def initialize(_loader)
+          @loader = _loader
+          reload!
+        end
+
+        def reload!
+          unless @context.nil?
+            puts "Reloading crawler source".color(:green)
+            @context.release
+            @loader.unload
+          end
+
+          @context = @loader.load_context
+        end
+
+        def transition(_name=nil, _params={})
+          if _name.nil?
+            puts "Must provide a state name".color(:red)
+            return
+          end
+
+          begin
+            state = @context.run_state _name, _params
+            puts JSON.pretty_generate(state.output.attributes!).color(:green)
+          rescue EntityNotFoundError => e
+            puts "#{e.to_s}".color(:red)
+          rescue => e
+            puts "#{e.to_s}".color(:red)
+            puts e.backtrace
+          end
+        end
+
+        def help
+          puts "Ejem..."
+        end
+
+        def reset
+          puts "Resetting crawling context".color(:green)
+          @context.reset
+        end
+
+        alias :t :transition
+        alias :r :reset
+      end
+
+      def self.console_loop
+
+        if defined? CF_LOADER
+          # TODO: generated app should load itself
+          dsl = ConsoleDsl.new(CF_LOADER)
+
+          loop do
+            begin
+              dsl.instance_eval Readline.readline("> ", true)
+            rescue SyntaxError => se
+              puts "Syntax error: #{se.message}".color(:red)
+            rescue SystemExit, Interrupt
+              break
+            rescue => e
+              puts "Unknown command".color(:red)
+            end
+          end
+
+          puts "Releasing crawling context".color(:green)
+          dsl.context.release
+        else
+          puts "This command can only be run inside a crabfarm application".color(:red)
+        end
+      end
+
+    end
+  end
+end

data/lib/crabfarm/modes/generator.rb

@@ -0,0 +1,120 @@
+require 'rainbow'
+require 'rainbow/ext/string'
+require 'active_support'
+require 'erb'
+require 'ostruct'
+
+module Crabfarm
+  module Modes
+    class Generator
+
+      def generate_app(_name, _target)
+        with_external_path _target do
+          binding = {
+            name: _name,
+            version: Crabfarm::VERSION
+          }
+
+          path(_name).ensure
+          path(_name, '.gitignore').render('dot_gitignore')
+          path(_name, 'Gemfile').render('Gemfile', binding)
+          path(_name, 'Crabfile').render('Crabfile', binding)
+          path(_name, '.rspec').render('dot_rspec', binding)
+          path(_name, 'boot.rb').render('boot.rb', binding)
+          path(_name, 'bin', 'crabfarm').render('crabfarm_bin', binding, 0755)
+          path(_name, 'app', 'parsers', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'app', 'states', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'app', 'helpers', '.gitkeep').render('dot_gitkeep')
+          path(_name, 'spec', 'spec_helper.rb').render('spec_helper.rb', binding)
+          path(_name, 'spec', 'snapshots', '.gitkeep').render('dot_gitkeep')
+        end
+      end
+
+      def generate_state(_name)
+        with_crawler_path do
+          binding = { state_class: _name.camelize }
+          path('app', 'states', _name.parameterize + '.rb').render('state.rb', binding)
+          path('spec', 'states', _name.parameterize + '_spec.rb').render('state_spec.rb', binding)
+        end
+      end
+
+      def generate_parser(_name)
+        with_crawler_path do
+          binding = { parser_class: _name.camelize }
+          path('app', 'parsers', _name.parameterize + '.rb').render('parser.rb', binding)
+          path('spec', 'parsers', _name.parameterize + '_spec.rb').render('parser_spec.rb', binding)
+        end
+      end
+
+      def with_external_path(_target)
+        @base_path = _target
+        yield
+      end
+
+      def with_crawler_path
+        if defined? CF_PATH
+          @base_path = CF_PATH
+          yield
+        else
+          puts "This command can only be run inside a crabfarm application"
+        end
+      end
+
+      def path(*_args)
+        @path = _args
+        self
+      end
+
+      def ensure
+        generate_dir([@base_path] + @path, false)
+        self
+      end
+
+      def render(_template, _binding={}, _mod=nil)
+        path = [@base_path] + @path
+        generate_dir(path[0..-2], true)
+        render_template(_template, _binding, path, _mod)
+        self
+      end
+
+
+
+  private
+
+      def generate_dir(_path, _silent)
+        path = File.join(*_path)
+        dir = Pathname.new path
+        unless dir.exist?
+          puts "Generating #{path}".color(:green)
+          dir.mkpath
+        else
+          puts "Skipping #{path}".color(:yellow) unless _silent
+        end
+      end
+
+      def render_template(_template, _binding, _path, _mod)
+        template = File.join(template_dir, _template) + '.erb'
+        output = File.join(*_path)
+
+        unless Pathname.new(output).exist?
+          puts "Rendering #{output}".color(:green)
+          File.open(output, "w") do |f|
+            f.write eval_template_with_hash(template, _binding)
+            f.chmod(_mod) unless _mod.nil?
+          end
+        else
+          puts "Skipping #{output}, already exists".color(:yellow)
+        end
+      end
+
+      def eval_template_with_hash(_path, _hash)
+        erb = ERB.new(File.read _path)
+        erb.result(OpenStruct.new(_hash).instance_eval { binding })
+      end
+
+      def template_dir
+        File.expand_path('../../templates', __FILE__)
+      end
+    end
+  end
+end

data/lib/crabfarm/modes/server.rb

@@ -0,0 +1,78 @@
+require 'grape'
+require 'crabfarm/support/custom_puma'
+require 'crabfarm/engines/safe_state_loop'
+
+module Crabfarm
+  module Modes
+    class Server
+
+      class API < Grape::API
+
+        MAX_WAIT = 60.0 * 5
+
+        format :json
+        prefix :api
+
+        rescue_from Grape::Exceptions::ValidationErrors do |e|
+          rack_response({ errors: e.as_json }.to_json, 400)
+        end
+
+        rescue_from Crabfarm::ApiError do |e|
+          rack_response(e.to_json.to_json, e.code)
+        end
+
+        helpers do
+          def evaluator
+            Server.evaluator
+          end
+
+          def wait
+            if params.has_key? :wait
+              [params[:wait].to_f, MAX_WAIT].min
+            else MAX_WAIT end
+          end
+
+          def print_state(_state)
+            {
+              name: _state.name,
+              params: _state.params,
+              doc: _state.doc
+            }
+          end
+        end
+
+        desc "Return the current crawler status."
+        params do
+          optional :wait, type: Float
+        end
+        get :state do
+          print_state evaluator.wait_for_state wait
+        end
+
+        desc "Change the crawler state"
+        params do
+          requires :name, type: String, desc: "Crawler state name"
+          optional :wait, type: Float
+        end
+        put :state do
+          print_state evaluator.change_state params[:name], params[:params], wait
+        end
+      end
+
+      def self.evaluator
+        @@evaluator
+      end
+
+      def self.start(_options)
+        @@evaluator = Engines::SafeStateLoop.new CF_LOADER
+        begin
+          Support::CustomPuma.run API, _options
+        ensure
+          @@evaluator.release
+        end
+      end
+
+    end
+  end
+end
+

data/lib/crabfarm/module_helper.rb

@@ -0,0 +1,35 @@
+require 'active_support'
+
+module Crabfarm
+  class ModuleHelper
+
+    attr_reader :dsl
+
+    def initialize(_module)
+      @module = _module
+    end
+
+    def settings
+      @module::CF_CONFIG
+    end
+
+    def load_state(_name)
+      load_entity _name, 'state', BaseState
+    end
+
+    def load_parser(_name)
+      load_entity _name, 'parser', BaseParser
+    end
+
+  private
+
+    def load_entity(_name, _role, _type)
+      name = _name.to_s.gsub(/[^A-Z0-9:]+/i, '_').camelize
+      mod = @module.const_get(name) rescue nil
+      raise EntityNotFoundError.new _role, name if mod.nil?
+      raise EntityNotFoundError.new _role, name unless mod < _type
+      mod
+    end
+
+  end
+end

data/lib/crabfarm/phantom_driver_factory.rb

@@ -0,0 +1,33 @@
+module Crabfarm
+  class PhantomDriverFactory
+
+    def initialize(_phantom, _config={})
+      @phantom = _phantom
+      @config = _config
+    end
+
+    def build_driver(_session_id)
+
+      # setup a custom client to use longer timeouts
+      client = Selenium::WebDriver::Remote::Http::Default.new
+      client.timeout = @config[:remote_timeout]
+
+      driver = Selenium::WebDriver.for :remote, {
+        :url => phantom_url,
+        :http_client => client,
+        :desired_capabilities => @config[:capabilities]
+      }
+
+      driver.send(:bridge).setWindowSize(@config[:window_width], @config[:window_height])
+
+      return driver
+    end
+
+  private
+
+    def phantom_url
+      "http://localhost:#{@phantom.port}"
+    end
+
+  end
+end

data/lib/crabfarm/phantom_runner.rb

@@ -0,0 +1,74 @@
+require 'net/http'
+
+module Crabfarm
+  class PhantomRunner
+
+    attr_reader :port
+
+    def initialize(_config={})
+      @config = _config;
+      @pid = nil
+    end
+
+    def start
+      find_available_port
+      @pid = Process.spawn({}, phantomjs_cmd)
+      wait_for_server
+    end
+
+    def stop
+      unless @pid.nil?
+        Process.kill("TERM", @pid)
+        Process.wait @pid
+        @pid = nil
+      end
+    end
+
+  private
+
+    def phantomjs_cmd
+      cmd = [@config[:bin_path]]
+      cmd << '--load-images=false' unless @config[:load_images]
+      cmd << "--proxy=#{@config[:proxy]}" unless @config[:proxy].nil?
+      cmd << "--webdriver=#{@port}"
+      cmd << "--ssl-protocol=#{@config[:ssl]}" unless @config[:ssl].nil?
+      cmd << "--ignore-ssl-errors=true"
+      cmd << "--webdriver-loglevel=NONE" # TODO: remove when log path is choosen
+      # cmd << "--webdriver-logfile=/path/to/log/phantom.log"
+      cmd.join(' ')
+    end
+
+    def find_available_port
+      with_lock do
+        server = TCPServer.new('127.0.0.1', 0)
+        @port = server.addr[1]
+        server.close
+      end
+    end
+
+    def wait_for_server
+      loop do
+        begin
+          # TODO: generate a valid request to prevent warnings
+          Net::HTTP.get_response(URI.parse("http://127.0.0.1:#{@port}"))
+          break
+        rescue
+        end
+      end
+    end
+
+    def with_lock
+      return yield if @config[:lock_file].nil?
+
+      File.open(@config[:lock_file], 'a+') do |file|
+        begin
+          file.flock File::LOCK_EX
+          return yield
+        ensure
+          file.flock File::LOCK_UN
+        end
+      end
+    end
+
+  end
+end

data/lib/crabfarm/rspec.rb

@@ -0,0 +1,39 @@
+CF_LOADER.load
+
+CF_TEST_CONTEXT = CF_LOADER.load_context
+CF_TEST_BUCKET = CF_TEST_CONTEXT.driver
+
+module Crabfarm
+  module RSpec
+
+    def parse(_snap_or_url, _options={})
+      fixture = Pathname.new(File.join(ENV['SNAPSHOT_DIR'], _snap_or_url))
+      if fixture.exist?
+        CF_TEST_BUCKET.get("file://#{fixture.realpath}")
+      else
+        CF_TEST_BUCKET.get(_snap_or_url)
+      end
+
+      CF_TEST_BUCKET.parse(described_class, _options)
+    end
+
+    def parser
+      @parser
+    end
+
+  end
+end
+
+RSpec.configure do |config|
+  config.include Crabfarm::RSpec
+
+  config.before(:example) do |example|
+    if example.metadata[:parsing]
+      @parser = parse example.metadata[:parsing], example.metadata[:using] || {}
+    end
+  end
+
+  config.after(:suite) do
+    CF_TEST_CONTEXT.release
+  end
+end