cangrejo 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b7efa941c84bc31f5de7d5b6da7e8d0a04167ed0
+  data.tar.gz: fca6d854265bc9eb67d7e85a6e13190e8ac0afa2
+SHA512:
+  metadata.gz: e0ae8d46ac60dd3b41144a95a1f4d0e1bc04b9aafddeeb927a5c942b7c65a5f71d94668282394a953bd5ee1ab6202d24520e25414c538f8fa0216fb2ea606d1c
+  data.tar.gz: 34065108e6e15336eda42d5fcbf543c537bccd581e4f0114cee0b3571a35973f5ab345cebf56d98a43be067944a10e27fca050f34e96f67072f89fc03816737e

data/lib/cangrejo.rb

@@ -0,0 +1,23 @@
+require "cangrejo/version"
+require "cangrejo/errors"
+require "cangrejo/configurator"
+require "cangrejo/session"
+
+module Cangrejo
+
+  @@config = OpenStruct.new({
+    crabfarm_host: 'http://www.crabfarm.com',
+    crawler_cache_path: 'tmp/crawler_cache',
+    temp_path: 'tmp',
+    hold_by_default: false,
+    crawlers: Hash.new
+  })
+
+  def self.config
+    @@config
+  end
+
+  def self.configure
+    yield Configurator.new @@config
+  end
+end

data/lib/cangrejo/configurator.rb

@@ -0,0 +1,26 @@
+module Cangrejo
+  class Configurator
+
+    [
+      :crabfarm_host,
+      :crawler_cache_path,
+      :temp_path,
+      :hold_by_default
+    ]
+    .each do |name|
+      define_method "set_#{name}" do |value|
+        @config.send("#{name}=", value)
+      end
+    end
+
+    def initialize(_config)
+      @config = _config
+    end
+
+    def crawler(_name, _options)
+      @config.crawlers[_name] = _options
+    end
+
+  end
+
+end

data/lib/cangrejo/errors.rb

@@ -0,0 +1,7 @@
+module Cangrejo
+
+	class Error < StandardError; end
+
+  class ConfigurationError < Error; end
+
+end

data/lib/cangrejo/modes/git.rb

@@ -0,0 +1,64 @@
+require "git"
+require "cangrejo/modes/local"
+
+module Cangrejo
+  module Modes
+    class Git < Local
+
+      def initialize(_url, _commit, _relative_path, _name=nil)
+        @url = _url
+        @commit = _commit
+        @relative_path = _relative_path
+        @needs_bundle = false
+        @name = _name || Digest::MD5.hexdigest(@url)
+        super deploy_path
+      end
+
+      def setup
+        ensure_repo_clone
+        ensure_repo_commit
+        ensure_deps
+        super
+      end
+
+    private
+
+      def deploy_path
+        if @relative_path.present?
+          File.join repo_path, @relative_path
+        else
+          repo_path
+        end
+      end
+
+      def cache_path
+        Cangrejo.config.crawler_cache_path
+      end
+
+      def repo_path
+        File.join cache_path, @name
+      end
+
+      def ensure_repo_clone
+        unless File.exists? File.join(repo_path, '.git')
+          ::Git.clone(@url, @name, :path => cache_path)
+          @needs_bundle = true
+        end
+      end
+
+      def ensure_repo_commit
+         g = ::Git.open repo_path
+         if g.log.first.sha != @commit
+          g.fetch
+          g.checkout @commit
+          @needs_bundle = true
+         end
+      end
+
+      def ensure_deps
+        %x[cd '#{deploy_path}' && bundle install] if @needs_bundle
+        @needs_bundle = false
+      end
+    end
+  end
+end

data/lib/cangrejo/modes/local.rb

@@ -0,0 +1,35 @@
+require "cangrejo/restclient/request_extensions"
+require "cangrejo/restclient/json_resource"
+require "cangrejo/support/launcher"
+
+module Cangrejo
+  module Modes
+    class Local
+
+      def initialize(_path)
+        @path = _path
+      end
+
+      def setup
+        init_launcher
+        init_rest_client
+      end
+
+      def release
+        @launcher.kill
+      end
+
+    private
+
+      def init_launcher
+        @launcher = Support::Launcher.new @path
+        @launcher.launch
+      end
+
+      def init_rest_client
+        RestClient::JsonResource.new Net::SocketUri.new(@launcher.host, '/api/state')
+      end
+
+    end
+  end
+end

data/lib/cangrejo/modes/remote.rb

@@ -0,0 +1,33 @@
+require "cangrejo/restclient/json_resource"
+
+module Cangrejo
+  module Modes
+    class Remote
+
+      def initialize(_name)
+        @name = _name
+      end
+
+      def setup
+        sessions = prepare_resource "api/crawlers/#{@name}/sessions"
+        sessions.post({}.to_json)
+        return prepare_resource "api/sessions/#{sessions.id}"
+      end
+
+      def release
+        # nothing
+      end
+
+    private
+
+      def prepare_resource(_path)
+        RestClient::JsonResource.new URI.join(remote_host, _path)
+      end
+
+      def remote_host
+        Cangrejo.config.crabfarm_host
+      end
+
+    end
+  end
+end

data/lib/cangrejo/net/socket_http.rb

@@ -0,0 +1,24 @@
+module Net
+  #  Overrides the connect method to simply connect to a unix domain socket.
+  class SocketHttp < HTTP
+    attr_reader :socket_path
+
+    #  URI should be a relative URI giving the path on the HTTP server.
+    #  socket_path is the filesystem path to the socket the server is listening to.
+    def initialize(_socket_path, _dummy=nil)
+      @socket_path = _socket_path
+      super
+    end
+
+    #  Create the socket object.
+    def connect
+      @socket = Net::BufferedIO.new UNIXSocket.new socket_path
+      on_connect
+    end
+
+    #  Override to prevent errors concatenating relative URI objects.
+    def addr_port
+      File.basename(socket_path)
+    end
+  end
+end

data/lib/cangrejo/net/socket_uri.rb

@@ -0,0 +1,46 @@
+module Net
+  class SocketUri
+
+    def initialize(_socket_path, _path)
+      @socket_path = _socket_path[7..-1]
+      @path = _path
+    end
+
+    def hostname
+      @socket_path
+    end
+
+    def host
+      @socket_path
+    end
+
+    def request_uri
+      @path
+    end
+
+    def empty?
+      false
+    end
+
+    def port
+      nil
+    end
+
+    def path
+      @path
+    end
+
+    def user
+      nil
+    end
+
+    def password
+      nil
+    end
+
+    def to_s
+      "#{@socket_path}::@path"
+    end
+
+  end
+end

data/lib/cangrejo/restclient/json_resource.rb

@@ -0,0 +1,38 @@
+require "rest_client"
+
+module RestClient
+
+  class JsonResource < Resource
+
+    def get(additional_headers={}, &block)
+      additional_headers['Accept'] = 'json'
+      r = super additional_headers, &block
+      @state = JSON.parse r
+    end
+
+    def post(payload, additional_headers={}, &block)
+      r = super payload.to_json, decorate_headers(additional_headers), &block
+      @state = JSON.parse r
+    end
+
+    def put(payload, additional_headers={}, &block)
+      r = super payload.to_json, decorate_headers(additional_headers), &block
+      @state = JSON.parse r
+    end
+
+  private
+
+    def decorate_headers(_headers)
+      _headers['Content-Type'] = 'application/json'
+      _headers
+    end
+
+    def method_missing(_method, *_args, &_block)
+      if @state.is_a? Hash and @state.has_key? _method.to_s
+        @state[_method.to_s]
+      else super end
+    end
+
+  end
+
+end

data/lib/cangrejo/restclient/request_extensions.rb

@@ -0,0 +1,27 @@
+require "cangrejo/net/socket_http"
+require "cangrejo/net/socket_uri"
+require "rest_client"
+
+module RestClient
+  module RequestExtensions
+
+    def process_url_params(_url, _headers)
+      if _url.is_a? String then super(_url, _headers) else _url end
+    end
+
+    def parse_url(_url)
+      if _url.is_a? String then super(_url) else _url end
+    end
+
+    def net_http_class
+      if url.is_a? Net::SocketUri then
+        Net::SocketHttp
+      else super end
+    end
+
+  end
+
+  class Request
+    prepend RequestExtensions
+  end
+end

data/lib/cangrejo/session.rb

@@ -0,0 +1,62 @@
+require "ostruct"
+require "cangrejo/modes/remote"
+require "cangrejo/modes/git"
+require "cangrejo/modes/local"
+
+module Cangrejo
+  class Session
+    include Forwardable
+
+    attr_reader :doc
+
+    def initialize(_name, _options={})
+      @name = _name
+      options = Cangrejo.config.crawlers.fetch(_name, {}).merge _options
+      select_mode options
+      start unless _options.fetch :hold, Cangrejo.config.hold_by_default
+    end
+
+    def start
+      raise ConfigurationError.new 'Session already started' unless @rest.nil?
+      @rest = @mode.setup
+      self
+    end
+
+    def state_name
+      @rest.name
+    end
+
+    def state_params
+      @rest.params
+    end
+
+    def raw_doc
+      @rest.doc
+    end
+
+    def crawl(_state, _params={})
+      raise ConfigurationError.new 'Session not started' if @rest.nil?
+      @rest.put(name: _state, params: _params)
+      @doc = OpenStruct.new @rest.doc
+      self
+    end
+
+    def release
+      @mode.release
+      self
+    end
+
+  private
+
+    def select_mode(_options)
+      @mode = if _options.has_key? :path
+        Modes::Local.new _options[:path]
+      elsif _options.has_key? :git_remote
+        Modes::Git.new _options[:git_remote], _options[:git_commit], _options[:relative_path], @name
+      else
+        Modes::Remote.new @name
+      end
+    end
+
+  end
+end

data/lib/cangrejo/support/launcher.rb

@@ -0,0 +1,43 @@
+module Cangrejo
+  module Support
+    class Launcher
+
+      def initialize(_path)
+        @path = _path
+        select_socket_file
+      end
+
+      def host
+        "unix://#{@socket_file}"
+      end
+
+      def launch
+        gem_path = File.join(@path, 'Gemfile')
+        # TODO: for some reason, the gemfile path must be specified here, maybe because of rbenv?
+        @pid = Process.spawn({ 'BUNDLE_GEMFILE' => gem_path }, "bin/crabfarm s --host #{host}", chdir: @path)
+        wait_for_socket
+      end
+
+      def kill
+        unless @pid.nil?
+          Process.kill 'INT', @pid
+          Process.wait @pid
+        end
+      end
+
+    private
+
+      def select_socket_file
+        @socket_file = random_filename while @socket_file.nil? or File.exist? @socket_file
+      end
+
+      def random_filename
+        File.join(Cangrejo.config.temp_path, "csocket-#{Random.rand(1000000)}.sock")
+      end
+
+      def wait_for_socket
+        sleep 0.1 while not File.exist? @socket_file
+      end
+    end
+  end
+end

data/lib/cangrejo/version.rb

@@ -0,0 +1,3 @@
+module Cangrejo
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: cangrejo
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ignacio Baixas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- ignacio@platan.us
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/cangrejo/configurator.rb
+- lib/cangrejo/errors.rb
+- lib/cangrejo/modes/git.rb
+- lib/cangrejo/modes/local.rb
+- lib/cangrejo/modes/remote.rb
+- lib/cangrejo/net/socket_http.rb
+- lib/cangrejo/net/socket_uri.rb
+- lib/cangrejo/restclient/json_resource.rb
+- lib/cangrejo/restclient/request_extensions.rb
+- lib/cangrejo/session.rb
+- lib/cangrejo/support/launcher.rb
+- lib/cangrejo/version.rb
+- lib/cangrejo.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Crabfarm client for ruby
+test_files: []