fetch 0.0.0 → 0.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b9c344461a2663701702dd9fda6ce2429671106
-  data.tar.gz: aa57e886b43146a90c724251e3b96e76d792cc52
+  metadata.gz: 0596dd2fcaa4c9e3be34f29b5346794cd98cfb1e
+  data.tar.gz: cf9479fe7e9c26b350efa01d64b5e880bdac6aa4
 SHA512:
-  metadata.gz: 40d701ecdcd26c12c0ed34a8a9c50cd93cc8aa4fd4907e6bb87187fa5595a9b7777b411c71bc5cf8ba9344b10e2b348cec4051586cf7a288c6a5e4c43ce2b4b7
-  data.tar.gz: af7e724a8df6864e55bf46c098c8a7b7415faa74e128d0962b0e7f70eb28be8438011d794b3fd65578643ae970c225b552223d63c85e9ecf03e51ecd6b297fc3
+  metadata.gz: 34308fe5ee23a51d4974823f61020f2db9bb6b77966acbd9c72e41170523b27be4bb559b556e71c15899b5cd6b215d905c2c1489f0f110ed78a93b988c643f3e
+  data.tar.gz: f2004793493fd8bdefab68bcc50178a6519a205e768f3e2e7bdddc6757dd85408b0ab94f1e961208a574c7cccbaee4bf21ce62c0fa26e00e8c5f01c9b0c26560

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+*.log

data/README.md

@@ -1,28 +1,41 @@
-[![Build Status](https://secure.travis-ci.org/lassebunk/fetch.png)](http://travis-ci.org/lassebunk/fetch)
+[![Build Status](https://secure.travis-ci.org/bogrobotten/fetch.png)](http://travis-ci.org/bogrobotten/fetch)
 
-# Fetch!
+# Fetch
 
 ![Fetch](http://i.imgur.com/B8TXlri.png)
 
+Fetch enables easy fetching of data from multiple web sources.
+It was extracted from [Bogrobotten](http://www.bogrobotten.dk) where we use it
+to fetch prices and other stuff from multiple merchants.
+We use it for price comparison, but you can use it for anything that involves
+fetching data from external sources.
+
+Fetch uses the [Typhoeus](https://github.com/typhoeus/typhoeus) gem for fast
+and reliable asynchronous fetches from multiple URLs.
+
 ## Installation
 
 Add this line to your application's *Gemfile*:
 
-    gem "fetch"
+```ruby
+gem "fetch"
+```
 
 Then run:
 
-    $ bundle
-
-Or install it yourself:
-
-    $ gem install fetch
+```bash
+$ bundle
+```
 
 ## Contributing
 
+Contributions are much appreciated. To contribute:
+
 1. Fork the project
 2. Create a feature branch (`git checkout -b my-new-feature`)
 3. Make your changes, including tests so it doesn't break in the future
 4. Commit your changes (`git commit -am 'Add feature'`)
 5. Push to the branch (`git push origin my-new-feature`)
 6. Create new pull request
+
+Please do not touch the version, as this will be updated by the owners when the gem is ready for a new release.

data/Rakefile

@@ -3,7 +3,7 @@ require "rake/testtask"
 
 Rake::TestTask.new do |t|
   t.libs << "test"
-  t.test_files = FileList["test/*_test.rb"]
+  t.test_files = FileList["test/**/*_test.rb"]
   t.verbose = true
 end
 

data/fetch.gemspec

@@ -3,20 +3,23 @@ lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'fetch/version'
 
-Gem::Specification.new do |spec|
-  spec.name          = "fetch"
-  spec.version       = Fetch::VERSION
-  spec.authors       = ["Lasse Bunk"]
-  spec.email         = ["lassebunk@gmail.com"]
-  spec.summary       = %q{Coming}
-  spec.description   = %q{Coming}
-  spec.homepage      = "https://github.com/lassebunk/fetch"
-  spec.license       = "MIT"
+Gem::Specification.new do |s|
+  s.name          = "fetch"
+  s.version       = Fetch::VERSION
+  s.authors       = ["Lasse Bunk"]
+  s.email         = ["lassebunk@gmail.com"]
+  s.summary       = %q{Coming}
+  s.description   = %q{Coming}
+  s.homepage      = "https://github.com/lassebunk/fetch"
+  s.license       = "MIT"
 
-  spec.files         = `git ls-files`.split($/)
-  spec.test_files    = spec.files.grep(%r{^test/})
-  spec.require_paths = ["lib"]
+  s.files         = `git ls-files`.split($/)
+  s.test_files    = s.files.grep(%r{^test/})
+  s.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.5"
-  spec.add_development_dependency "rake"
+  s.add_dependency "typhoeus", ">= 0.6.0"
+  s.add_development_dependency "json"
+  s.add_development_dependency "minitest",  ">= 5.4"
+  s.add_development_dependency "webmock",   ">= 1.20"
+  s.add_development_dependency "rake"
 end

data/lib/fetch.rb

@@ -1 +1,49 @@
-require "fetch/version"
+require "typhoeus"
+
+%w{
+  version
+  callbacks
+  base
+  request
+  async
+  simple
+  module
+  backend
+  configuration
+}.each do |file|
+  require "fetch/#{file}"
+end
+
+module Fetch
+  class HttpError < StandardError
+    attr_reader :code, :url
+
+    def initialize(code, url)
+      @code, @url = code, url
+    end
+
+    def message
+      "HTTP Error #{code}: #{url}"
+    end
+  end
+
+  class << self
+    # Returns a configuration object.
+    def config
+      @config ||= Configuration.new
+    end
+
+    # Yields a configuration block (+Fetch::Configuration+).
+    #
+    #   Fetch.configure do |config|
+    #     config.user_agent = "Custom User Agent"
+    #   end
+    def configure(&block)
+      yield config
+    end
+
+    def module_cache
+      @module_cache ||= ModuleCache.new
+    end
+  end
+end

data/lib/fetch/async.rb

@@ -0,0 +1,21 @@
+module Fetch
+  module Async
+    def self.included(base)
+      base.define_callback :request,
+                           :before_process,
+                           :after_process
+    end
+
+    def requests
+      self.class.callbacks[:request].map do |callback|
+        Request.new.tap do |req|
+          req.before_process { before_process } if callback?(:before_process)
+          req.after_process { after_process } if callback?(:after_process)
+          req.failure { |code, url| failure(code, url) } if callback?(:failure)
+          req.error { |e| error(e) } if callback?(:error)
+          instance_exec(req, &callback)
+        end
+      end.select(&:url)
+    end
+  end
+end

data/lib/fetch/backend.rb

@@ -0,0 +1,2 @@
+require "fetch/backend/base"
+require "fetch/backend/typhoeus"

data/lib/fetch/backend/base.rb

@@ -0,0 +1,15 @@
+module Fetch
+  module Backend
+    class Base
+      attr_reader :requests
+
+      def initialize(requests)
+        @requests = requests
+      end
+
+      def run(&progress)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/fetch/backend/typhoeus.rb

@@ -0,0 +1,43 @@
+module Fetch
+  module Backend
+    class Typhoeus < Base
+      def run(&progress)
+        hydra = ::Typhoeus::Hydra.new
+
+        build_requests(&progress).each do |request|
+          hydra.queue(request)
+        end
+
+        hydra.run
+      end
+
+      private
+
+      def build_requests(&progress)
+        requests.map do |req|
+          request = ::Typhoeus::Request.new(
+            req.url,
+            method: req.method,
+            body: req.body_string,
+            followlocation: req.follow_redirects,
+            timeout: req.timeout,
+            forbid_reuse: true,
+            headers: req.headers
+          )
+
+          request.on_success do |res|
+            req.process!(res.body, req.url, res.effective_url)
+            progress.call
+          end
+
+          request.on_failure do |res|
+            req.failed!(res.code, req.url)
+            progress.call
+          end
+
+          request
+        end
+      end
+    end
+  end
+end

data/lib/fetch/base.rb

@@ -0,0 +1,64 @@
+# Base module for fetch handlers, e.g. +ProductFetch+, +UserFetch+, etc.
+module Fetch
+  class Base
+    include Callbacks
+
+    # Set callbacks to be called when fetching.
+    #
+    #   before_fetch do
+    #     # do something before fetching
+    #   end
+    #
+    #   after_fetch do
+    #     # do something after fetching
+    #   end
+    #
+    #   progress do |progress|
+    #     # update progress in percent
+    #   end
+    define_callback :modules,
+                    :init,
+                    :before_fetch,
+                    :after_fetch,
+                    :progress
+
+    # Begin fetching.
+    # Will run synchronous fetches first and async fetches afterwards.
+    # Updates progress when each module finishes its fetch.
+    def fetch
+      requests = instantiate_modules.select(&:fetch?).map(&:requests).flatten
+
+      total, done = requests.size, 0
+      update_progress(total, done)
+
+      before_fetch
+
+      backend.new(requests).run do
+        update_progress(total, done += 1)
+      end
+
+      after_fetch
+
+      true
+    end
+
+    private
+
+    # Array of instantiated fetch modules.
+    def instantiate_modules
+      Array(modules).map do |klass|
+        init(klass) || klass.new
+      end
+    end
+
+    # Updates progress with a percentage calculated from +total+ and +done+.
+    def update_progress(total, done)
+      percentage = total.zero? ? 100 : ((done.to_f / total) * 100).to_i
+      progress(percentage)
+    end
+
+    def backend
+      Fetch.config.backend
+    end
+  end
+end

data/lib/fetch/callbacks.rb

@@ -0,0 +1,62 @@
+module Fetch
+  module Callbacks
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    private
+
+    # Check if a callback has been used.
+    def callback?(name)
+      self.class.callbacks[name].any?
+    end
+
+    # Run specific callbacks.
+    #
+    #   run_callbacks_for(:before_fetch)
+    #   run_callbacks_for(:progress, 12) # 12 percent done
+    def run_callbacks_for(name, *args)
+      self.class.callbacks[name].map do |block|
+        instance_exec(*args, &block)
+      end
+    end
+
+    module ClassMethods
+      # Hash of callback blocks to be called.
+      def callbacks
+        @callbacks ||= Hash.new { |h, k| h[k] = [] }
+      end
+
+      # Defines callback methods on the class level.
+      def define_callback(*names)
+        names.each do |name|
+          define_singleton_method name do |*values, &block|
+            create_callback_for(name, *values, &block)
+          end
+
+          define_method name do |*args|
+            run_callbacks_for(name, *args).last
+          end
+        end
+      end
+
+      def inherited(base)
+        super
+        callbacks.each do |name, callbacks|
+          base.callbacks[name] = callbacks.dup
+        end
+      end
+
+      private
+
+      def create_callback_for(name, *values, &block)
+        add_callback(name) { values } if values.any?
+        add_callback(name, &block) if block
+      end
+
+      def add_callback(name, &block)
+        callbacks[name] << block
+      end
+    end
+  end
+end

data/lib/fetch/configuration.rb

@@ -0,0 +1,30 @@
+module Fetch
+  class Configuration
+    DEFAULTS = {
+      user_agent:     "Mozilla/5.0",
+      timeout:        10,
+      namespaces:     ["fetch_sources"],
+      raise_on_error: -> { defined?(Rails.env) && %w{development test}.include?(Rails.env) },
+      backend:        Backend::Typhoeus
+    }
+
+    DEFAULTS.each do |option, value|
+      ivar = "@#{option}"
+
+      define_method(option) do
+        return instance_variable_get(ivar) if instance_variable_defined?(ivar)
+        value = value.call if value.is_a?(Proc)
+        instance_variable_set(ivar, value)
+      end
+
+      define_method("#{option}=") do |value|
+        instance_variable_set(ivar, value)
+      end
+    end
+
+    # Convenience method for defining a single namespace that contains fetch modules.
+    def namespace=(value)
+      self.namespaces = [value]
+    end
+  end
+end

data/lib/fetch/module.rb

@@ -0,0 +1,17 @@
+module Fetch
+  class Module
+    include Callbacks
+    include Async
+
+    define_callback :fetch_if,
+                    :failure,
+                    :error
+
+    # Whether or not the module should be used when fetching.
+    # Set with `fetch_if do ... end`.
+    def fetch?
+      return true unless callback?(:fetch_if)
+      !!fetch_if
+    end
+  end
+end

data/lib/fetch/request.rb

@@ -0,0 +1,156 @@
+require "cgi"
+
+module Fetch
+  # A request to be completed with Typhoeus.
+  class Request
+    # Initializes the request and sets properties to the values defined in
+    # +options+.
+    #
+    #   request = Fetch::Request.new("http://www.google.com", timeout: 5)
+    #   request.url     # => "http://www.google.com"
+    #   request.timeout # => 5
+    #
+    #   request = Fetch::Request.new(timeout: 5)
+    #   request.url     # => nil
+    #   request.timeout # => 5
+    def initialize(*args)
+      options = args.pop if args.last.is_a?(Hash)
+
+      if args.any?
+        self.url = args.first
+      end
+
+      if options
+        options.each { |key, value| send("#{key}=", value) }
+      end
+    end
+
+    # The URL to be requested.
+    attr_accessor :url
+
+    # Whether to follow redirects. Default: +true+
+    def follow_redirects
+      return @follow_redirects if defined?(@follow_redirects)
+      @follow_redirects = true
+    end
+
+    # Sets whether to follow redirects.
+    attr_writer :follow_redirects
+
+    # The method to be used for the request.
+    def method
+      @method || :get
+    end
+
+    # Sets the method to be used for the request.
+    attr_writer :method
+
+    # The post body to be sent with the request.
+    def body
+      @body ||= {}
+    end
+
+    # Sets the post body to be sent with the request.
+    attr_writer :body
+
+    # The post body represented as a string.
+    def body_string
+      body.map { |k, v| "#{CGI::escape(k.to_s)}=#{CGI::escape(v.to_s)}" }.join("&")
+    end
+
+    # The timeout for the request.
+    # Default: Taken from +Fetch.config.timeout+
+    def timeout
+      return @timeout if defined?(@timeout)
+      Fetch.config.timeout
+    end
+
+    # Sets the timeout for the request.
+    attr_writer :timeout
+
+    # The headers to be sent with the request.
+    def headers
+      @headers ||= {
+        "User-Agent" => Fetch.config.user_agent
+      }
+    end
+
+    # Sets the headers to be sent with the request.
+    attr_writer :headers
+
+    # The user agent being sent with the request.
+    def user_agent
+      headers["User-Agent"]
+    end
+
+    # Sets the user agent to be sent with the request.
+    def user_agent=(value)
+      headers.merge! "User-Agent" => value
+    end
+
+    # Sets a callback to be run before each process.
+    def before_process(&block)
+      raise "You must supply a block to #{self.class.name}#before_process" unless block
+      @before_process_callback = block
+    end
+
+    # Runs the before process callback.
+    def before_process!
+      @before_process_callback.call if @before_process_callback
+    end
+
+    # Sets the callback to be run when the request completes.
+    def process(&block)
+      raise "You must supply a block to #{self.class.name}#process" unless block
+      @process_callback = block
+    end
+
+    # Runs the process callback. If it fails with an exception, it will send
+    # the exception to the error callback.
+    def process!(body, url, effective_url)
+      before_process!
+      @process_callback.call(body, url, effective_url) if @process_callback
+      after_process!
+    rescue => e
+      error!(e)
+    end
+
+    # Sets a callback to be run after each process.
+    def after_process(&block)
+      raise "You must supply a block to #{self.class.name}#after_process" unless block
+      @after_process_callback = block
+    end
+
+    # Runs the after process callback.
+    def after_process!
+      @after_process_callback.call if @after_process_callback
+    end
+
+    # Sets the callback to be run if a request fails.
+    def failure(&block)
+      raise "You must supply a block to #{self.class.name}#failure" unless block
+      @failure_callback = block
+    end
+
+    # Runs the failure callback.
+    def failed!(code, url)
+      @failure_callback.call(code, url) if @failure_callback
+    end
+
+    # Sets the callback to be run if the processing fails due to an exception.
+    def error(&block)
+      raise "You must supply a block to #{self.class.name}#error" unless block
+      @error_callback = block
+    end
+
+    # Runs the error callback. Raises the exception given in +exception+ if an
+    # error callback isn't defined.
+    def error!(exception)
+      if @error_callback
+        @error_callback.call(exception)
+      else
+        raise exception
+      end
+    end
+  end
+end