fetch 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0596dd2fcaa4c9e3be34f29b5346794cd98cfb1e
-  data.tar.gz: cf9479fe7e9c26b350efa01d64b5e880bdac6aa4
+  metadata.gz: 1f2e34b25ea8efe2d5681328da56ba8d5141775b
+  data.tar.gz: 582f71ff59b4e30b02733a5f777096e4ac785aad
 SHA512:
-  metadata.gz: 34308fe5ee23a51d4974823f61020f2db9bb6b77966acbd9c72e41170523b27be4bb559b556e71c15899b5cd6b215d905c2c1489f0f110ed78a93b988c643f3e
-  data.tar.gz: f2004793493fd8bdefab68bcc50178a6519a205e768f3e2e7bdddc6757dd85408b0ab94f1e961208a574c7cccbaee4bf21ce62c0fa26e00e8c5f01c9b0c26560
+  metadata.gz: 750a2cbaca8debe5512947da70ae5c01a35eb72e71873390013f059ce9694d7777fc3978d224b4292c7039900673c171842438df1c3c15524f584e64f03b3bd3
+  data.tar.gz: 1da726ec7708099401e988d66702f37d4b49ba90b0d80be9dcbdfe667808d49f16e1e37b361a9b04c890b9eb910b4c9e7512269325db9b25ba648d5d60957439

data/CHANGELOG.md

@@ -0,0 +1,14 @@
+# Changelog
+
+## Version 0.0.2
+
+* Sends fetchable to fetch modules by default.
+* The `init` callback only runs once.
+* Adds a `defaults` callback for setting up requests.
+* Adds a `parse` callback for parsing response bodies before they are processed.
+* Adds a `Fetch::JSON` module for automatic JSON parsing.
+* Adds a `load` callback for loading fetch modules.
+
+## Version 0.0.1
+
+* Initial release

data/README.md

@@ -27,6 +27,319 @@ Then run:
 $ bundle
 ```
 
+## Example
+
+In *app/models/user.rb*:
+
+```ruby
+class User < ActiveRecord::Base
+  def fetcher
+    @fetcher ||= UserFetcher.new(self)
+  end
+end
+```
+
+In *app/fetchers/user_fetcher.rb*:
+
+```ruby
+class UserFetcher < Fetch::Base
+  modules Facebook::UserInfoFetch,
+          Github::UserInfoFetch
+end
+```
+
+In *lib/facebook/user_info_fetch.rb*:
+
+```ruby
+module Facebook
+  class UserInfoFetch < Fetch::Module
+    include Fetch::Simple
+    include Fetch::JSON
+
+    url do
+      "http://graph.facebook.com/#{fetchable.login}"
+    end
+
+    process do |user_info|
+      fetchable.update_attribute :facebook_id, user_info["id"]
+    end
+  end
+end
+```
+
+In *lib/github/user_info_fetch.rb*
+
+```ruby
+module Github
+  class UserInfoFetch < Fetch::Module
+    include Fetch::JSON
+
+    # Request for user ID
+    request do |req|
+      req.url = "https://api.github.com/users/#{fetchable.login}"
+      req.process do |user|
+        fetchable.update_attribute :github_id, user["id"]
+      end
+    end
+
+    # Request for repos
+    request do |req|
+      req.url = "https://api.github.com/users/#{fetchable.login}/repos"
+      req.process do |repos|
+        repo_names = repos.map { |r| r["name"] }
+        fetchable.update_attribute :github_repos, repo_names
+      end
+    end
+  end
+end
+```
+
+Then, when everything is set up, you can do:
+
+```ruby
+user = User.find(123)
+user.fetcher.fetch
+```
+
+This will run three requests – one for Facebook and two for GitHub – and update
+the user model with a Facebook user ID, a GitHub user ID, and a list of GitHub
+repos.
+
+## Good to know
+
+### Adding defaults to your requests
+
+Each fetch module has a `defaults` callback that you can use to set up defaults
+for all requests in that modules.
+
+```ruby
+class UserInfoFetch < Fetch::Module
+  defaults do |req|
+    req.user_agent = "My Awesome Bot!"
+  end
+
+  request do |req|
+    req.url = "http://test.com"
+    req.process do |body|
+      # Do some processing
+    end
+  end
+end
+```
+
+This will add the user agent `My Awesome Bot!` to all requests in the
+`UserInfoFetch` module.
+
+The `defaults` callback is inherited, like all other callbacks, so if you have
+a base fetch class that you subclass, the `defaults` callback in the superclass
+will be run in all subclasses.
+
+### Handling HTTP failures
+
+HTTP failures can be handled using the `failure` callback. If you want to
+handle failures for all requests generally, you can use the module-wide
+`failure` callback:
+
+```ruby
+class UserInfoFetch < Fetch::Module
+  request do |req|
+    req.url = "http://test.com/something-failing"
+    req.process do |body|
+      # Do something if successful.
+    end
+  end
+
+  failure do |code, url|
+    Rails.logger.info "Fetching from #{url} failed: #{code}"
+  end
+end
+```
+
+If you want to handle failures on the specific requests instead:
+
+```ruby
+class UserInfoFetch < Fetch::Module
+  request do |req|
+    req.url = "http://test.com/something-failing"
+    req.process do |body|
+      # Do something if successful.
+    end
+    req.failure do |code, url|
+      # Handle the failure
+    end
+  end
+end
+```
+
+When you handle failures directly on the request, the general `failure`
+callback isn't called.
+
+**Note:** If you don't specify a `failure` callback at all, HTTP failures are ignored,
+and processing skipped for the failed request.
+
+### Handling errors
+
+Sometimes a URL will return something that potentially makes your processing
+code fail. To prevent this from breaking your whole fetch, you can handle
+errors using the `error` callback:
+
+```ruby
+class UserInfoFetch < Fetch::Module
+  request do |req|
+    req.url = "http://test.com/something-failing"
+    req.process do |body|
+      # Do something if successful.
+    end
+  end
+
+  error do |exception|
+    Rails.logger.info "An error occured: #{exception.message}\n" +
+                      exception.backtrace.join("\n")
+    raise exception if ["development", "test"].include?(Rails.env)
+  end
+end
+```
+
+You can also do it directly on the requests:
+
+```ruby
+class UserInfoFetch < Fetch::Module
+  request do |req|
+    req.url = "http://test.com/something-failing"
+    req.process do |body|
+      # Do something if successful.
+    end
+    req.error do |exception|
+      # Handle the error
+    end
+  end
+end
+```
+
+If you handle errors directly on the requests, the general `error` callback
+isn't run.
+
+**Note:** If you don't do any error handling in one of the two ways shown
+above, any exceptions that occur when processing will be raised, causing the
+whole fetch to fail. So please add error handling :blush:
+
+### Parsing JSON
+
+Fetch has a module for automatically parsing the request body as JSON before
+it is sent to the process block.
+
+```ruby
+class UserInfoFetch < Fetch::Module
+  include Fetch::JSON
+
+  request do |req|
+    req.url = "http://api.test.com/user"
+    req.process do |json|
+      # Do something with the JSON.
+    end
+  end
+end
+```
+
+### Dynamically loading fetch modules
+
+You can load fetch modules dynamically using the `load` callback. Normally, the
+modules defined with `modules` are instantiated directly. When you use the
+`load` callback, this will determine how your modules are loaded.
+
+```ruby
+class UserFetcher < Fetch::Base
+  modules :user_info_fetch, :status_fetch
+
+  load do |modules|
+    namespaces.product(modules).map do |path|
+      path.join("/").camelize.safe_constantize
+    end.compact
+  end
+
+  private
+
+  def namespaces
+    [:github, :facebook]
+  end
+end
+```
+
+This will load the modules `Github::UserInfoFetch`, `Github::StatusFetch`,
+`Facebook::UserInfoFetch` and `Facebook::StatusFetch`, if they are present.
+
+The `load` callback is only run once, so you can safely inherit it – only the
+last one defined will be run.
+
+### Initializing fetch modules
+
+Normally, a fetcher is initialized with an optional `fetchable` that is sent
+along to the fetch modules when they are initialized. You can change how this
+works with the `init` callback.
+
+Let's say you have a `Search` model with a `SearchFetcher` that gets results
+from various search engines. Normally, the `Search` instance would be sent to
+the fetch modules as a fetchable. Let's say you just want to send the keyword
+to reduce coupling.
+
+In *app/fetchers/search_fetcher.rb*:
+
+```ruby
+class SearchFetcher < Fetch::Base
+  modules Google::KeywordFetch,
+          Bing::KeywordFetch
+
+  init do |klass|
+    klass.new(fetchable.keyword)
+  end
+end
+```
+
+In *lib/base/keyword_fetch.rb*:
+
+```ruby
+module Base
+  class KeywordFetch < Fetch::Module
+    attr_reader :keyword
+
+    def initialize(keyword)
+      @keyword = keyword
+    end
+  end
+end
+```
+
+In *lib/google/keyword_fetch.rb*:
+
+```ruby
+module Google
+  class KeywordFetch < Base::KeywordFetch
+    request do |req|
+      req.url = "https://www.google.com/search?q=#{CGI::escape(keyword)}"
+      req.process do |body|
+        # Do something with the body.
+      end
+    end
+  end
+end
+```
+
+And *lib/bing/keyword_fetch.rb* something similar to Google.
+
+Then:
+
+```ruby
+search = Search.find(123)
+SearchFetcher.new(search).fetch
+```
+
+Now the keyword will be sent to the fetch modules instead of the fetchable.
+
+## Changelog
+
+See the [changelog](https://github.com/bogrobotten/fetch/blob/master/CHANGELOG.md)
+for changes in the different versions.
+
 ## Contributing
 
 Contributions are much appreciated. To contribute:

data/lib/fetch.rb

@@ -7,6 +7,7 @@ require "typhoeus"
   request
   async
   simple
+  json
   module
   backend
   configuration

data/lib/fetch/async.rb

@@ -2,17 +2,22 @@ module Fetch
   module Async
     def self.included(base)
       base.define_callback :request,
+                           :defaults,
                            :before_process,
                            :after_process
+
+      base.defaults do |req|
+        req.before_process { before_process } if callback?(:before_process)
+        req.after_process { after_process } if callback?(:after_process)
+        req.failure { |code, url| failure(code, url) } if callback?(:failure)
+        req.error { |e| error(e) } if callback?(:error)
+      end
     end
 
     def requests
       self.class.callbacks[:request].map do |callback|
         Request.new.tap do |req|
-          req.before_process { before_process } if callback?(:before_process)
-          req.after_process { after_process } if callback?(:after_process)
-          req.failure { |code, url| failure(code, url) } if callback?(:failure)
-          req.error { |e| error(e) } if callback?(:error)
+          defaults(req)
           instance_exec(req, &callback)
         end
       end.select(&:url)

data/lib/fetch/base.rb

@@ -17,11 +17,16 @@ module Fetch
     #     # update progress in percent
     #   end
     define_callback :modules,
+                    :load,
                     :init,
                     :before_fetch,
                     :after_fetch,
                     :progress
 
+    def initialize(fetchable = nil)
+      @fetchable = fetchable
+    end
+
     # Begin fetching.
     # Will run synchronous fetches first and async fetches afterwards.
     # Updates progress when each module finishes its fetch.
@@ -44,10 +49,15 @@ module Fetch
 
     private
 
+    # The optional instance being fetched.
+    attr_reader :fetchable
+
     # Array of instantiated fetch modules.
     def instantiate_modules
-      Array(modules).map do |klass|
-        init(klass) || klass.new
+      mods = Array(modules)
+      mods = load!(mods) if callback?(:load)
+      Array(mods).map do |klass|
+        init!(klass) || klass.new(fetchable)
       end
     end
 

data/lib/fetch/callbacks.rb

@@ -17,10 +17,20 @@ module Fetch
     #   run_callbacks_for(:progress, 12) # 12 percent done
     def run_callbacks_for(name, *args)
       self.class.callbacks[name].map do |block|
-        instance_exec(*args, &block)
+        run_callback(*args, &block)
       end
     end
 
+    def run_last_callback_for(name, *args)
+      if callback = self.class.callbacks[name].last
+        run_callback(*args, &callback)
+      end
+    end
+
+    def run_callback(*args, &block)
+      instance_exec(*args, &block)
+    end
+
     module ClassMethods
       # Hash of callback blocks to be called.
       def callbacks
@@ -37,6 +47,10 @@ module Fetch
           define_method name do |*args|
             run_callbacks_for(name, *args).last
           end
+
+          define_method "#{name}!" do |*args|
+            run_last_callback_for(name, *args)
+          end
         end
       end
 

data/lib/fetch/json.rb

@@ -0,0 +1,9 @@
+module Fetch
+  module JSON
+    def self.included(base)
+      base.defaults do |req|
+        req.parse { |body| ::JSON.parse(body) }
+      end
+    end
+  end
+end

data/lib/fetch/module.rb

@@ -7,11 +7,20 @@ module Fetch
                     :failure,
                     :error
 
+    def initialize(fetchable = nil)
+      @fetchable = fetchable
+    end
+
     # Whether or not the module should be used when fetching.
     # Set with `fetch_if do ... end`.
     def fetch?
       return true unless callback?(:fetch_if)
       !!fetch_if
     end
+
+    private
+
+    # The optional instance being fetched.
+    attr_reader :fetchable
   end
 end

data/lib/fetch/request.rb

@@ -99,6 +99,19 @@ module Fetch
       @before_process_callback.call if @before_process_callback
     end
 
+    # Sets a parse callback to be run on the body returned from the request.
+    # It is run before processing and its result send to process.
+    def parse(&block)
+      raise "You must supply a block to #{self.class.name}#parse" unless block
+      @parse_callback = block
+    end
+
+    # Runs the before process callback.
+    def parse!(body)
+      return body unless @parse_callback
+      @parse_callback.call(body)
+    end
+
     # Sets the callback to be run when the request completes.
     def process(&block)
       raise "You must supply a block to #{self.class.name}#process" unless block
@@ -109,6 +122,7 @@ module Fetch
     # the exception to the error callback.
     def process!(body, url, effective_url)
       before_process!
+      body = parse!(body)
       @process_callback.call(body, url, effective_url) if @process_callback
       after_process!
     rescue => e

data/lib/fetch/version.rb

@@ -1,3 +1,3 @@
 module Fetch
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/test/defaults_test.rb

@@ -0,0 +1,23 @@
+require "test_helper"
+
+class DefaultsTest < Minitest::Test
+  def test_simple_fetch
+    words = %w{one two}
+    words.each { |w| stub_request(:get, "http://test.com/#{w}").to_return(body: "got #{w}") }
+    actions = []
+    mod = Class.new(Fetch::Module) do
+      defaults do |req|
+        req.process do |body|
+          actions << "process: #{body}"
+        end
+      end
+      words.each do |word|
+        request do |req|
+          req.url = "http://test.com/#{word}"
+        end
+      end
+    end
+    MockFetcher(mod).new.fetch
+    assert_equal ["process: got one", "process: got two"], actions
+  end
+end

data/test/fetch_test.rb

@@ -67,34 +67,20 @@ class FetchTest < Minitest::Test
     assert_equal [0, 50, 100], updates
   end
 
-  def test_initializes_modules
-    stub_request(:get, "http://test.com/one").to_return(body: "got one")
+  def test_sends_fetchable_to_modules
+    stub_request(:get, "https://api.github.com/users/lassebunk").to_return(body: "id: 1234")
     actions = []
     mod = Class.new(Fetch::Module) do
-      attr_reader :word
-      def initialize(word)
-        @word = word
-      end
       request do |req|
-        req.url = "http://test.com/one"
+        req.url = "https://api.github.com/users/#{fetchable.login}"
         req.process do |body|
-          actions << "process: #{body} (#{word})"
+          actions << "process: #{body}"
         end
       end
     end
-
-    klass = Class.new(MockFetcher(mod)) do
-      attr_reader :something
-      def initialize(something)
-        @something = something
-      end
-      init do |klass|
-        klass.new(something)
-      end
-    end
-
-    klass.new("a word").fetch
-    assert_equal ["process: got one (a word)"], actions
+    user = OpenStruct.new(login: "lassebunk")
+    MockFetcher(mod).new(user).fetch
+    assert_equal ["process: id: 1234"], actions
   end
 
   def test_process_block_scope

data/test/init_test.rb

@@ -0,0 +1,47 @@
+require "test_helper"
+
+class InitTest < Minitest::Test
+  def test_initializes_modules
+    stub_request(:get, "https://api.github.com/users/lassebunk").to_return(body: "id: 1234")
+    actions = []
+    mod = Class.new(Fetch::Module) do
+      attr_reader :email, :login
+      def initialize(email, login)
+        @email, @login = email, login
+      end
+      request do |req|
+        req.url = "https://api.github.com/users/#{login}"
+        req.process do |body|
+          actions << "process: #{body} (email: #{email}, login: #{login})"
+        end
+      end
+    end
+
+    klass = Class.new(MockFetcher(mod)) do
+      alias :user :fetchable
+      init do |klass|
+        klass.new(user.email, user.login)
+      end
+    end
+
+    user = OpenStruct.new(email: "lasse@bogrobotten.dk", login: "lassebunk")
+    klass.new(user).fetch
+    assert_equal ["process: id: 1234 (email: lasse@bogrobotten.dk, login: lassebunk)"], actions
+  end
+
+  def test_init_runs_only_once
+    actions = []
+    mod = Class.new(Fetch::Module)
+    klass = Class.new(MockFetcher(mod)) do
+      init do |klass|
+        this_cant_be_run!
+      end
+      init do |klass|
+        actions << "got init"
+        klass.new
+      end
+    end
+    klass.new.fetch
+    assert_equal ["got init"], actions
+  end
+end

data/test/json_test.rb

@@ -0,0 +1,19 @@
+require "test_helper"
+
+class JsonTest < Minitest::Test
+  def test_parsing_json
+    stub_request(:get, "http://api.test.com/user").to_return(body: '{"id":123}')
+    actions = []
+    mod = Class.new(Fetch::Module) do
+      include Fetch::JSON
+      request do |req|
+        req.url = "http://api.test.com/user"
+        req.process do |json|
+          actions << "user id: #{json['id'].inspect}"
+        end
+      end
+    end
+    MockFetcher(mod).new.fetch
+    assert_equal ["user id: 123"], actions
+  end
+end

data/test/load_test.rb

@@ -0,0 +1,52 @@
+require "test_helper"
+
+class LoadTest < Minitest::Test
+  def test_loading_modules
+    %w{one two}.each { |w| stub_request(:get, "http://test.com/#{w}").to_return(body: "got #{w}") }
+    actions = []
+    mod1 = Class.new(Fetch::Module) do
+      request do |req|
+        req.url = "http://test.com/one"
+        req.process do |body|
+          actions << "process: #{body}"
+        end
+      end
+    end
+    mod2 = Class.new(Fetch::Module) do
+      request do |req|
+        req.url = "http://test.com/two"
+        req.process do |body|
+          actions << "process: #{body}"
+        end
+      end
+    end
+    klass = Class.new(Fetch::Base) do
+      modules :one, :two
+
+      load do |mods|
+        actions << "got #{mods.inspect}"
+        [mod1, mod2]
+      end
+    end
+    klass.new.fetch
+
+    assert_equal ["got [:one, :two]", "process: got one", "process: got two"], actions
+  end
+
+  def test_only_last_load_callback_is_called
+    actions = []
+    mod = Class.new(Fetch::Module)
+    klass = Class.new(Fetch::Base) do
+      load do |mods|
+        this_cant_be_run!
+      end
+      load do |mods|
+        actions << "got last callback"
+        [mod]
+      end
+    end
+    klass.new.fetch
+
+    assert_equal ["got last callback"], actions
+  end
+end

data/test/parse_test.rb

@@ -0,0 +1,21 @@
+require "test_helper"
+
+class ParseTest < Minitest::Test
+  def test_parsing_body
+    stub_request(:get, "http://test.com/one").to_return(body: "got one")
+    actions = []
+    mod = Class.new(Fetch::Module) do
+      request do |req|
+        req.url = "http://test.com/one"
+        req.parse do |body|
+          "parsed(#{body})"
+        end
+        req.process do |body|
+          actions << "process: #{body}"
+        end
+      end
+    end
+    MockFetcher(mod).new.fetch
+    assert_equal ["process: parsed(got one)"], actions
+  end
+end

data/test/test_helper.rb

@@ -1,5 +1,7 @@
 require "minitest/autorun"
 require "webmock/minitest"
+require "ostruct"
+require "json"
 
 require "fetch"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fetch
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Lasse Bunk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-02 00:00:00.000000000 Z
+date: 2014-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: typhoeus
@@ -89,6 +89,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -102,12 +103,18 @@ files:
 - lib/fetch/base.rb
 - lib/fetch/callbacks.rb
 - lib/fetch/configuration.rb
+- lib/fetch/json.rb
 - lib/fetch/module.rb
 - lib/fetch/request.rb
 - lib/fetch/simple.rb
 - lib/fetch/version.rb
 - test/callbacks_test.rb
+- test/defaults_test.rb
 - test/fetch_test.rb
+- test/init_test.rb
+- test/json_test.rb
+- test/load_test.rb
+- test/parse_test.rb
 - test/simple_test.rb
 - test/test_helper.rb
 homepage: https://github.com/lassebunk/fetch
@@ -136,6 +143,11 @@ specification_version: 4
 summary: Coming
 test_files:
 - test/callbacks_test.rb
+- test/defaults_test.rb
 - test/fetch_test.rb
+- test/init_test.rb
+- test/json_test.rb
+- test/load_test.rb
+- test/parse_test.rb
 - test/simple_test.rb
 - test/test_helper.rb