web_fetch 0.1.3 → 0.2.0

data/lib/web_fetch/promise.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module WebFetch
+  class Promise
+    attr_reader :uid, :request, :result
+
+    def initialize(client, options = {})
+      @client = client
+      @uid = options[:uid]
+      @request = Request.from_hash(options[:request])
+    end
+
+    def fetch(options = {})
+      return @result if complete?
+
+      block = options.fetch(:wait, true)
+      @raw_result = find_or_retrieve(block)
+      (@result = build_result)
+    end
+
+    def custom
+      request&.custom
+    end
+
+    def complete?
+      return false if @result.nil?
+      return false if pending?
+      return true if @result
+
+      false
+    end
+
+    def pending?
+      return false if @result.nil?
+
+      @result == :pending
+    end
+
+    def success?
+      complete? && @raw_result[:response][:success]
+    end
+
+    def error
+      return nil unless complete?
+
+      @raw_result[:response][:error]
+    end
+
+    private
+
+    def find_or_retrieve(block)
+      block ? @client.retrieve_by_uid(@uid) : @client.find_by_uid(@uid)
+    end
+
+    def build_result
+      return nil if @raw_result.nil?
+      return :pending if @raw_result[:pending]
+      return nil unless @raw_result[:response]
+
+      response = @raw_result[:response]
+      new_result(response)
+    end
+
+    def new_result(response)
+      Result.new(
+        body: response[:body],
+        headers: response[:headers],
+        status: response[:status],
+        success: @raw_result[:response][:success],
+        error: @raw_result[:response][:error],
+        uid: @uid
+      )
+    end
+  end
+end

data/lib/web_fetch/request.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module WebFetch
+  class Request
+    attr_writer :url, :query, :headers, :body, :custom
+    attr_reader :url, :query, :headers, :body, :custom
+
+    def initialize
+      yield self
+    end
+
+    def method=(val)
+      @method = val.downcase.to_sym
+    end
+
+    def method
+      @method ||= :get
+    end
+
+    def to_h
+      {
+        url: url,
+        query: query,
+        headers: headers,
+        body: body,
+        method: method,
+        custom: custom
+      }
+    end
+
+    def eql?(other)
+      # Makes testing WebFetch a bit easier (based on real world case I hit
+      # using WebFetch in a Rails app)
+      other.to_h == to_h
+    end
+
+    def ==(other)
+      eql?(other)
+    end
+
+    def self.from_hash(hash)
+      hash_copy = hash.dup
+      request = build_request(hash_copy)
+      raise ArgumentError, "Unrecognized keys: #{hash}" unless hash_copy.empty?
+
+      request
+    end
+
+    class << self
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity
+      def build_request(hash)
+        Request.new do |request|
+          request.url = hash.delete(:url) if hash.key?(:url)
+          request.query = hash.delete(:query) if hash.key?(:query)
+          request.headers = hash.delete(:headers) if hash.key?(:headers)
+          request.body = hash.delete(:body) if hash.key?(:body)
+          request.method = hash.delete(:method) if hash.key?(:method)
+          request.custom = hash.delete(:custom) if hash.key?(:custom)
+        end
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity
+    end
+  end
+end

data/lib/web_fetch/resources.rb

@@ -19,8 +19,8 @@ module WebFetch
         end
       end
 
-      def retrieve(server, params)
-        retriever = Retriever.new(server, params)
+      def retrieve(server, params, options = {})
+        retriever = Retriever.new(server, params, options)
         unless retriever.valid?
           return { status: status(:unprocessable),
                    payload: { error: retriever.errors } }
@@ -28,6 +28,10 @@ module WebFetch
         defer_if_found(retriever)
       end
 
+      def find(server, params)
+        retrieve(server, params, block: false)
+      end
+
       private
 
       def status(name)
@@ -51,7 +55,7 @@ module WebFetch
           { status: status(:not_found),
             payload: { error: retriever.not_found_error } }
         else
-          { deferred: found }
+          { request: found }
         end
       end
     end

data/lib/web_fetch/result.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module WebFetch
+  class Result
+    attr_reader :body, :headers, :status, :error, :uid
+
+    def initialize(options = {})
+      @pending = options.fetch(:pending, false)
+      return if pending?
+
+      @body = options.fetch(:body)
+      @headers = options.fetch(:headers)
+      @status = options.fetch(:status)
+      @success = options.fetch(:success)
+      @error = options.fetch(:error)
+      @uid = options.fetch(:uid)
+    end
+
+    def pending?
+      @pending
+    end
+
+    def complete?
+      !pending?
+    end
+
+    def success?
+      @success
+    end
+  end
+end

data/lib/web_fetch/retriever.rb

@@ -7,17 +7,20 @@ module WebFetch
 
     attr_reader :not_found_error
 
-    def initialize(server, params)
+    def initialize(server, params, options)
       @uid = params[:uid]
       @hash = params[:hash]
       @server = server
+      @block = options.fetch(:block, true)
     end
 
     def find
-      stored = @server.storage.fetch(@uid)
-      return not_found if stored.nil?
+      request = @server.storage.fetch(@uid)
+      return not_found if request.nil?
+      return not_found if request.nil?
+      return request.merge(pending: true) if pending?(request)
 
-      stored
+      request
     end
 
     private
@@ -35,5 +38,15 @@ module WebFetch
                          end
       nil
     end
+
+    def pending?(request)
+      return false if request.nil?
+      return false if request[:succeeded]
+      # User requested blocking operation so we will wait until item is ready
+      # rather than return a `pending` status
+      return false if @block
+
+      true
+    end
   end
 end

data/lib/web_fetch/router.rb

@@ -15,19 +15,21 @@ module WebFetch
     def route(url, options = {})
       @server = options.delete(:server)
       options = { query_string: nil, method: 'GET' }.merge(options)
-      method = options[:method].downcase.to_sym
+
       Logger.info("#{url}: #{options}")
-      begin
-        params = build_params(options)
-      rescue JSON::ParserError
-        return { status: 400, payload: I18n.t(:bad_json) }
-      end
-      @router.recognize(url, method: method).call(params)
+
+      json_params = build_params(options)
+      return { status: 400, payload: I18n.t(:bad_json) } if json_params.nil?
+
+      resource = @router.recognize(
+        url, method: normalize_http_method(options[:method])
+      )
+      resource.call(resource.params.merge(json_params))
     end
 
     private
 
-    # rubocop:disable Metrics/MethodLength
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
     def setup
       resource_finder = lambda do |name, env|
         Resources.public_send(name, @server, env)
@@ -42,12 +44,16 @@ module WebFetch
           resource_finder.call(:gather, params)
         }
 
-        get '/retrieve', to: lambda { |params|
+        get '/retrieve/:uid', to: lambda { |params|
           resource_finder.call(:retrieve, params)
         }
+
+        get '/find/:uid', to: lambda { |params|
+          resource_finder.call(:find, params)
+        }
       end
     end
-    # rubocop:enable Metrics/MethodLength
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
 
     def build_params(options)
       params = Rack::Utils.parse_nested_query(options[:query_string])
@@ -55,6 +61,8 @@ module WebFetch
       params = symbolize(params)
       params.merge!(options[:post_data] || {})
       params
+    rescue JSON::ParserError
+      nil
     end
 
     def merge_json(params)
@@ -67,5 +75,9 @@ module WebFetch
     def merge_json!(params)
       params.merge!(merge_json(params))
     end
+
+    def normalize_http_method(method)
+      method.downcase.to_sym
+    end
   end
 end

data/lib/web_fetch/server.rb

@@ -20,14 +20,8 @@ module WebFetch
     def process_http_request
       result = @router.route(@http_request_uri, request_params)
       response = EM::DelegatedHttpResponse.new(self)
-
       default_headers(response)
-
-      if result[:deferred].nil?
-        respond_immediately(result, response)
-      else
-        wait_for_response(result[:deferred], response)
-      end
+      outcome(result, response)
     end
 
     # Note that #gather is called by WebFetch itself to asynchronously gather
@@ -35,15 +29,28 @@ module WebFetch
     # #process_http_request and subsequently WebFetch::Router#route
     def gather(targets)
       targets.each do |target|
-        request = target[:request]
-        async_request = EM::HttpRequest.new(request[:url])
-        method = request.fetch(:method, 'GET').downcase.to_sym
-        http = async_request.public_send(method,
-                                         head: request[:headers],
-                                         query: request.fetch(:query, {}),
-                                         body: request.fetch(:body, nil))
-        @storage.store(target[:uid], uid: target[:uid], http: http)
+        http = request_async(target[:request])
+        request = { uid: target[:uid], deferred: http }
+        apply_callbacks(request)
+        @storage.store(target[:uid], request)
       end
     end
+
+    private
+
+    def outcome(result, response)
+      # User requested an unrecognised ID
+      return respond_immediately(result, response) if result[:request].nil?
+
+      # Fetch has already completed
+      return succeed(result, response) if result[:succeeded]
+      return fail_(result, response) if result[:failed]
+
+      # User requested non-blocking call
+      return pending(result, response) if result[:request][:pending]
+
+      # User requested blocking call
+      wait_for_response(result[:request], response)
+    end
   end
 end

data/lib/web_fetch/storage.rb

@@ -1,16 +1,21 @@
 # frozen_string_literal: true
 
 module WebFetch
-  # Rudimentary global storage for responses
+  # Rudimentary global storage for responses. The intention is that this will
+  # one day prescribe an interface to e.g. memcached
   class Storage
-    @_storage = {}
+    @storage = {}
 
     def self.store(key, obj)
-      @_storage[key] = obj
+      @storage[key] = obj
     end
 
     def self.fetch(key)
-      @_storage.delete(key)
+      @storage.fetch(key, nil)
+    end
+
+    def self.delete(key)
+      @storage.delete(key)
     end
   end
 end

data/lib/web_fetch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebFetch
-  VERSION = '0.1.3'
+  VERSION = '0.2.0'
 end

data/lib/web_fetch.rb

@@ -30,6 +30,7 @@ require 'web_fetch/event_machine_helpers'
 require 'web_fetch/http_helpers'
 require 'web_fetch/concerns/validatable'
 require 'web_fetch/concerns/http_helpers'
+require 'web_fetch/concerns/client_http'
 require 'web_fetch/storage'
 require 'web_fetch/server'
 require 'web_fetch/router'
@@ -37,4 +38,8 @@ require 'web_fetch/resources'
 require 'web_fetch/gatherer'
 require 'web_fetch/retriever'
 require 'web_fetch/client'
+require 'web_fetch/request'
+require 'web_fetch/promise'
+require 'web_fetch/result'
+require 'web_fetch/errors'
 require 'web_fetch/version'

data/spec/client_spec.rb

@@ -6,6 +6,12 @@ describe WebFetch::Client do
   before(:each) do
     stub_request(:any, 'http://blah.blah/success')
       .to_return(body: 'hello, everybody')
+
+    # XXX: This does not do what we would hope in EventMachine context as it
+    # locks the entire reactor. I really don't know how to make webmock hook
+    # into EM and delay the response so we can test #find_by_uid :(
+    stub_request(:any, 'http://blah.blah/slow_success')
+      .to_return(body: ->(_req) { sleep(0.1) && 'hi' })
   end
 
   it 'can be instantiated with host and port params' do
@@ -20,15 +26,40 @@ describe WebFetch::Client do
 
   describe '#gather' do
     it 'makes `gather` requests to a running server' do
-      result = client.gather([{ url: 'http://blah.blah/success' }])
-      expect(result.first[:uid]).to_not be_nil
+      web_request = WebFetch::Request.new do |request|
+        request.url = 'http://blah.blah/success'
+        request.custom = { my_key: 'my_value' }
+      end
+      result = client.gather([web_request])
+      expect(result.first).to be_a WebFetch::Promise
+      expect(result.first.uid).to_not be_nil
+      expect(result.first.custom).to eql(my_key: 'my_value')
+    end
+
+    it 'passes any WebFetch server errors back to the user' do
+      expect { client.gather([]) }.to raise_error WebFetch::ClientError
     end
   end
 
+  describe '#fetch' do
+    let(:responses) { client.gather([{ url: 'http://blah.blah/success' }]) }
+
+    subject { client.fetch(responses.first.uid) }
+
+    it { is_expected.to be_a WebFetch::Result }
+    context 'no matching request found' do
+      subject { proc { client.fetch('not-found') } }
+      it { is_expected.to raise_error WebFetch::RequestNotFoundError }
+    end
+
+    # Tested more extensively in supporting methods #retrieve_by_uid and
+    # #find_by_uid below
+  end
+
   describe '#retrieve_by_uid' do
     it 'retrieves a gathered item' do
       result = client.gather([{ url: 'http://blah.blah/success' }])
-      uid = result.first[:uid]
+      uid = result.first.uid
 
       retrieved = client.retrieve_by_uid(uid)
       expect(retrieved[:response][:status]).to eql 200
@@ -44,6 +75,24 @@ describe WebFetch::Client do
     end
   end
 
+  describe '#find_by_uid' do
+    it 'returns a ready status when has been fetched' do
+      pending 'Find a good way to create a slow response without locking EM'
+      result = client.gather([{ url: 'http://blah.blah/slow_success' }])
+      uid = result.first[:uid]
+
+      found = client.find_by_uid(uid)
+      expect(found[:pending]).to be true
+    end
+
+    it 'returns nil for non-requested items' do
+      client.gather([{ url: 'http://blah.blah/success' }])
+
+      retrieved = client.find_by_uid('lalalala')
+      expect(retrieved).to be_nil
+    end
+  end
+
   describe '#create' do
     it 'spawns a server and returns a client able to connect' do
       client = described_class.create('localhost', 8077, log: File::NULL)

data/spec/promise_spec.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+RSpec.describe WebFetch::Promise do
+  let(:uid) { 'abc123' }
+  let(:request) { { url: 'http://blah', custom: { my_key: 'my_value' } } }
+  let(:options) { { uid: uid, request: request } }
+  let(:client) { WebFetch::Client.new('test-host', 8080) }
+  let(:response) { described_class.new(client, options) }
+  let(:retrieve_url) { "http://#{client.host}:#{client.port}/retrieve/#{uid}" }
+  let(:find_url) { "http://#{client.host}:#{client.port}/find/#{uid}" }
+
+  let(:client_success) do
+    double(retrieve_by_uid: { response: { success: true } })
+  end
+
+  let(:client_failure) do
+    double(retrieve_by_uid: { response: { success: false, error: 'foo' } })
+  end
+
+  let(:client_pending) do
+    double(retrieve_by_uid: { pending: true })
+  end
+
+  let(:client_not_started) do
+    double(retrieve_by_uid: nil)
+  end
+
+  subject { response }
+
+  it { is_expected.to be_a described_class }
+
+  describe '#fetch' do
+    before do
+      stub_request(:get, retrieve_url).to_return(body: {}.to_json)
+      stub_request(:get, find_url).to_return(body: {}.to_json)
+    end
+
+    subject { response.fetch(fetch_options) }
+
+    context 'blocking call' do
+      let(:fetch_options) { { wait: true } }
+      it { is_expected.to have_requested(:get, retrieve_url) }
+    end
+
+    context 'non-blocking call' do
+      let(:fetch_options) { { wait: false } }
+      it { is_expected.to have_requested(:get, find_url) }
+    end
+
+    context 'default (blocking)' do
+      subject { response.fetch }
+      it { is_expected.to have_requested(:get, retrieve_url) }
+    end
+  end
+
+  describe '#result' do
+    before do
+      stub_request(:get, retrieve_url)
+        .to_return(
+          body: { response: { success: true, body: 'abc123' } }.to_json
+        )
+      response.fetch
+    end
+
+    subject { response.result }
+    it { is_expected.to be_a WebFetch::Result }
+    its(:body) { is_expected.to eql 'abc123' }
+  end
+
+  describe '#complete?, #success?, #pending?, #error' do
+    before { response.fetch }
+
+    subject { response }
+
+    context 'request succeeded' do
+      let(:client) { client_success }
+      its(:complete?) { is_expected.to be true }
+      its(:success?) { is_expected.to be true }
+      its(:pending?) { is_expected.to be false }
+      its(:error) { is_expected.to be_nil }
+    end
+
+    context 'request failed' do
+      let(:client) { client_failure }
+      its(:complete?) { is_expected.to be true }
+      its(:success?) { is_expected.to be false }
+      its(:pending?) { is_expected.to be false }
+      its(:error) { is_expected.to eql 'foo' }
+    end
+
+    context 'request pending' do
+      let(:client) { client_pending }
+      its(:complete?) { is_expected.to be false }
+      its(:success?) { is_expected.to be false }
+      its(:pending?) { is_expected.to be true }
+      its(:error) { is_expected.to be_nil }
+    end
+
+    context 'request not started' do
+      let(:client) { client_not_started }
+      its(:complete?) { is_expected.to be false }
+      its(:success?) { is_expected.to be false }
+      its(:pending?) { is_expected.to be false }
+      its(:error) { is_expected.to be_nil }
+    end
+  end
+
+  describe '#request' do
+    subject { response.request }
+    it { is_expected.to be_a WebFetch::Request }
+    its(:custom) { is_expected.to eql(my_key: 'my_value') }
+  end
+
+  describe '#custom' do
+    # I think it's good to expose this directly on the response even though its
+    # accessible on as `response.request.custom`
+    its(:custom) { is_expected.to eql(my_key: 'my_value') }
+  end
+end

data/spec/request_spec.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+RSpec.describe WebFetch::Request do
+  let(:request) do
+    described_class.new do |request|
+      request.url = 'http://blah'
+      request.method = 'GET'
+      request.query = { foo: 'bar' }
+      request.headers = { 'Content-Type' => 'application/baz' }
+      request.body = 'abc123'
+      request.custom = { my_custom_key: 'my_custom_value' }
+    end
+  end
+
+  let(:hash) do
+    {
+      url: 'http://blah',
+      method: :get,
+      query: { foo: 'bar' },
+      headers: { 'Content-Type' => 'application/baz' },
+      body: 'abc123',
+      custom: { my_custom_key: 'my_custom_value' }
+    }
+  end
+
+  subject { request }
+  it { is_expected.to be_a described_class }
+  its(:url) { is_expected.to eql 'http://blah' }
+  its(:method) { is_expected.to eql :get } # Normalised to lower-case symbols
+  its(:query) { is_expected.to eql(foo: 'bar') }
+  its(:headers) { are_expected.to eql('Content-Type' => 'application/baz') }
+  its(:body) { is_expected.to eql 'abc123' }
+  its(:custom) { is_expected.to eql(my_custom_key: 'my_custom_value') }
+  its(:to_h) { is_expected.to eql hash }
+
+  describe '#get' do
+    let(:request) { described_class.new { |request| } }
+    subject { request.method }
+    it { is_expected.to eql :get }
+  end
+
+  describe '#from_hash' do
+    subject do
+      described_class.from_hash(
+        url: 'a', method: 'GET', query: {},
+        headers: {}, body: 'abc', custom: {}
+      )
+    end
+
+    it { is_expected.to be_a described_class }
+    its(:url) { is_expected.to eql 'a' }
+    its(:method) { is_expected.to eql :get }
+    its(:query) { is_expected.to eql({}) }
+    its(:headers) { are_expected.to eql({}) }
+    its(:body) { is_expected.to eql 'abc' }
+    its(:custom) { is_expected.to eql({}) }
+
+    context 'unrecognised keys' do
+      subject { proc { described_class.from_hash(unkown_key: 'foo') } }
+      it { is_expected.to raise_error(ArgumentError) }
+    end
+  end
+
+  describe '#==' do
+    let(:request_copy) { described_class.from_hash(request.to_h) }
+    it { is_expected.to eq(request_copy) }
+  end
+
+  describe '#eql' do
+    let(:request_copy) { described_class.from_hash(request.to_h) }
+    it { is_expected.to eql(request_copy) }
+  end
+end