web_fetch 0.4.0 → 0.5.0

data/lib/web_fetch/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module WebFetch
-  VERSION = '0.4.0'
+  VERSION = '0.5.0'
 end

data/manifest

@@ -0,0 +1,64 @@
+.gitignore
+.rspec
+.rubocop.yml
+.ruby-version
+.strong_versions.yml
+Gemfile
+LICENSE
+Makefile
+README.md
+Rakefile
+TODO
+bin/rspec
+bin/rubocop
+bin/strong_versions
+bin/web_fetch_control
+bin/web_fetch_server
+config/locales/en.yml
+doc/examples/blocking_requests.rb
+doc/examples/non_blocking_requests.rb
+doc/examples/use_uid_for_request.rb
+doc/web_fetch_architecture.png
+docker/Dockerfile
+lib/web_fetch.rb
+lib/web_fetch/client.rb
+lib/web_fetch/concerns/client_http.rb
+lib/web_fetch/concerns/http_helpers.rb
+lib/web_fetch/concerns/validatable.rb
+lib/web_fetch/errors.rb
+lib/web_fetch/gatherer.rb
+lib/web_fetch/helpers.rb
+lib/web_fetch/logger.rb
+lib/web_fetch/promise.rb
+lib/web_fetch/request.rb
+lib/web_fetch/resources.rb
+lib/web_fetch/response.rb
+lib/web_fetch/retriever.rb
+lib/web_fetch/router.rb
+lib/web_fetch/server.rb
+lib/web_fetch/storage.rb
+lib/web_fetch/storage/memcached.rb
+lib/web_fetch/storage/memory.rb
+lib/web_fetch/storage/redis.rb
+lib/web_fetch/version.rb
+manifest
+spec/client_spec.rb
+spec/concerns/validatable_spec.rb
+spec/features/http_fetching_spec.rb
+spec/gatherer_spec.rb
+spec/helpers_spec.rb
+spec/i18n_spec.rb
+spec/promise_spec.rb
+spec/request_spec.rb
+spec/resources_spec.rb
+spec/response_spec.rb
+spec/retriever_spec.rb
+spec/router_spec.rb
+spec/server_spec.rb
+spec/spec_helper.rb
+spec/storage/memcached_spec.rb
+spec/storage/memory_spec.rb
+spec/storage/redis_spec.rb
+spec/storage/shared_examples.rb
+swagger.yaml
+web_fetch.gemspec

data/spec/client_spec.rb

@@ -65,13 +65,10 @@ describe WebFetch::Client do
 
     it { is_expected.to be_a WebFetch::Response }
 
-    context 'no matching request found' do
-      subject { proc { client.fetch('not-found') } }
-      it { is_expected.to raise_error WebFetch::RequestNotFoundError }
+    context 'pending' do
+      subject { client.fetch('not-ready-yet') }
+      it { is_expected.to be_pending }
     end
-
-    # Tested more extensively in supporting methods #retrieve_by_uid and
-    # #find_by_uid below
   end
 
   describe '#retrieve_by_uid' do

data/spec/gatherer_spec.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 describe WebFetch::Gatherer do
-  let(:server) { WebFetch::MockServer.new }
+  let(:storage) { double }
 
   let(:valid_params) do
     { requests: [
@@ -11,31 +11,33 @@ describe WebFetch::Gatherer do
   end
 
   it 'is initialisable with params' do
-    expect(described_class.new(server, valid_params)).to be_a described_class
+    expect(described_class.new(storage, valid_params)).to be_a described_class
   end
 
   describe 'validation' do
     context 'invalid' do
       it 'is invalid if `requests` parameter is not passed' do
-        gatherer = described_class.new(server, {})
+        gatherer = described_class.new(storage, {})
         expect(gatherer.valid?).to be false
         expect(gatherer.errors).to include I18n.t(:requests_missing)
       end
 
       it 'is invalid if `requests` is not an array parameter' do
-        gatherer = described_class.new(server, requests: 'hello')
+        gatherer = described_class.new(storage, requests: 'hello')
         expect(gatherer.valid?).to be false
         expect(gatherer.errors).to include I18n.t(:requests_not_array)
       end
 
       it 'is invalid if `requests` is an empty array' do
-        gatherer = described_class.new(server, requests: [])
+        gatherer = described_class.new(storage, requests: [])
         expect(gatherer.valid?).to be false
         expect(gatherer.errors).to include I18n.t(:requests_empty)
       end
 
       it 'is invalid if `url` missing from any requests' do
-        gatherer = described_class.new(server, requests: [{ url: 'hello' }, {}])
+        gatherer = described_class.new(
+          storage, requests: [{ url: 'hello' }, {}]
+        )
         expect(gatherer.valid?).to be false
         expect(gatherer.errors).to include I18n.t(:missing_url)
       end
@@ -43,14 +45,29 @@ describe WebFetch::Gatherer do
 
     context 'valid' do
       it 'is valid when passed valid params' do
-        expect(described_class.new(server, valid_params).valid?).to be true
+        expect(described_class.new(storage, valid_params).valid?).to be true
       end
     end
   end
 
   describe '#start' do
+    let(:http) do
+      double(new: double(public_send: double(callback: nil, errback: nil)))
+    end
+
+    let(:logger) { double(debug: nil) }
+
+    before do
+      stub_request(:get, 'http://remotehost:8089/').to_return(status: 200)
+      stub_request(:get, 'http://blah/').to_return(status: 200)
+      stub_request(:put, 'http://blah/').to_return(status: 200)
+      stub_request(:get, 'http://hello/').to_return(status: 200)
+    end
+
     it 'returns a hash containing sha1 hashes of requests' do
-      response = described_class.new(server, valid_params).start
+      deferred = double(callback: nil, errback: nil)
+      http = double(new: double(public_send: deferred))
+      response = described_class.new(storage, valid_params, logger, http).start
       hash = Digest::SHA1.new.digest(JSON.dump(valid_params[:requests].first))
       expect(response[:requests].first[:hash]).to eql Digest.hexencode(hash)
     end
@@ -68,14 +85,16 @@ describe WebFetch::Gatherer do
                headers: { 'Content-Type' => 'hello' },
                method: 'PUT' }
       responses = [req1, req2, req3, req4, req5].map do |req|
-        described_class.new(server, requests: [req], _server: server).start
+        described_class.new(
+          storage, { requests: [req] }, logger, http
+        ).start
       end
       hashes = responses.map { |res| res[:requests].first[:hash] }
       expect(hashes.uniq.length).to eql 5
     end
 
     it 'returns a hash containing unique IDs for requests' do
-      response = described_class.new(server, valid_params).start
+      response = described_class.new(storage, valid_params, logger, http).start
       uid1 = response[:requests][0][:uid]
       uid2 = response[:requests][1][:uid]
       expect(uid1).to_not eql uid2
@@ -85,9 +104,8 @@ describe WebFetch::Gatherer do
       it 'is included in response' do
         # Ensure that the requester can embed their own identifiers to link to
         # the uid of the delegated request
-        params = { requests: [url: '-', bob: 'hello'],
-                   _server: server }
-        response = described_class.new(server, params).start
+        params = { requests: [url: 'http://blah/', bob: 'hello'] }
+        response = described_class.new(storage, params, logger, http).start
         expect(response[:requests].first[:request][:bob]).to eql 'hello'
       end
 
@@ -95,14 +113,13 @@ describe WebFetch::Gatherer do
         # Ensure that only pertinent values are used to compute hash (i.e.
         # adding auxiliary data will still allow retrieval by hash for
         # otherwise duplicate requests
-        params1 = { requests: [url: 'http://blah', bob: 'hello'],
-                    _server: server }
-        response1 = described_class.new(server, params1).start
-
-        params2 = { requests: [url: 'http://blah', not_bob: 'good bye'],
-                    _server: server }
-        response2 = described_class.new(server, params2).start
-        expect(response1[:requests][0][:hash]).to eql response2[:requests][0][:hash]
+        params1 = { requests: [url: 'http://blah', bob: 'hello'] }
+        response1 = described_class.new(storage, params1, logger, http).start
+
+        params2 = { requests: [url: 'http://blah', not_bob: 'good bye'] }
+        response2 = described_class.new(storage, params2, logger, http).start
+        expect(response1[:requests][0][:hash])
+          .to eql response2[:requests][0][:hash]
       end
     end
   end

data/spec/promise_spec.rb

@@ -9,21 +9,28 @@ RSpec.describe WebFetch::Promise do
   let(:find_url) { "http://#{client.host}:#{client.port}/find/#{uid}" }
 
   let(:client_success) do
-    double(fetch: double('success',
-      complete?: true, pending?: false, success?: true, error: nil
-    ))
+    double(
+      fetch: double('success',
+                    complete?: true,
+                    pending?: false,
+                    success?: true,
+                    error: nil)
+    )
   end
 
   let(:client_failure) do
-    double(fetch: double('failure',
-      complete?: true, pending?: false, success?: false, error: 'foo'
-    ))
+    double(
+      fetch: double('failure',
+                    complete?: true,
+                    pending?: false,
+                    success?: false,
+                    error: 'foo')
+    )
   end
 
   let(:client_pending) do
     double(fetch: double('pending',
-      complete?: false, pending?: true, error: nil
-    ))
+                         complete?: false, pending?: true, error: nil))
   end
 
   let(:client_not_started) do
@@ -36,8 +43,30 @@ RSpec.describe WebFetch::Promise do
 
   describe '#fetch' do
     before do
-      stub_request(:get, retrieve_url).to_return(body: { request: {} }.to_json)
-      stub_request(:get, find_url).to_return(body: { request: {} }.to_json)
+      stub_request(:get, retrieve_url)
+        .to_return(
+          body: {
+            request: {
+              uid: 123,
+              request: {},
+              response: {
+                success: true, status: 200, body: 'abc123', headers: {}
+              }
+            }
+          }.to_json
+        )
+
+      stub_request(:get, find_url).to_return(
+        body: {
+          request: {
+            uid: 123,
+            request: {},
+            response: {
+              success: true, status: 200, body: 'abc123', headers: {}
+            }
+          }
+        }.to_json
+      )
     end
 
     subject { promise.fetch(fetch_options) }
@@ -63,7 +92,16 @@ RSpec.describe WebFetch::Promise do
       stub_request(:get, retrieve_url)
         .to_return(
           body: {
-            response: { success: true, body: 'abc123' }, request: {}
+            request: {
+              uid: 123,
+              request: {},
+              response: {
+                success: true,
+                status: 200,
+                body: Base64.encode64('abc123'),
+                headers: {}
+              }
+            }
           }.to_json
         )
       promise.fetch

data/spec/resources_spec.rb

@@ -6,11 +6,16 @@ describe WebFetch::Resources do
   describe '.root' do
     it 'responds with application name' do
       expect(described_class.root(nil, nil))
-        .to eql(status: 200, payload: { application: 'WebFetch' })
+        .to eql(
+          status: 200,
+          command: 'root',
+          payload: { application: 'WebFetch' }
+        )
     end
   end
 
   describe '.gather' do
+    before { stub_request(:any, 'http://google.com').to_return(status: 200) }
     let(:response) do
       described_class.gather(server, requests: [{ url: 'http://google.com' }])
     end
@@ -25,34 +30,26 @@ describe WebFetch::Resources do
   end
 
   describe '.retrieve' do
-    it 'gives 404 not found when unrecognised uid requested' do
+    it 'gives pending when unrecognised uid requested' do
       response = described_class.retrieve(server, uid: '123')
-      expect(response[:status]).to eql 404
-      error = response[:payload][:error]
-      expect(error).to eql I18n.t(:uid_not_found)
+      expect(response[:request][:pending]).to be true
     end
 
-    it 'gives 404 not found when unrecognised hash requested' do
+    it 'gives pending when unrecognised hash requested' do
       response = described_class.retrieve(server, hash: 'abc')
-      expect(response[:status]).to eql 404
-      error = response[:payload][:error]
-      expect(error).to eql I18n.t(:hash_not_found)
+      expect(response[:request][:pending]).to be true
     end
   end
 
   describe '.find' do
-    it 'gives 404 not found when unrecognised uid requested' do
+    it 'gives pending when unrecognised uid requested' do
       response = described_class.find(server, uid: '123')
-      expect(response[:status]).to eql 404
-      error = response[:payload][:error]
-      expect(error).to eql I18n.t(:uid_not_found)
+      expect(response[:request][:pending]).to be true
     end
 
-    it 'gives 404 not found when unrecognised hash requested' do
+    it 'gives pending when unrecognised hash requested' do
       response = described_class.find(server, hash: 'abc')
-      expect(response[:status]).to eql 404
-      error = response[:payload][:error]
-      expect(error).to eql I18n.t(:hash_not_found)
+      expect(response[:request][:pending]).to be true
     end
   end
 end

data/spec/response_spec.rb

@@ -3,15 +3,19 @@
 RSpec.describe WebFetch::Response do
   let(:response) do
     described_class.new(
-      body: 'abc123',
-      headers: { 'Foo' => 'Bar' },
-      status: 200,
-      pending: false,
-      success: false,
-      error: 'foo error happened',
-      request: { url: 'http://blah/' },
-      uid: 'uid123',
-      response_time: 123.45
+      request: {
+        response: {
+          body: Base64.encode64('abc123'),
+          headers: { 'Foo' => 'Bar' },
+          error: 'foo error happened',
+          status: 200,
+          success: false
+        },
+        pending: false,
+        request: { url: 'http://blah/' },
+        uid: 'uid123',
+        response_time: 123.45
+      }
     )
   end
 

data/spec/retriever_spec.rb

@@ -1,23 +1,23 @@
 # frozen_string_literal: true
 
 describe WebFetch::Retriever do
-  let(:server) { WebFetch::MockServer.new }
+  let(:storage) { double(fetch: nil, store: nil) }
   let(:valid_params) { { uid: 'abc123' } }
 
   it 'is initialisable with params' do
-    expect(described_class.new(server, valid_params, {}))
+    expect(described_class.new(storage, valid_params, {}))
       .to be_a described_class
   end
 
   describe 'validation' do
     context 'valid' do
       it 'is valid when `uid` given' do
-        retriever = described_class.new(server, { uid: 'abc123' }, {})
+        retriever = described_class.new(storage, { uid: 'abc123' }, {})
         expect(retriever.valid?).to be true
       end
 
       it 'is valid when `hash` given' do
-        retriever = described_class.new(server, { hash: 'def456' }, {})
+        retriever = described_class.new(storage, { hash: 'def456' }, {})
         expect(retriever.valid?).to be true
       end
     end
@@ -25,14 +25,14 @@ describe WebFetch::Retriever do
     context 'invalid' do
       it 'is invalid if both `hash` and `uid` given' do
         retriever = described_class.new(
-          server, { hash: 'def456', uid: 'abc123' }, {}
+          storage, { hash: 'def456', uid: 'abc123' }, {}
         )
         expect(retriever.valid?).to be false
         expect(retriever.errors).to include I18n.t(:hash_or_uid_but_not_both)
       end
 
       it 'is invalid if neither `hash` nor `uid` given' do
-        retriever = described_class.new(server, {}, {})
+        retriever = described_class.new(storage, {}, {})
         expect(retriever.valid?).to be false
         expect(retriever.errors).to include I18n.t(:missing_hash_and_uid)
       end
@@ -40,14 +40,14 @@ describe WebFetch::Retriever do
   end
 
   describe '#find' do
-    it 'returns `nil` when given uid has not been requested' do
-      retriever = described_class.new(server, { uid: 'nope' }, {})
-      expect(retriever.find).to be_nil
+    it 'returns "pending" when given uid has not been requested' do
+      retriever = described_class.new(storage, { uid: 'nope' }, {})
+      expect(retriever.find[:pending]).to be true
     end
 
-    it 'returns `nil` when given hash has not been requested' do
-      retriever = described_class.new(server, { hash: 'also nope' }, {})
-      expect(retriever.find).to be_nil
+    it 'returns "pending" when given hash has not been requested' do
+      retriever = described_class.new(storage, { hash: 'also nope' }, {})
+      expect(retriever.find[:pending]).to be true
     end
 
     it 'returns payload when request has been retrieved' do
@@ -58,14 +58,13 @@ describe WebFetch::Retriever do
       url = 'http://blah.blah/success'
       stub_request(:any, url)
 
-      gatherer = WebFetch::Gatherer.new(server, requests: [{ url: url }])
+      gatherer = WebFetch::Gatherer.new(storage, requests: [{ url: url }])
       response = gatherer.start
       uid = response[:requests].first[:uid]
-      expect(server).to receive(:storage)
-        .and_return(uid => { body: 'fake body' })
+      allow(storage).to receive(:fetch).and_return('test')
 
-      retriever = described_class.new(server, { uid: uid }, {})
-      expect(retriever.find[:body]).to eql 'fake body'
+      retriever = described_class.new(storage, { uid: uid }, {})
+      expect(retriever.find).to eql 'test'
     end
   end
 end