web_fetch 0.1.0

data/spec/gatherer_spec.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+describe WebFetch::Gatherer do
+  let(:server) { WebFetch::MockServer.new }
+
+  let(:valid_params) do
+    { requests: [
+      { url: 'http://localhost:8089' },
+      { url: 'http://remotehost:8089' }
+    ] }
+  end
+
+  it 'is initialisable with params' do
+    expect(described_class.new(server, valid_params)).to be_a described_class
+  end
+
+  describe 'validation' do
+    context 'invalid' do
+      it 'is invalid if `requests` parameter is not passed' do
+        gatherer = described_class.new(server, {})
+        expect(gatherer.valid?).to be false
+        expect(gatherer.errors).to include I18n.t(:requests_missing)
+      end
+
+      it 'is invalid if `requests` is not an array parameter' do
+        gatherer = described_class.new(server, requests: 'hello')
+        expect(gatherer.valid?).to be false
+        expect(gatherer.errors).to include I18n.t(:requests_not_array)
+      end
+
+      it 'is invalid if `requests` is an empty array' do
+        gatherer = described_class.new(server, requests: [])
+        expect(gatherer.valid?).to be false
+        expect(gatherer.errors).to include I18n.t(:requests_empty)
+      end
+
+      it 'is invalid if `url` missing from any requests' do
+        gatherer = described_class.new(server, requests: [{ url: 'hello' }, {}])
+        expect(gatherer.valid?).to be false
+        expect(gatherer.errors).to include I18n.t(:missing_url)
+      end
+    end
+
+    context 'valid' do
+      it 'is valid when passed valid params' do
+        expect(described_class.new(server, valid_params).valid?).to be true
+      end
+    end
+  end
+
+  describe '#start' do
+    it 'returns a hash containing sha1 hashes of requests' do
+      result = described_class.new(server, valid_params).start
+      hash = Digest::SHA1.new.digest(JSON.dump(valid_params[:requests].first))
+      expect(result[:requests].first[:hash]).to eql Digest.hexencode(hash)
+    end
+
+    it 'respects url, headers, http method and query when calculating sha1' do
+      req1 = { url: 'http://blah', query_string: 'a=1',
+               headers: { 'Content-Type' => 'whatever' } }
+      req2 = { url: 'http://blah', query_string: 'b=2',
+               headers: { 'Content-Type' => 'whatever' } }
+      req3 = { url: 'http://hello', query_string: 'a=1',
+               headers: { 'Content-Type' => 'whatever' } }
+      req4 = { url: 'http://blah', query_string: 'a=1',
+               headers: { 'Content-Type' => 'hello' } }
+      req5 = { url: 'http://blah', query_string: 'a=1',
+               headers: { 'Content-Type' => 'hello' },
+               method: 'PUT' }
+      results = [req1, req2, req3, req4, req5].map do |req|
+        described_class.new(server, requests: [req], _server: server).start
+      end
+      hashes = results.map { |res| res[:requests].first[:hash] }
+      expect(hashes.uniq.length).to eql 5
+    end
+
+    it 'returns a hash containing unique IDs for requests' do
+      result = described_class.new(server, valid_params).start
+      uid1 = result[:requests][0][:uid]
+      uid2 = result[:requests][1][:uid]
+      expect(uid1).to_not eql uid2
+    end
+
+    describe 'auxiliary request data' do
+      it 'is included in response' do
+        # Ensure that the requester can embed their own identifiers to link to
+        # the uid of the delegated request
+        params = { requests: [url: '-', bob: 'hello'],
+                   _server: server }
+        result = described_class.new(server, params).start
+        expect(result[:requests].first[:request][:bob]).to eql 'hello'
+      end
+
+      it 'does not affect request hash' do
+        # Ensure that only pertinent values are used to compute hash (i.e.
+        # adding auxiliary data will still allow retrieval by hash for
+        # otherwise duplicate requests
+        params1 = { requests: [url: 'http://blah', bob: 'hello'],
+                    _server: server }
+        result1 = described_class.new(server, params1).start
+
+        params2 = { requests: [url: 'http://blah', not_bob: 'good bye'],
+                    _server: server }
+        result2 = described_class.new(server, params2).start
+        expect(result1[:requests][0][:hash]).to eql result2[:requests][0][:hash]
+      end
+    end
+  end
+end

data/spec/helpers_spec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+describe WebFetch::Helpers do
+  class HelperIncluder
+    include WebFetch::Helpers
+  end
+
+  subject { HelperIncluder.new }
+
+  describe '#symbolize' do
+    it 'handles variously-nested hashes and symbolizes all keys' do
+      nested_hash = { 'a': 1, 'b': { 'c': 2, 'd': [{ 'e': 3 }] } }
+      expect(subject.symbolize(nested_hash)).to eql(
+        a: 1, b: { c: 2, d: [{ e: 3 }] }
+      )
+    end
+  end
+end

data/spec/i18n_spec.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+describe 'Internationalisation' do
+  it 'accesses translation files and generates translations' do
+    # Just a quick sanity check to make sure the translation file is loaded
+    expect(I18n.t(:requests_missing)).to eql '`requests` parameter missing'
+  end
+end

data/spec/resources_spec.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+describe WebFetch::Resources do
+  let(:server) { WebFetch::MockServer.new }
+
+  describe '.root' do
+    it 'responds with application name' do
+      expect(described_class.root(nil, nil))
+        .to eql(status: 200, payload: { application: 'WebFetch' })
+    end
+  end
+
+  describe '.gather' do
+    let(:result) do
+      described_class.gather(server, requests: [{ url: 'http://google.com' }])
+    end
+
+    it 'provides a `gather` resource' do
+      expect(result[:status]).to eql 200
+    end
+
+    it 'responds with hash' do
+      expect(result[:payload]).to be_a Hash
+    end
+  end
+
+  describe 'retrieve' do
+    it 'gives 404 not found when unrecognised uid requested' do
+      result = described_class.retrieve(server, uid: '123')
+      expect(result[:status]).to eql 404
+      error = result[:payload][:error]
+      expect(error).to eql I18n.t(:uid_not_found)
+    end
+
+    it 'gives 404 not found when unrecognised hash requested' do
+      result = described_class.retrieve(server, hash: 'abc')
+      expect(result[:status]).to eql 404
+      error = result[:payload][:error]
+      expect(error).to eql I18n.t(:hash_not_found)
+    end
+  end
+end

data/spec/retriever_spec.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+describe WebFetch::Retriever do
+  let(:server) { WebFetch::MockServer.new }
+  let(:valid_params) { { uid: 'abc123' } }
+
+  it 'is initialisable with params' do
+    expect(described_class.new(server, valid_params)).to be_a described_class
+  end
+
+  describe 'validation' do
+    context 'valid' do
+      it 'is valid when `uid` given' do
+        retriever = described_class.new(server, uid: 'abc123')
+        expect(retriever.valid?).to be true
+      end
+
+      it 'is valid when `hash` given' do
+        retriever = described_class.new(server, hash: 'def456')
+        expect(retriever.valid?).to be true
+      end
+    end
+
+    context 'invalid' do
+      it 'is invalid if both `hash` and `uid` given' do
+        retriever = described_class.new(server, hash: 'def456', uid: 'abc123')
+        expect(retriever.valid?).to be false
+        expect(retriever.errors).to include I18n.t(:hash_or_uid_but_not_both)
+      end
+
+      it 'is invalid if neither `hash` nor `uid` given' do
+        retriever = described_class.new(server, {})
+        expect(retriever.valid?).to be false
+        expect(retriever.errors).to include I18n.t(:missing_hash_and_uid)
+      end
+    end
+  end
+
+  describe '#find' do
+    it 'returns `nil` when given uid has not been requested' do
+      retriever = described_class.new(server, uid: 'nope')
+      expect(retriever.find).to be_nil
+    end
+
+    it 'returns `nil` when given hash has not been requested' do
+      retriever = described_class.new(server, hash: 'also nope')
+      expect(retriever.find).to be_nil
+    end
+
+    it 'returns payload when request has been retrieved' do
+      # This test is somewhat useless as we have to mock the behaviour of our
+      # fake Server instance a little too much (since the actual Server object
+      # we're interested in exists in a separate EventMachine thread while we
+      # run our tests). The full stack is tested by the Client specs, however.
+      url = 'http://blah.blah/success'
+      stub_request(:any, url)
+
+      gatherer = WebFetch::Gatherer.new(server, requests: [{ url: url }])
+      response = gatherer.start
+      uid = response[:requests].first[:uid]
+      expect(server).to receive(:storage)
+        .and_return(uid => { body: 'fake body' })
+
+      retriever = described_class.new(server, uid: uid)
+      expect(retriever.find[:body]).to eql 'fake body'
+    end
+  end
+end

data/spec/router_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+describe WebFetch::Router do
+  let(:router) { described_class.new }
+
+  it 'can be initialised' do
+    expect(router).to be_a described_class
+  end
+
+  describe '#route' do
+    it 'provides a route to GET /' do
+      expect(router.route('/'))
+        .to eql(status: 200, payload: { application: 'WebFetch' })
+    end
+
+    it 'provides a route to POST /gather' do
+      expect(WebFetch::Resources).to receive(:gather).and_return('hello')
+      expect(router.route('/gather', method: 'POST')).to eql 'hello'
+    end
+
+    it 'provides a route to GET /retrieve' do
+      expect(WebFetch::Resources).to receive(:retrieve).and_return('hello')
+      expect(router.route('/retrieve', method: 'GET')).to eql 'hello'
+    end
+
+    it 'decodes `json` parameter and merges into request params' do
+      json = { a: 10, b: [1, 2, 3] }
+      expect(WebFetch::Resources).to receive(:gather).with(nil, json)
+      router.route('/gather',
+                   method: 'POST',
+                   query_string: "json=#{JSON.dump(json)}",
+                   server: nil)
+    end
+
+    it 'returns appropriate response when invaid json provided' do
+      result = router.route('/gather',
+                            method: 'POST',
+                            query_string: 'json=uh oh :(',
+                            server: nil)
+      expect(result).to eql(status: 400, payload: I18n.t(:bad_json))
+    end
+  end
+end

data/spec/server_spec.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+describe WebFetch::Server do
+  let(:port) { 8089 }
+  let(:host) { 'localhost' }
+  let(:host_uri) { "http://#{host}:#{port}" }
+
+  it 'accepts HTTP connections' do
+    response = get(host_uri)
+    expect(response.success?).to be true
+    expect(JSON.parse(response.body)['application']).to eql 'WebFetch'
+  end
+
+  describe '/gather' do
+    before(:each) do
+      stub_request(:any, 'http://blah.blah/success')
+      stub_request(:any, 'http://blah.blah/success?a=1')
+    end
+
+    it 'responds "Unprocessable Entity" when incomplete params passed' do
+      response = post("#{host_uri}/gather", bob: ':(')
+      expect(response.status).to eql 422
+    end
+
+    it 'responds with uid for each request' do
+      params = { requests: [{ url: 'http://blah.blah/success' }] }
+      response = post("#{host_uri}/gather", params)
+      expect(JSON.parse(response.body)['requests'].first['uid'])
+        .to_not be_empty
+    end
+
+    it 'responds with a hash that respects url, query string and headers' do
+      params1 = { url: 'http://blah.blah/success' }
+      params2 = { url: 'http://blah.blah/success?a=1' }
+      params3 = { url: 'http://blah.blah/success',
+                  headers: { 'Content-Type' => 'whatever' } }
+
+      responses = [params1, params2, params3].map do |params|
+        post("#{host_uri}/gather", requests: [params])
+      end
+      hashes = responses.map do |res|
+        JSON.parse(res.body)['requests'].first['hash']
+      end
+      expect(hashes.uniq.length).to eql 3
+    end
+  end
+
+  describe '#gather' do
+    it 'respects a given url' do
+      stub = stub_request(:any, 'http://blah.blah/success')
+      params = { requests: [{ url: 'http://blah.blah/success' }] }
+      post("#{host_uri}/gather", params)
+      expect(stub).to have_been_requested
+    end
+
+    it 'respects given query parameters' do
+      stub = stub_request(:any, 'http://blah.blah/success?a=1')
+      params = { requests: [{ url: 'http://blah.blah/success?a=1' }] }
+      post("#{host_uri}/gather", params)
+      expect(stub).to have_been_requested
+    end
+
+    it 'respects given headers' do
+      stub = stub_request(:any, 'http://blah.blah/success')
+             .with(headers: { 'Content-Type' => 'whatever' })
+      request = { url: 'http://blah.blah/success',
+                  headers: { 'Content-Type' => 'whatever' } }
+      post("#{host_uri}/gather", requests: [request])
+      expect(stub).to have_been_requested
+    end
+
+    it 'respects given http method' do
+      stub = stub_request(:post, 'http://blah.blah/success?a=1')
+      params = { requests: [{ url: 'http://blah.blah/success?a=1',
+                              method: 'POST' }] }
+      post("#{host_uri}/gather", params)
+      expect(stub).to have_been_requested
+    end
+  end
+
+  private
+
+  def get(uri)
+    Faraday.get(uri)
+  end
+
+  def post(uri, params)
+    parsed = URI.parse(uri)
+    base_uri = "#{parsed.scheme}://#{parsed.host}:#{parsed.port}"
+    conn = Faraday.new(url: base_uri)
+    conn.post do |request|
+      request.url parsed.path
+      request.body = params.to_json
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'web_fetch'
+require 'pp'
+require 'byebug'
+require 'webmock/rspec'
+
+WebMock.disable_net_connect!(allow_localhost: true)
+
+# This is pretty ugly but seems to do the job
+puts 'Starting test server'
+WebFetch::Logger.logger(File::NULL)
+
+Thread.new do
+  EM.run do
+    EM.start_server 'localhost', 8089, WebFetch::Server
+  end
+end
+waiting = true
+while waiting
+  begin
+    res = Faraday.get('http://localhost:8089/')
+  rescue Faraday::ConnectionFailed
+    res = nil
+  end
+  waiting = !res.nil? && res.status != 200
+  sleep 0.1
+end
+puts 'Test server started'
+
+module WebFetch
+  class MockServer
+    def gather(requests); end
+
+    def storage
+      Storage
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+
+  config.order = :random
+
+  Kernel.srand config.seed
+end

data/spec/storage_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+describe WebFetch::Storage do
+  describe '.store' do
+    it 'accepts a key and value to store' do
+      expect do
+        described_class.store(:key, :value)
+      end.to_not raise_error
+    end
+  end
+
+  describe '.fetch' do
+    it 'fetches stored values' do
+      described_class.store(:key, :value)
+      expect(described_class.fetch(:key)).to eql :value
+    end
+
+    it 'removes item from storage after retrieval' do
+      described_class.store(:key, :value)
+      described_class.fetch(:key)
+      expect(described_class.fetch(:key)).to be_nil
+    end
+  end
+end

data/swagger.yaml

@@ -0,0 +1,115 @@
+swagger: '2.0'
+info:
+  title: WebFetch API
+  description: Asynchronously fetch HTTP entities
+  version: "0.0.1"
+schemes:
+  - http
+produces:
+  - application/json
+paths:
+  /:
+    get:
+      summary: Status
+      description: |
+        Identifies WebFetch server.
+      responses:
+        200:
+          description: An object with an application identifier
+          schema:
+            type: object
+            properties:
+              application:
+                type: string
+                description: WebFetch
+                
+  /gather:
+    post:
+      summary: Initiate fetching one or more HTTP entities.
+      description: |
+        Receives an array of HTTP entities as objects and returns an array of objects providing a unique identifier, a hash, and the original request parameters. The unique identifier can be used to retrieve the entity when it has completed downloading.
+      parameters:
+        - name: requests
+          in: body
+          description: HTTP entities to be gathered.
+          required: true
+          schema:
+            type: array
+            items:
+              $ref: '#/definitions/Request'
+      responses:
+        200:
+          description: An array of objects providing job IDs and original parameters.
+          schema:
+            $ref: '#/definitions/GatherResponse'
+              
+
+  /retrieve/{id}:
+    get:
+      summary: Retrieve a gathered HTTP entity.
+      description: |
+        Receives a unique identifier and returns the previously requested HTTP entity. This action will block until the entity has been successfully downloaded.
+      parameters:
+        - name: id
+          in: path
+          type: string
+          description: Unique identifier for HTTP entity.
+          required: true
+      responses:
+        200:
+          description: An object containing HTTP entity elements.
+          schema:
+            type: object
+            items:
+              $ref: '#/definitions/Retrieved'
+    
+definitions:
+  Request:
+    type: object
+    properties:
+      url:
+        type: string
+        description: URL of desired HTTP entity.
+      method:
+        type: string
+        default: GET
+        description: HTTP method.
+      headers:
+        type: object
+        description: HTTP headers.
+      query:
+        type: object
+        description: Query parameters.
+      body:
+        type: string
+        description: HTTP body.
+  Retrieved:
+    type: object
+    properties:
+      response:
+        type: object
+        description: Requested HTTP entity elements.
+        properties:
+          success:
+            type: boolean
+          body:
+            type: string
+          headers:
+            type: object
+          status:
+            type: integer
+            
+  GatherResponse:
+    type: array
+    items:
+      type: object
+      properties:
+        uid:
+          type: string
+          description: Unique identifier for requested HTTP entity.
+        hash:
+          type: string
+          description: SHA1 hash of request based on url, query parameters, headers, method [currently this serves no purpose].
+        request:
+          type: object
+          description: Original requested HTTP parameters.

data/web_fetch.gemspec

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'web_fetch/version'
+
+# rubocop:disable Metrics/BlockLength
+Gem::Specification.new do |s|
+  s.name          = 'web_fetch'
+  s.version       = WebFetch::VERSION
+  s.date          = '2016-10-16'
+  s.summary       = 'Async HTTP fetcher'
+  s.description   = 'Fetches HTTP responses as batch requests concurrently'
+  s.authors       = ['Bob Farrell']
+  s.email         = 'robertanthonyfarrell@gmail.com'
+  s.files         = `git ls-files`.split($RS)
+  s.homepage      = 'https://github.com/bobf/web_fetch'
+  s.licenses      = ['MIT']
+  s.require_paths = ['lib']
+  s.executables << 'web_fetch_server'
+  s.executables << 'web_fetch_control'
+
+  s.add_dependency 'activesupport', '~> 4.0'
+  s.add_dependency 'childprocess', '~> 0.5'
+  s.add_dependency 'daemons', '~> 1.2'
+  s.add_dependency 'em-http-request', '~> 1.1'
+  s.add_dependency 'em-logger', '~> 0.1'
+  s.add_dependency 'eventmachine', '~> 1.0'
+  s.add_dependency 'eventmachine_httpserver', '~> 0.2'
+  s.add_dependency 'faraday', '~> 0.9'
+  s.add_dependency 'hanami-router', '~> 0.7'
+  s.add_dependency 'hanami-utils', '0.8.0'
+  s.add_dependency 'i18n', '~> 0.7'
+  s.add_dependency 'rack', '~> 1.6'
+
+  s.add_development_dependency 'byebug', '~> 9.0'
+  s.add_development_dependency 'rspec', '~> 3.5'
+  s.add_development_dependency 'rubocop', '~> 0.59.2'
+  s.add_development_dependency 'webmock', '~> 3.4'
+end
+# rubocop:enable Metrics/BlockLength