web_fetch 0.4.0 → 0.5.0

data/spec/router_spec.rb

@@ -10,7 +10,9 @@ describe WebFetch::Router do
   describe '#route' do
     it 'provides a route to GET /' do
       expect(router.route('/'))
-        .to eql(status: 200, payload: { application: 'WebFetch' })
+        .to eql(
+          status: 200, command: 'root', payload: { application: 'WebFetch' }
+        )
     end
 
     it 'provides a route to POST /gather' do
@@ -39,9 +41,9 @@ describe WebFetch::Router do
 
     it 'returns appropriate response when invaid json provided' do
       response = router.route('/gather',
-                            method: 'POST',
-                            query_string: 'json=uh oh :(',
-                            server: nil)
+                              method: 'POST',
+                              query_string: 'json=uh oh :(',
+                              server: nil)
       expect(response).to eql(status: 400, payload: I18n.t(:bad_json))
     end
   end

data/spec/spec_helper.rb

@@ -1,10 +1,13 @@
 # frozen_string_literal: true
 
-require 'web_fetch'
-require 'pp'
+require 'betterp'
 require 'byebug'
-require 'webmock/rspec'
+require 'pp'
 require 'rspec/its'
+require 'web_fetch'
+require 'webmock/rspec'
+
+require File.join(__dir__, 'storage', 'shared_examples')
 
 WebMock.disable_net_connect!(allow_localhost: true)
 
@@ -33,8 +36,12 @@ module WebFetch
   class MockServer
     def gather(requests); end
 
+    def self.storage
+      @storage ||= Storage::Memory.new
+    end
+
     def storage
-      Storage
+      self.class.storage
     end
   end
 end
@@ -44,13 +51,14 @@ RSpec.configure do |config|
     expectations.include_chain_clauses_in_custom_matcher_descriptions = true
   end
 
+  config.before(:each) { WebFetch::MockServer.storage.clear }
   config.mock_with :rspec do |mocks|
     mocks.verify_partial_doubles = true
   end
 
   config.shared_context_metadata_behavior = :apply_to_host_groups
 
-  config.order = :random
-
-  Kernel.srand config.seed
+  # config.order = :random
+  #
+  # Kernel.srand config.seed
 end

data/spec/storage/memcached_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+describe WebFetch::Storage::Memcached do
+  before(:all) do
+    class MockDalliClient
+      def initialize(*_args)
+        @state = {}
+      end
+
+      def set(key, value)
+        @state[key] = value
+      end
+
+      def get(key)
+        @state[key]
+      end
+
+      def delete(key)
+        @state.delete(key)
+      end
+    end
+  end
+
+  subject { described_class.new(MockDalliClient) }
+
+  it_behaves_like 'a storage adapter'
+end

data/spec/storage/memory_spec.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+describe WebFetch::Storage::Memory do
+  it_behaves_like 'a storage adapter'
+end

data/spec/storage/redis_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+describe WebFetch::Storage::Redis do
+  before(:all) do
+    class MockRedisClient
+      def initialize(*_args)
+        @state = {}
+      end
+
+      def set(key, value, _options = {})
+        @state[key] = value
+      end
+
+      def get(key)
+        @state[key]
+      end
+
+      def del(key)
+        @state.delete(key)
+      end
+    end
+  end
+
+  subject { described_class.new(MockRedisClient) }
+
+  it_behaves_like 'a storage adapter'
+end

data/spec/storage/shared_examples.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+RSpec.shared_examples 'a storage adapter' do
+  describe '.store' do
+    it 'accepts a key and value to store' do
+      expect do
+        subject.store(:key, 'value')
+      end.to_not raise_error
+    end
+  end
+
+  describe '.fetch' do
+    it 'fetches stored values' do
+      subject.store(:key, 'value')
+      expect(subject.fetch(:key)).to eql 'value'
+    end
+  end
+
+  describe '.delete' do
+    it 'deletes stored values' do
+      subject.store(:key, 'value')
+      expect(subject.fetch(:key)).to eql 'value'
+      subject.delete(:key)
+      expect(subject.fetch(:key)).to eql nil
+    end
+  end
+end

data/web_fetch.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.description   = 'Fetches HTTP responses as batch requests concurrently'
   s.authors       = ['Bob Farrell']
   s.email         = 'robertanthonyfarrell@gmail.com'
-  s.files         = `git ls-files`.split($RS)
+  s.files         = File.read(File.join(__dir__, 'manifest')).split
   s.homepage      = 'https://github.com/bobf/web_fetch'
   s.licenses      = ['MIT']
   s.require_paths = ['lib']
@@ -22,22 +22,26 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'activesupport', '>= 4.0'
   s.add_dependency 'daemons', '~> 1.2'
+  s.add_dependency 'dalli', '~> 2.7'
   s.add_dependency 'em-http-request', '~> 1.1'
-  s.add_dependency 'em-logger', '~> 0.1'
+  s.add_dependency 'em-logger', '~> 0.1.0'
   s.add_dependency 'eventmachine', '~> 1.0'
-  s.add_dependency 'eventmachine_httpserver', '~> 0.2'
-  s.add_dependency 'faraday', '~> 0.9'
+  s.add_dependency 'eventmachine_httpserver', '~> 0.2.1'
+  s.add_dependency 'faraday', '~> 0.15.4'
   s.add_dependency 'hanami-router', '~> 1.0'
   s.add_dependency 'hanami-utils', '~> 1.0'
   s.add_dependency 'i18n', '>= 0.7'
   s.add_dependency 'rack', '>= 1.6'
+  s.add_dependency 'redis', '~> 4.1'
   s.add_dependency 'subprocess', '~> 1.3'
 
+  s.add_development_dependency 'betterp', '~> 0.1.2'
   s.add_development_dependency 'byebug', '~> 9.0'
   s.add_development_dependency 'rake', '~> 12.3'
   s.add_development_dependency 'rspec', '~> 3.5'
   s.add_development_dependency 'rspec-its', '~> 1.2'
-  s.add_development_dependency 'rubocop', '~> 0.59.2'
+  s.add_development_dependency 'rubocop', '~> 0.60.0'
+  s.add_development_dependency 'strong_versions', '~> 0.3.2'
   s.add_development_dependency 'webmock', '~> 3.4'
 end
 # rubocop:enable Metrics/BlockLength

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: web_fetch
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Bob Farrell
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: dalli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
 - !ruby/object:Gem::Dependency
   name: em-http-request
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: eventmachine
   requirement: !ruby/object:Gem::Requirement
@@ -86,28 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: 0.2.1
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.15.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.15.4
 - !ruby/object:Gem::Dependency
   name: hanami-router
   requirement: !ruby/object:Gem::Requirement
@@ -164,6 +178,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
 - !ruby/object:Gem::Dependency
   name: subprocess
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: betterp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -240,14 +282,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.59.2
+        version: 0.60.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.60.0
+- !ruby/object:Gem::Dependency
+  name: strong_versions
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.59.2
+        version: 0.3.2
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -274,6 +330,7 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-version"
+- ".strong_versions.yml"
 - Gemfile
 - LICENSE
 - Makefile
@@ -282,6 +339,7 @@ files:
 - TODO
 - bin/rspec
 - bin/rubocop
+- bin/strong_versions
 - bin/web_fetch_control
 - bin/web_fetch_server
 - config/locales/en.yml
@@ -293,7 +351,6 @@ files:
 - lib/web_fetch.rb
 - lib/web_fetch/client.rb
 - lib/web_fetch/concerns/client_http.rb
-- lib/web_fetch/concerns/event_machine_helpers.rb
 - lib/web_fetch/concerns/http_helpers.rb
 - lib/web_fetch/concerns/validatable.rb
 - lib/web_fetch/errors.rb
@@ -308,7 +365,11 @@ files:
 - lib/web_fetch/router.rb
 - lib/web_fetch/server.rb
 - lib/web_fetch/storage.rb
+- lib/web_fetch/storage/memcached.rb
+- lib/web_fetch/storage/memory.rb
+- lib/web_fetch/storage/redis.rb
 - lib/web_fetch/version.rb
+- manifest
 - spec/client_spec.rb
 - spec/concerns/validatable_spec.rb
 - spec/features/http_fetching_spec.rb
@@ -323,7 +384,10 @@ files:
 - spec/router_spec.rb
 - spec/server_spec.rb
 - spec/spec_helper.rb
-- spec/storage_spec.rb
+- spec/storage/memcached_spec.rb
+- spec/storage/memory_spec.rb
+- spec/storage/redis_spec.rb
+- spec/storage/shared_examples.rb
 - swagger.yaml
 - web_fetch.gemspec
 homepage: https://github.com/bobf/web_fetch

data/lib/web_fetch/concerns/event_machine_helpers.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-module WebFetch
-  # EventMachine layer-specific helpers
-  module EventMachineHelpers
-    def request_async(target)
-      request = target[:request]
-      target[:start_time] = Time.now.utc
-      async_request = EM::HttpRequest.new(request[:url])
-      method = request.fetch(:method, 'GET').downcase.to_sym
-      async_request.public_send(
-        method,
-        head: request[:headers],
-        query: request.fetch(:query, {}),
-        body: request.fetch(:body, nil)
-      )
-    end
-
-    def apply_callbacks(request)
-      request[:deferred].callback do
-        Logger.debug("HTTP fetch complete for uid: #{request[:uid]}")
-        save_response_time(request)
-        request[:succeeded] = true
-      end
-
-      request[:deferred].errback do
-        Logger.debug("HTTP fetch failed for uid: #{request[:uid]}")
-        save_response_time(request)
-        request[:failed] = true
-      end
-    end
-
-    def wait_for_response(request, response)
-      tick_loop(request, response)
-    end
-
-    def tick_loop(request, response)
-      # XXX There may be a much nicer way to wait for an async task to complete
-      # before returning a response but I couldn't figure it out, so I used
-      # EM.tick_loop which effectively does the same as a Twisted deferred
-      # callback chain, just much more explicitly.
-      EM.tick_loop do
-        if request[:succeeded]
-          succeed(request, response)
-          :stop
-        elsif request[:failed]
-          fail_(request, response)
-          :stop
-        end
-      end
-    end
-
-    def save_response_time(request)
-      request[:response_time] = Time.now.utc - request[:start_time]
-    end
-  end
-end