routemaster-drain 1.0.5 → 1.1.0

data/lib/routemaster/config.rb

@@ -37,6 +37,10 @@ module Routemaster
       end
     end
 
+    def queue_adapter
+      ENV.fetch('ROUTEMASTER_QUEUE_ADAPTER', 'resque').to_sym
+    end
+
     def queue_name
       ENV.fetch('ROUTEMASTER_QUEUE_NAME', 'routemaster')
     end

data/lib/routemaster/dirty/map.rb

@@ -8,7 +8,7 @@ module Routemaster
     # refreshed.
     # Typically +mark+ is called when notified state has changed (e.g. from the
     # bus) and +sweep+ when one wants to know what has changed.
-    # 
+    #
     # Use case: when some entites are very volatile, the map will hold a "dirty"
     # state for multiple updates until the client is ready to update.
     #

data/lib/routemaster/drain.rb

@@ -1,5 +1,5 @@
 module Routemaster
   module Drain
-    VERSION = '1.0.5'
+    VERSION = '1.1.0'
   end
 end

data/lib/routemaster/fetcher.rb

@@ -1,30 +1,23 @@
+require 'base64'
 require 'faraday'
 require 'faraday_middleware'
 require 'hashie'
 require 'routemaster/config'
+require 'routemaster/middleware/caching'
 
 module Routemaster
-  # Fetches URLs from JSON APIs.
   class Fetcher
-    module ClassMethods
-      # Calls `get` with the same arguments on a memoized instance
-      # for the URL's host.
-      def get(url, params:nil, headers:nil)
-        _connection_for(url).get(url, params:params, headers:headers)
-      end
-
-      private
-
-      def _connection_for(url)
-        host = URI.parse(url).host
-        @connections ||= {}
-        @connections[host] ||= new(host)
-      end
-    end
-    extend ClassMethods
+    DEFAULT_MIDDLEWARE = Routemaster::Middleware::Caching
 
-    def initialize(host)
-      @host = host
+    #
+    # Usage:
+    #
+    # You can extend Fetcher with custom middlewares like:
+    # Fetcher.new(middlewares: [[MyCustomMiddleWare, option1, option2]])
+    #
+    def initialize(middlewares: [], listener: nil)
+      @listener = listener
+      @middlewares = generate_middlewares(middlewares)
     end
 
     # Performs a GET HTTP request for the `url`, with optional
@@ -33,17 +26,21 @@ module Routemaster
     # @return an object that responds to `status` (integer), `headers` (hash),
     # and `body`. The body is a `Hashie::Mash` if the response was JSON, a
     # string otherwise.
-    def get(url, params:nil, headers:nil)
-      r = _connection.get(url, params, headers)
-      Hashie::Mash.new(status: r.status, headers: r.headers, body: r.body)
+    def get(url, params: {}, headers: {})
+      host = URI.parse(url).host
+      connection.get(url, params, headers.merge(auth_header(host)))
     end
 
     private
 
-    def _connection
-      @_connection ||= Faraday.new do |f|
+    def generate_middlewares(middlewares)
+      default_middleware = [DEFAULT_MIDDLEWARE, listener: @listener]
+      middlewares.unshift(default_middleware)
+    end
+
+    def connection
+      @connection ||= Faraday.new do |f|
         f.request  :retry, max: 2, interval: 100e-3, backoff_factor: 2
-        f.request  :basic_auth, *_uuid
         f.response :mashify
         f.response :json, content_type: /\bjson/
         f.adapter  :net_http_persistent
@@ -51,12 +48,16 @@ module Routemaster
         f.options.timeout      = ENV.fetch('ROUTEMASTER_CACHE_TIMEOUT', 1).to_f
         f.options.open_timeout = ENV.fetch('ROUTEMASTER_CACHE_TIMEOUT', 1).to_f
         f.ssl.verify           = ENV.fetch('ROUTEMASTER_CACHE_VERIFY_SSL', 'false') == 'true'
+
+        @middlewares.each do |middleware|
+          f.use(*middleware)
+        end
       end
     end
 
-    def _uuid
-      @_uuid ||= Config.cache_auth[@host]
+    def auth_header(host)
+      auth_string = Config.cache_auth.fetch(host, []).join(':')
+      { 'Authorization' => "Basic #{Base64.strict_encode64(auth_string)}" }
     end
   end
 end
-

data/lib/routemaster/jobs/backends/resque.rb

@@ -0,0 +1,25 @@
+require 'resque'
+require 'routemaster/jobs/job'
+
+module Routemaster
+  module Jobs
+    module Backends
+      class Resque
+        def initialize(adapter = nil)
+          @adapter = adapter || ::Resque
+        end
+
+        def enqueue(queue, job_class, *args)
+          job_data = Job.data_for(job_class, args)
+          @adapter.enqueue_to(queue, JobWrapper, job_data)
+        end
+
+        class JobWrapper
+          def self.perform(job_data)
+            Job.execute(job_data)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/routemaster/jobs/backends/sidekiq.rb

@@ -0,0 +1,27 @@
+require 'sidekiq'
+require 'routemaster/jobs/job'
+
+module Routemaster
+  module Jobs
+    module Backends
+      class Sidekiq
+        def initialize(adapter = nil)
+          @adapter = adapter || ::Sidekiq::Client
+        end
+
+        def enqueue(queue, job_class, *args)
+          job_data = Job.data_for(job_class, args)
+          @adapter.push('queue' => queue, 'class' => JobWrapper, 'args' => [job_data])
+        end
+
+        class JobWrapper
+          include ::Sidekiq::Worker
+
+          def perform(job_data)
+            Job.execute(job_data)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/routemaster/jobs/cache.rb

@@ -4,8 +4,8 @@ module Routemaster
   module Jobs
     # Caches a URL using {Cache}.
     class Cache
-      def self.perform(url)
-        Cache.new.get(url)
+      def perform(url)
+        Routemaster::Cache.new.get(url)
       end
     end
   end

data/lib/routemaster/jobs/cache_and_sweep.rb

@@ -6,7 +6,7 @@ module Routemaster
     # Caches a URL using {Cache}, and sweeps the dirty map
     # if sucessful.
     class CacheAndSweep
-      def self.perform(url)
+      def perform(url)
         Dirty::Map.new.sweep_one(url) do
           Cache.new.get(url)
         end

data/lib/routemaster/jobs/client.rb

@@ -0,0 +1,30 @@
+require 'routemaster/config'
+
+module Routemaster
+  module Jobs
+    class Client
+      extend Forwardable
+
+      def_delegators :@backend, :enqueue
+
+      def initialize(adapter = nil)
+        @backend = build_backend(adapter)
+      end
+
+      private
+
+      def build_backend(adapter)
+        case Config.queue_adapter
+        when :resque
+          require 'routemaster/jobs/backends/resque'
+          Backends::Resque.new(adapter)
+        when :sidekiq
+          require 'routemaster/jobs/backends/sidekiq'
+          Backends::Sidekiq.new(adapter)
+        else
+          raise "Unsupported queue adapter '#{Config.queue_adapter}"
+        end
+      end
+    end
+  end
+end

data/lib/routemaster/jobs/job.rb

@@ -0,0 +1,23 @@
+module Routemaster
+  module Jobs
+    class Job
+      class << self
+        def data_for(job_class, args)
+          { 'class' => job_class.to_s, 'args'  => args }
+        end
+
+        def execute(job_data)
+          job = create_job(job_data)
+          job.new.perform(*job_data['args'])
+        end
+
+        private
+
+        def create_job(job_data)
+          job_class = job_data['class']
+          Kernel.const_get(job_class)
+        end
+      end
+    end
+  end
+end

data/lib/routemaster/middleware/cache.rb

@@ -1,22 +1,22 @@
 require 'routemaster/cache'
 require 'routemaster/config'
+require 'routemaster/jobs/client'
 require 'routemaster/jobs/cache_and_sweep'
-require 'resque'
 
 module Routemaster
   module Middleware
     class Cache
-      def initialize(app, cache:nil, resque:nil, queue:nil)
+      def initialize(app, cache:nil, client:nil, queue:nil)
         @app    = app
         @cache  = cache || Routemaster::Cache.new
-        @resque = resque || Resque
+        @client = client || Routemaster::Jobs::Client.new
         @queue  = queue || Config.queue_name
       end
 
       def call(env)
         env.fetch('routemaster.dirty', []).each do |url|
           @cache.bust(url)
-          @resque.enqueue_to(@queue, Routemaster::Jobs::CacheAndSweep, url)
+          @client.enqueue(@queue, Routemaster::Jobs::CacheAndSweep, url)
         end
         @app.call(env)
       end

data/lib/routemaster/middleware/caching.rb

@@ -0,0 +1,71 @@
+require 'wisper'
+
+module Routemaster
+  module Middleware
+    class Caching
+      KEY_TEMPLATE = 'cache:{url}'
+      FIELD_TEMPLATE = 'v:{version},l:{locale}'
+      VERSION_REGEX = /application\/json;v=(?<version>\S*)/.freeze
+
+      def initialize(app, cache: Config.cache_redis, listener: nil)
+        @app = app
+        @cache = cache
+        @expiry  = Config.cache_expiry
+        @listener = listener
+      end
+
+      def call(env)
+        return @app.call(env) unless env.method == :get
+
+        fetch_from_cache(env) || fetch_from_service(env)
+      end
+
+      private
+
+      def fetch_from_service(env)
+        @app.call(env).on_complete do |response_env|
+          response = response_env.response
+
+          if response.success?
+            @cache.hset(cache_key(env), cache_field(env), response.body)
+            @cache.expire(cache_key(env), @expiry)
+            @listener._publish(:cache_miss, url(env)) if @listener
+          end
+        end
+      end
+
+      def fetch_from_cache(env)
+        payload = @cache.hget(cache_key(env), cache_field(env))
+
+        if payload
+          @listener._publish(:cache_hit, url(env)) if @listener
+          Faraday::Response.new(status: 200,
+                                body: payload,
+                                response_headers: {})
+        end
+      end
+
+      def cache_field(env)
+        FIELD_TEMPLATE
+        .gsub('{version}', version(env).to_s)
+          .gsub('{locale}', locale(env).to_s)
+      end
+
+      def cache_key(env)
+        KEY_TEMPLATE.gsub('{url}', url(env))
+      end
+
+      def url(env)
+        env.url.to_s
+      end
+
+      def version(env)
+        (env.request_headers['Accept'] || "")[VERSION_REGEX, 1]
+      end
+
+      def locale(env)
+        env.request_headers['Accept-Language']
+      end
+    end
+  end
+end

data/routemaster-drain.gemspec

@@ -18,11 +18,10 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency     'faraday', '>= 0.9.0'
   spec.add_runtime_dependency     'faraday_middleware'
-  spec.add_runtime_dependency     'net-http-persistent'
-  spec.add_runtime_dependency     'rack'
-  spec.add_runtime_dependency     'wisper', '>= 1.4.0'
+  spec.add_runtime_dependency     'net-http-persistent', '< 3' # 3.x is currently incompatible with faraday
+  spec.add_runtime_dependency     'rack', '>= 1.6.2'
+  spec.add_runtime_dependency     'wisper', '~> 1.6.1'
   spec.add_runtime_dependency     'hashie'
   spec.add_runtime_dependency     'redis-namespace'
-  spec.add_runtime_dependency     'resque'
   spec.add_runtime_dependency     'thread'
 end

data/spec/routemaster/cache_spec.rb

@@ -1,115 +1,103 @@
 require 'spec_helper'
 require 'spec/support/uses_redis'
 require 'spec/support/uses_dotenv'
+require 'spec/support/uses_webmock'
 require 'routemaster/cache'
 require 'routemaster/fetcher'
 
-describe Routemaster::Cache do
-  uses_dotenv
-  uses_redis
+module Routemaster
+  describe Cache do
+    uses_dotenv
+    uses_redis
+    uses_webmock
 
-  let(:fetcher) { double 'fetcher' }
-  let(:url) { make_url(1) }
-  let(:listener) { double 'listener' }
-  subject { described_class.new(fetcher: fetcher) }
+    let(:url) { 'https://www.example.com/widgets/123' }
 
-  def make_url(idx)
-    "https://example.com/stuff/#{idx}"
-  end
-
-  def make_response(idx)
-    Hashie::Mash.new(
-      status:   200,
-      headers:  {},
-      body:     { id: 123, t: idx }
-    )
-  end
-
-  shared_examples 'a response getter' do 
     before do
-      @counter = 0
-      allow(fetcher).to receive(:get) do |url, **options|
-        make_response(@counter += 1)
-      end
+      stub_request(:get, /example\.com/).to_return(
+        status:   200,
+        body:     { id: 123, type: 'widget' }.to_json,
+        headers:  {
+          'content-type' => 'application/json;v=1'
+        }
+      )
     end
 
-    it 'fetches the url' do
-      expect(fetcher).to receive(:get).with(url, anything)
-      performer.call(url).status
-    end
+    shared_examples 'a GET request' do
+      context 'with no options' do
+        let(:options) { {} }
 
-    it 'passes the locale header' do
-      expect(fetcher).to receive(:get) do |url, **options|
-        expect(options[:headers]['Accept-Language']).to eq('fr')
-        make_response(1)
-      end
-      performer.call(url, locale: 'fr').status
-    end
+        it 'calls get on the fetcher with no version and locale headers' do
+          expect_any_instance_of(Fetcher)
+            .to receive(:get)
+            .with(url, headers: { 'Accept' => 'application/json' })
+            .and_call_original
 
-    it 'passes the version header' do
-      expect(fetcher).to receive(:get) do |url, **options|
-        expect(options[:headers]['Accept']).to eq('application/json;v=33')
-        make_response(1)
+          perform.status
+        end
       end
-      performer.call(url, version: 33).status
-    end
 
-    it 'uses the cache' do
-      expect(fetcher).to receive(:get).once
-      3.times { performer.call(url).status }
-      expect(performer.call(url).body.t).to eq(1)
-    end
+      context 'with a specific version' do
+        let(:options) { { version: 2 } }
 
-    context 'with a listener' do
-      before { subject.subscribe(listener) }
+        it 'calls get on the fetcher with version header' do
+          expect_any_instance_of(Fetcher)
+            .to receive(:get)
+            .with(url, headers: { 'Accept' => 'application/json;v=2' })
+            .and_call_original
 
-      it 'emits :cache_miss' do
-        expect(listener).to receive(:cache_miss)
-        performer.call(url).status
+          perform.status
+        end
       end
 
-      it 'misses on different locale' do
-        expect(listener).to receive(:cache_miss).twice
-        performer.call(url, locale: 'en').status
-        performer.call(url, locale: 'fr').status
-      end
+      context 'with a specific locale' do
+        let(:options) { { locale: 'fr' } }
 
-      it 'emits :cache_miss' do
-        allow(listener).to receive(:cache_miss)
-        performer.call(url).status
-        expect(listener).to receive(:cache_hit)
-        performer.call(url).status
+        it 'calls get on the fetcher with locale header' do
+          expect_any_instance_of(Fetcher)
+            .to receive(:get)
+            .with(url, headers: { 'Accept' => 'application/json', 'Accept-Language' => 'fr' })
+            .and_call_original
+
+          perform.status
+        end
       end
     end
-  end
 
+    describe '#get' do
+      let(:perform) { subject.get(url, **options) }
 
-  describe '#get' do
-    let(:performer) { subject.method(:get) }
+      it_behaves_like 'a GET request'
+    end
 
-    it_behaves_like 'a response getter'
-  end
+    describe '#fget' do
+      let(:perform) { subject.fget(url, **options) }
 
+      it_behaves_like 'a GET request'
+    end
 
-  describe '#fget' do
-    let(:performer) { subject.method(:fget) }
+    describe '#bust' do
+      let(:cache)   { Config.cache_redis }
+      let(:perform) { subject.bust(url) }
 
-    it_behaves_like 'a response getter'
-  end
+      before do
+        cache.set("cache:#{url}", "cached response")
+      end
 
+      it 'busts the cache for a given URL' do
+        expect { perform }
+          .to change { cache.get("cache:#{url}") }
+          .from('cached response')
+          .to(nil)
+      end
 
-  describe '#bust' do
-    it 'causes next get to query' do
-      expect(fetcher).to receive(:get).and_return(make_response(1)).exactly(:twice)
-      subject.get(url)
-      subject.bust(url)
-      subject.get(url)
-    end
+      it 'publishes the cache_bust event for that URL' do
+        expect_any_instance_of(described_class)
+          .to receive(:publish)
+          .with(:cache_bust, url)
 
-    it 'emits :cache_bust' do
-      subject.subscribe(listener, prefix: true)
-      expect(listener).to receive(:on_cache_bust).once
-      subject.bust(url)
+        perform
+      end
     end
   end
 end