routemaster-drain 1.0.0

data/lib/routemaster/drain/terminator.rb

@@ -0,0 +1,31 @@
+require 'wisper'
+
+module Routemaster
+  module Drain
+    # Tiny Rack app to terminates a Routemaster middleware chain.
+    # 
+    # Respond 204 if a payload has been parsed (i.e. present in the environment)
+    # and 400 if not.
+    #
+    # If an event payload has been placed in `env['routemaster.payload']`
+    # by upper middleware, broadcasts the `:events_received` event with the
+    # payload.
+    #
+    # Nothing will be broadcast if the payload is empty.
+    #
+    class Terminator
+      include Wisper::Publisher
+
+      def call(env)
+        payload = env['routemaster.payload']
+        if payload.nil?
+          return [400, {'Content-Type' => 'text/plain'}, 'no payload parsed']
+        end
+
+        publish(:events_received, payload) if payload.any?
+        [204, {}, []]
+      end
+    end
+  end
+end
+

data/lib/routemaster/fetcher.rb

@@ -0,0 +1,63 @@
+require 'faraday'
+require 'faraday_middleware'
+require 'hashie'
+require 'routemaster/config'
+
+module Routemaster
+  # Fetches URLs from JSON APIs.
+  class Fetcher
+    module ClassMethods
+      # Calls `get` with the same arguments on a memoized instance
+      # for the URL's host.
+      def get(url, params:nil, headers:nil)
+        _connection_for(url).get(url, params:params, headers:headers)
+      end
+
+      private
+
+      def _connection_for(url)
+        host = URI.parse(url).host
+        @connections ||= {}
+        @connections[host] ||= new(host)
+      end
+    end
+    extend ClassMethods
+
+    def initialize(host)
+      @host = host
+    end
+
+    # Performs a GET HTTP request for the `url`, with optional
+    # query parameters (`params`) and additional headers (`headers`).
+    #
+    # @return an object that responds to `status` (integer), `headers` (hash),
+    # and `body`. The body is a `Hashie::Mash` if the response was JSON, a
+    # string otherwise.
+    def get(url, params:nil, headers:nil)
+      r = _connection.get(url, params, headers)
+      Hashie::Mash.new(status: r.status, headers: r.headers, body: r.body)
+    end
+
+    private
+    
+    def _connection
+      @_connection ||= Faraday.new do |f|
+        f.request  :retry, max: 2, interval: 100e-3, backoff_factor: 2
+        f.request  :basic_auth, *_uuid
+        f.response :mashify
+        f.response :json, content_type: /\bjson/
+        f.adapter  :net_http_persistent
+
+        # TODO: make these configurable
+        f.options.timeout      = 1.0
+        f.options.open_timeout = 1.0
+        f.ssl.verify           = false
+      end
+    end
+
+    def _uuid
+      @_uuid ||= Config.cache_auth[@host]
+    end
+  end
+end
+

data/lib/routemaster/jobs/cache.rb

@@ -0,0 +1,12 @@
+require 'routemaster/cache'
+
+module Routemaster
+  module Jobs
+    # Caches a URL using {Cache}.
+    class Cache
+      def self.perform(url)
+        Cache.new.get(url)
+      end
+    end
+  end
+end

data/lib/routemaster/jobs/cache_and_sweep.rb

@@ -0,0 +1,16 @@
+require 'routemaster/cache'
+require 'routemaster/dirty/map'
+
+module Routemaster
+  module Jobs
+    # Caches a URL using {Cache}, and sweeps the dirty map
+    # if sucessful.
+    class CacheAndSweep
+      def self.perform(url)
+        Dirty::Map.new.sweep_one(url) do
+          Cache.new.get(url)
+        end
+      end
+    end
+  end
+end

data/lib/routemaster/middleware/authenticate.rb

@@ -0,0 +1,59 @@
+require 'wisper'
+require 'base64'
+require 'routemaster/config'
+
+module Routemaster
+  module Middleware
+    # Authenticates requests according to the Routemaster spec.
+    #
+    # Broadcasts `:authenticate` with one of `:missing`, `failed`, or
+    # `:succeeded`.
+    #
+    # This is very close to `Rack::Auth::Basic`, in that HTTP Basic
+    # is used; but the password part is ignored. In other words, this performs
+    # token authentication using HTTP Basic.
+    #
+    class Authenticate
+      include Wisper::Publisher
+
+      # @param uuid [Enumerable] a set of accepted authentication tokens
+      def initialize(app, uuid: nil)
+        @app  = app
+        @uuid = uuid || Config.drain_tokens
+
+        unless @uuid.kind_of?(String) || @uuid.kind_of?(Enumerable)
+          raise ArgumentError, ':uuid must be a String or Enumerable'
+        end
+      end
+
+      def call(env)
+        unless _has_auth?(env)
+          publish(:authenticate, :missing, env)
+          return [401, {}, []]
+        end
+
+        unless _valid_auth?(env)
+          publish(:authenticate, :failed, env)
+          return [403, {}, []]
+        end
+
+        publish(:authenticate, :succeeded, env)
+        @app.call(env)
+      end
+
+      private
+
+      def _has_auth?(env)
+        env.has_key?('HTTP_AUTHORIZATION')
+      end
+
+      def _valid_auth?(env)
+        token = Base64.
+          decode64(env['HTTP_AUTHORIZATION'].gsub(/^Basic /, '')).
+          split(':').first
+        @uuid.include?(token)
+      end
+    end
+  end
+end
+

data/lib/routemaster/middleware/cache.rb

@@ -0,0 +1,29 @@
+require 'routemaster/cache'
+require 'routemaster/config'
+require 'routemaster/jobs/cache_and_sweep'
+require 'resque'
+
+module Routemaster
+  module Middleware
+    class Cache
+      def initialize(app, cache:nil, resque:nil, queue:nil)
+        @app    = app
+        @cache  = cache || Routemaster::Cache.new
+        @resque = resque || Resque
+        @queue  = queue || Config.queue_name
+      end
+
+      def call(env)
+        env.fetch('routemaster.dirty', []).each do |url|
+          @cache.bust(url)
+          @resque.enqueue_to(@queue, Routemaster::Jobs::CacheAndSweep, url)
+        end
+        @app.call(env)
+      end
+    end
+  end
+end
+
+
+
+

data/lib/routemaster/middleware/dirty.rb

@@ -0,0 +1,33 @@
+require 'routemaster/dirty/map'
+
+module Routemaster
+  module Middleware
+    # If an event payload was place in the environment
+    # (`env['routemaster.payload']`) by a previous middleware,
+    # mark each corresponding entity as dirty.
+    #
+    # All events are passed through.
+    #
+    # The dirty map is passed as `:map` to the constructor and must respond to
+    # `#mark` (like `Routemaster::Dirty::Map`).
+    class Dirty
+      def initialize(app, dirty_map:nil)
+        @app = app
+        @map = dirty_map || Routemaster::Dirty::Map.new
+      end
+
+      def call(env)
+        env['routemaster.dirty'] = dirty = []
+        env.fetch('routemaster.payload', []).each do |event|
+          next if event['type'] == 'noop'
+          next unless @map.mark(event['url'])
+          dirty << event['url']
+        end
+        @app.call(env)
+      end
+    end
+  end
+end
+
+
+

data/lib/routemaster/middleware/filter.rb

@@ -0,0 +1,26 @@
+require 'routemaster/dirty/filter'
+
+module Routemaster
+  module Middleware
+    # Filters event payloads passed in the environment (in
+    # `env['routemaster.payload']`), is any.
+    #
+    # Will use `Routemaster::Dirty::Filter` by default.
+    class Filter
+      # @param filter [Routemaster::Dirty::Filter] an event filter (optional;
+      # will be created using the `redis` and `expiry` options if not provided)
+      def initialize(app, filter:nil)
+        @app    = app
+        @filter = filter || Routemaster::Dirty::Filter.new
+      end
+
+      def call(env)
+        payload = env['routemaster.payload']
+        if payload && payload.any?
+          env['routemaster.payload'] = @filter.run(payload)
+        end
+        @app.call(env) 
+      end
+    end
+  end
+end

data/lib/routemaster/middleware/parse.rb

@@ -0,0 +1,43 @@
+require 'json'
+require 'hashie'
+
+module Routemaster
+  module Middleware
+    # Receives a JSON payload of Routemaster events and parses it.
+    #
+    # It also ignores anything but POST with `application/json` MIMEs.
+    #
+    # Lower middlewares (or the app) can access the parsed payload as a hash
+    # in +env['routemaster.payload']+
+    class Parse
+      def initialize(app)
+        @app  = app
+      end
+
+      def call(env)
+        if env['CONTENT_TYPE'] != 'application/json'
+          return [415, {}, []]
+        end
+        if payload = _extract_payload(env)
+          env['routemaster.payload'] = payload
+        else
+          return [400, {}, []]
+        end
+        @app.call(env)
+      end
+
+      private
+
+      def _extract_payload(env)
+        data = JSON.parse(env['rack.input'].read).map { |e| Hashie::Mash.new(e) }
+        return nil unless data.kind_of?(Array)
+        return nil unless data.all? { |e| e.t && e.type && e.topic && e.url }
+        return data
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end
+
+

data/lib/routemaster/middleware/root_post_only.rb

@@ -0,0 +1,18 @@
+module Routemaster
+  module Middleware
+    # Rejects all requests but POST to the root path
+    class RootPostOnly
+      def initialize(app)
+        @app  = app
+      end
+
+      def call(env)
+        return [404, {}, []] if env['PATH_INFO'] != '/'
+        return [405, {}, []] if env['REQUEST_METHOD'] != 'POST'
+        @app.call(env)
+      end
+    end
+  end
+end
+
+

data/lib/routemaster/redis_broker.rb

@@ -0,0 +1,42 @@
+require 'redis-namespace'
+require 'uri'
+require 'singleton'
+
+module Routemaster
+  class RedisBroker
+    include Singleton
+
+    def initialize
+      @_connections = {}
+      _cleanup
+    end
+
+    def get(url)
+      _check_for_fork
+      @_connections[url] ||= begin
+        parsed_url = URI.parse(url)
+        namespace = parsed_url.path.split('/')[2] || 'rm'
+        Redis::Namespace.new(namespace, redis: Redis.new(url: url))
+      end
+    end
+
+    def cleanup
+      _cleanup
+    end
+
+    private
+
+    def _check_for_fork
+      return if Process.pid != @_pid
+      _cleanup
+    end
+
+    def _cleanup
+      @_pid = Process.pid
+      @_connections.each_value(&:quit)
+      @_connections = {}
+    end
+
+  end
+end
+

data/routemaster-drain.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'routemaster/drain'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'routemaster-drain'
+  spec.version       = Routemaster::Drain::VERSION
+  spec.authors       = ['Julien Letessier']
+  spec.email         = ['julien.letessier@gmail.com']
+  spec.summary       = %q{Event receiver for the Routemaster bus}
+  spec.homepage      = 'http://github.com/HouseTrip/routemaster_drain'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.test_files    = spec.files.grep(%r{^spec/})
+  spec.require_paths = %w(lib)
+
+  spec.add_runtime_dependency     'faraday'
+  spec.add_runtime_dependency     'faraday_middleware'
+  spec.add_runtime_dependency     'net-http-persistent'
+  spec.add_runtime_dependency     'rack'
+  spec.add_runtime_dependency     'wisper'
+  spec.add_runtime_dependency     'hashie'
+  spec.add_runtime_dependency     'redis-namespace'
+  spec.add_runtime_dependency     'resque'
+  spec.add_runtime_dependency     'thread'
+end

data/spec/routemaster/cache_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper'
+require 'spec/support/uses_redis'
+require 'spec/support/uses_dotenv'
+require 'routemaster/cache'
+require 'routemaster/fetcher'
+
+describe Routemaster::Cache do
+  uses_dotenv
+  uses_redis
+
+  let(:fetcher) { double 'fetcher' }
+  let(:url) { make_url(1) }
+  let(:listener) { double 'listener' }
+  subject { described_class.new(fetcher: fetcher) }
+
+  def make_url(idx)
+    "https://example.com/stuff/#{idx}"
+  end
+
+  def make_response(idx)
+    Hashie::Mash.new(
+      status:   200,
+      headers:  {},
+      body:     { id: 123, t: idx }
+    )
+  end
+
+  shared_examples 'a response getter' do 
+    before do
+      @counter = 0
+      allow(fetcher).to receive(:get) do |url, **options|
+        make_response(@counter += 1)
+      end
+    end
+
+    it 'fetches the url' do
+      expect(fetcher).to receive(:get).with(url, anything)
+      performer.call(url).status
+    end
+
+    it 'passes the locale header' do
+      expect(fetcher).to receive(:get) do |url, **options|
+        expect(options[:headers]['Accept-Language']).to eq('fr')
+        make_response(1)
+      end
+      performer.call(url, locale: 'fr').status
+    end
+
+    it 'passes the version header' do
+      expect(fetcher).to receive(:get) do |url, **options|
+        expect(options[:headers]['Accept']).to eq('application/json;v=33')
+        make_response(1)
+      end
+      performer.call(url, version: 33).status
+    end
+
+    it 'uses the cache' do
+      expect(fetcher).to receive(:get).once
+      3.times { performer.call(url).status }
+      expect(performer.call(url).body.t).to eq(1)
+    end
+
+    context 'with a listener' do
+      before { subject.subscribe(listener) }
+
+      it 'emits :cache_miss' do
+        expect(listener).to receive(:cache_miss)
+        performer.call(url).status
+      end
+
+      it 'misses on different locale' do
+        expect(listener).to receive(:cache_miss).twice
+        performer.call(url, locale: 'en').status
+        performer.call(url, locale: 'fr').status
+      end
+
+      it 'emits :cache_miss' do
+        allow(listener).to receive(:cache_miss)
+        performer.call(url).status
+        expect(listener).to receive(:cache_hit)
+        performer.call(url).status
+      end
+    end
+  end
+
+
+  describe '#get' do
+    let(:performer) { subject.method(:get) }
+
+    it_behaves_like 'a response getter'
+  end
+
+
+  describe '#fget' do
+    let(:performer) { subject.method(:fget) }
+
+    it_behaves_like 'a response getter'
+  end
+
+
+  describe '#bust' do
+    it 'causes next get to query' do
+      expect(fetcher).to receive(:get).and_return(make_response(1)).exactly(:twice)
+      subject.get(url)
+      subject.bust(url)
+      subject.get(url)
+    end
+
+    it 'emits :cache_bust' do
+      subject.subscribe(listener, prefix: true)
+      expect(listener).to receive(:on_cache_bust).once
+      subject.bust(url)
+    end
+  end
+end