routemaster-drain 1.0.0

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/routemaster/cache.rb

@@ -0,0 +1,133 @@
+require 'routemaster/fetcher'
+require 'thread/pool'
+require 'thread/future'
+require 'singleton'
+require 'delegate'
+require 'json'
+require 'wisper'
+
+module Routemaster
+  # Caches GET requests.
+  #
+  # Emits `cache_bust`, `cache_hit`, and `cache_miss` events.
+  #
+  # The requests themselves are handled by {Fetcher}.
+  # Note that `Cache-Control` headers are intentionally ignored, as it is
+  # assumed one will call {#bust} when the cache becomes stale.
+  #
+  # This is for instance done automatically by {Middleware::Cache}
+  # upon receiving events from Routemaster.
+  #
+  class Cache
+    include Wisper::Publisher
+
+    # A pool of threads, used for parallel/future request processing.
+    class Pool < SimpleDelegator
+      include Singleton
+
+      def initialize
+        Thread.pool(5, 20).tap do |p|
+          # TODO: configurable pool size and trim timeout?
+          p.auto_trim!
+          p.idle_trim! 10 # 10 seconds
+          super p
+        end
+      end
+    end
+
+    # Wraps a future response, so it quacks exactly like an ordinary response.
+    class FutureResponse
+      extend Forwardable
+     
+      # The `block` is expected to return a {Response}
+      def initialize(&block)
+        @future = Pool.instance.future(&block)
+      end
+
+      # @!attribute status
+      # @return [Integer]
+      # Delegated to the `block`'s return value.
+      
+      # @!attribute headers
+      # @return [Hash]
+      # Delegated to the `block`'s return value.
+      
+      # @!attribute body
+      # @return pHashie::Mash]
+      # Delegated to the `block`'s return value.
+      
+      delegate :value => :@future
+      delegate %i(status headers body) => :value
+    end
+
+    class Response < Hashie::Mash
+      # @!attribute status
+      # Integer
+      
+      # @!attribute headers
+      # Hash
+      
+      # @!attribute body
+      # Hashie::Mash
+    end
+
+
+    def initialize(redis:nil, fetcher:nil)
+      @redis   = redis || Config.cache_redis
+      @expiry  = Config.cache_expiry
+      @fetcher = fetcher || Fetcher
+    end
+
+    # Bust the cache for a given URL
+    def bust(url)
+      @redis.del("cache:#{url}")
+      publish(:cache_bust, url)
+    end
+
+    # Get the response from a URL, from the cache if possible.
+    # Stores to the cache on misses.
+    #
+    # Different versions and locales are stored separately in the cache.
+    #
+    # @param version [Integer] The version to pass in headers, as `Accept: application/json;v=2`
+    # @param locale [String] The language to request in the `Accept-Language`
+    # header.
+    #
+    # @return [Response], which responds to `status`, `headers`, and `body`.
+    def get(url, version:nil, locale:nil)
+      key   = "cache:#{url}"
+      field = "v:#{version},l:#{locale}"
+
+      # check cache
+      if payload = @redis.hget(key, field)
+        publish(:cache_hit, url)
+        return Response.new(JSON.parse(payload))
+      end
+
+      # fetch data
+      headers = {
+        'Accept' => version ?
+          "application/json;v=#{version}" :
+          "application/json"
+      }
+      headers['Accept-Language'] = locale if locale
+      response = @fetcher.get(url, headers: headers)
+
+      # store in redis
+      @redis.hset(key, field, response.to_json)
+      @redis.expire(key, @expiry)
+
+      publish(:cache_miss, url)
+      Response.new(response)
+    end
+
+    # Like {#get}, but schedules any request in the background using a thread
+    # pool. Handy to issue lots of requests in parallel.
+    #
+    # @return [FutureResponse], which responds to `status`, `headers`, and `body`
+    # like [Response].
+    def fget(*args)
+      FutureResponse.new { get(*args) }
+    end
+  end
+end

data/lib/routemaster/config.rb

@@ -0,0 +1,57 @@
+require 'singleton'
+require 'hashie/rash'
+require 'set'
+require 'routemaster/redis_broker'
+
+module Routemaster
+  class Config
+    module Classmethods
+      def method_missing(method, *args, &block)
+        new.send(method, *args, &block)
+      end
+
+      def respond_to?(method, include_all = false)
+        new.respond_to?(method, include_all)
+      end
+    end
+    extend Classmethods
+
+    def drain_redis
+      RedisBroker.instance.get(ENV.fetch('ROUTEMASTER_DRAIN_REDIS'))
+    end
+
+    def cache_redis
+      RedisBroker.instance.get(ENV.fetch('ROUTEMASTER_CACHE_REDIS'))
+    end
+
+    def cache_expiry
+      Integer(ENV.fetch('ROUTEMASTER_CACHE_EXPIRY', 86_400 * 365))
+    end
+
+    def cache_auth
+      Hashie::Rash.new.tap do |result|
+        ENV.fetch('ROUTEMASTER_CACHE_AUTH', '').split(',').each do |entry|
+          host, username, password = entry.split(':')
+          result[Regexp.new(host)] = [username, password]
+        end
+      end
+    end
+
+    def queue_name
+      ENV.fetch('ROUTEMASTER_QUEUE_NAME', 'routemaster')
+    end
+
+    def drain_tokens
+      Set.new(ENV.fetch('ROUTEMASTER_DRAIN_TOKENS').split(','))
+    end
+
+    def url_expansions
+      Hashie::Rash.new.tap do |result|
+        ENV.fetch('ROUTEMASTER_URL_EXPANSIONS', '').split(',').each do |entry|
+          host, username, password = entry.split(':')
+          result[Regexp.new(host)] = [username, password]
+        end
+      end
+    end
+  end
+end

data/lib/routemaster/dirty/filter.rb

@@ -0,0 +1,49 @@
+require 'routemaster/dirty/state'
+require 'routemaster/config'
+module Routemaster
+  module Dirty
+    # Service object, filters an event payload, only include events that reflect
+    # an entity state that is _more recent_ than previously received events.
+    #
+    # Can be used to Ignore events received out-of-order (e.g. an `update` event
+    # about en entity received after the `delete` event for that same entity),
+    # given Routemaster makes no guarantee of in-order delivery of events.
+    #
+    class Filter
+      EXPIRY = 86_400
+
+      # @param redis [Redis, Redis::Namespace] a connection to Redis, used to
+      # persists the known state
+      def initialize(redis:nil)
+        @redis  = redis || Config.drain_redis
+        @expiry = Config.cache_expiry
+      end
+
+      # Process a payload, and returns part if this payload containing
+      # only the latest event for a given entity.
+      #
+      # Events are skipped if they are older than a previously processed
+      # event for the same entity.
+      #
+      # Order of kept events is not guaranteed to be preserved.
+      def run(payload)
+        events = {} # url -> event
+
+        payload.each do |event|
+          known_state = State.get(@redis, event['url'])
+
+          # skip events older than what we already know
+          next if known_state.t > event['t']
+
+          new_state = State.new(event['url'], event['t'])
+
+          next if new_state == known_state
+          new_state.save(@redis, @expiry)
+          events[event['url']] = event
+        end
+
+        events.values
+      end
+    end
+  end
+end

data/lib/routemaster/dirty/map.rb

@@ -0,0 +1,63 @@
+require 'delegate'
+require 'set'
+require 'wisper'
+
+module Routemaster
+  module Dirty
+    # Collects information about entities whose state has changed and need to be
+    # refreshed.
+    # Typically +mark+ is called when notified state has changed (e.g. from the
+    # bus) and +sweep+ when one wants to know what has changed.
+    # 
+    # Use case: when some entites are very volatile, the map will hold a "dirty"
+    # state for multiple updates until the client is ready to update.
+    #
+    class Map
+      include Wisper::Publisher
+      KEY = 'dirtymap:items'
+
+      def initialize(redis: nil)
+        @redis = redis || Config.drain_redis
+      end
+
+      # Marks an entity as dirty.
+      # Return true if newly marked, false if re-marking.
+      def mark(url)
+        @redis.sadd(KEY, url).tap do |marked|
+          publish(:dirty_entity, url) if marked
+        end
+      end
+
+      # Runs the block.
+      # The entity will only be marked as clean if the block returns truthy.
+      def sweep_one(url, &block)
+        return unless block.call(url)
+        @redis.srem(KEY, url)
+      end
+
+      def all
+        @redis.smembers(KEY)
+      end
+
+      # Yields URLs for dirty entitities.
+      # The entity will only be marked as clean if the block returns truthy.
+      # It is possible to call +next+ or +break+ from the block.
+      def sweep(limit = 0)
+        unswept = []
+        while url = @redis.spop(KEY)
+          unswept.push url
+          is_swept = !! yield(url)
+          unswept.pop if is_swept
+          break if (limit -=1).zero?
+        end
+      ensure
+        @redis.sadd(KEY, unswept) if unswept.any?
+      end
+
+      # Number of currently dirty entities.
+      def count
+        @redis.scard(KEY)
+      end
+    end
+  end
+end

data/lib/routemaster/dirty/state.rb

@@ -0,0 +1,33 @@
+require 'delegate'
+require 'set'
+
+module Routemaster
+  module Dirty
+    # Locale prepresentation of the state of an entity.
+    # - url (string): the entity's authoritative locator
+    # - t (datetime, UTC): when the state was last refreshed
+    class State < Struct.new(:url, :t)
+      KEY = 'dirtymap:state:%s'
+
+      # Given a `redis` instance, return
+      #
+      # - a "blank" state for that URL (with time stamp 0), if the state is
+      #   unknown; or
+      # - the entity state, if known.
+      def self.get(redis, url)
+        data = redis.get(KEY % url)
+        return new(url, 0) if data.nil?
+        Marshal.load(data)
+      end
+
+      # Given a `redis` instance, save the state, expiring after
+      # `expiry` seconds.
+      def save(redis, expiry)
+        data = Marshal.dump(self)
+        redis.set(KEY % url, data, ex: expiry)
+      end
+    end
+  end
+end
+
+

data/lib/routemaster/drain.rb

@@ -0,0 +1,5 @@
+module Routemaster
+  module Drain
+    VERSION = '1.0.0'
+  end
+end

data/lib/routemaster/drain/basic.rb

@@ -0,0 +1,40 @@
+require 'routemaster/middleware/root_post_only'
+require 'routemaster/middleware/authenticate'
+require 'routemaster/middleware/parse'
+require 'routemaster/drain/terminator'
+require 'rack/builder'
+require 'delegate'
+
+module Routemaster
+  module Drain
+    # Rack application which authenticates, parses, and broadcasts events
+    # received from Routemaster.
+    #
+    # See the various corresponding middleware for details on operation:
+    # {Middleware::Authenticate}, {Middleware::Parse}, and terminates with
+    # {Terminator}.
+    #
+    class Basic
+      extend Forwardable
+
+      def initialize(options = {})
+        @terminator = terminator = Terminator.new
+        @app = ::Rack::Builder.app do
+          use Middleware::RootPostOnly
+          use Middleware::Authenticate, options
+          use Middleware::Parse
+          run terminator
+        end
+      end
+
+      # delegate :call => :@app
+
+      def call(env)
+        @app.call(env)
+      end
+
+      delegate [:on, :subscribe] => :@terminator
+    end
+  end
+end
+

data/lib/routemaster/drain/caching.rb

@@ -0,0 +1,43 @@
+require 'routemaster/middleware/root_post_only'
+require 'routemaster/middleware/authenticate'
+require 'routemaster/middleware/parse'
+require 'routemaster/middleware/filter'
+require 'routemaster/middleware/dirty'
+require 'routemaster/middleware/cache'
+require 'routemaster/drain/terminator'
+require 'rack/builder'
+require 'delegate'
+
+module Routemaster
+  module Drain
+    # Rack application which authenticates, parses, filters, pushes to a dirty map,
+    # busts cache, schedules preemptive caching, and finally broadcasts events
+    # received from Routemaster.
+    #
+    # See the various corresponding middleware for details on operation:
+    # {Middleware::RootPostOnly}, {Middleware::Authenticate},
+    # {Middleware::Parse}, {Middleware::Filter}, {Middleware::Dirty},
+    # {Middleware::Cache} and {Terminator}.
+    #
+    class Caching
+      extend Forwardable
+
+      def initialize(options = {})
+        @terminator = terminator = Terminator.new
+        @app = ::Rack::Builder.new do
+          use Middleware::RootPostOnly
+          use Middleware::Authenticate, options
+          use Middleware::Parse
+          use Middleware::Filter,       options
+          use Middleware::Dirty,        options
+          use Middleware::Cache,        options
+          run terminator
+        end
+      end
+
+      delegate :call => :@app
+      delegate [:on, :subscribe] => :@terminator
+    end
+  end
+end
+

data/lib/routemaster/drain/mapping.rb

@@ -0,0 +1,43 @@
+require 'routemaster/middleware/root_post_only'
+require 'routemaster/middleware/authenticate'
+require 'routemaster/middleware/parse'
+require 'routemaster/middleware/filter'
+require 'routemaster/middleware/dirty'
+require 'routemaster/drain/terminator'
+require 'rack/builder'
+require 'delegate'
+
+module Routemaster
+  module Drain
+    # Rack application which authenticates, parses, filters, pushes to a dirty map,
+    # and finally broadcasts events received from Routemaster.
+    #
+    # The dirty map can be obtained, for further processing, using
+    # `Dirty::Map.new`.
+    #
+    # See the various corresponding middleware for details on operation:
+    # {Middleware::RootPostOnly}, {Middleware::Authenticate},
+    # {Middleware::Parse}, {Middleware::Filter}, {Middleware::Dirty},
+    # and {Terminator}.
+    #
+    class Mapping
+      extend Forwardable
+
+      def initialize(options = {})
+        @terminator = terminator = Terminator.new
+        @app = ::Rack::Builder.new do
+          use Middleware::RootPostOnly
+          use Middleware::Authenticate, options
+          use Middleware::Parse
+          use Middleware::Filter,       options
+          use Middleware::Dirty,        options
+          run terminator
+        end
+      end
+
+      delegate :call => :@app
+      delegate [:on, :subscribe] => :@terminator
+    end
+  end
+end
+