routemaster-drain 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc848f2c1b75d8d36a4807ac8d8f0d871f9fca29
+  data.tar.gz: 5c627ab91e9497abb0acb81227484650f280853c
+SHA512:
+  metadata.gz: fdcd9b0cb52cb767ac97d9b7df8b7c26c1ff92755487b7801748f488b3946c256079e441a8c9264768c858b3b12f70eed38b6a23fd93a2d48526a40642314445
+  data.tar.gz: 2cb2c4af8600340854423def3364f3e1d4f3d51627765d2e6f8a4579e70c25ba31ee38058acf7b37941be52b4f00e1ad3d472ed9607590f4f242043b2d65b232

data/.env.test

@@ -0,0 +1,6 @@
+REDIS_TEST_URL=redis://localhost/13
+ROUTEMASTER_DRAIN_REDIS=redis://localhost/13
+ROUTEMASTER_CACHE_REDIS=redis://localhost/12
+ROUTEMASTER_CACHE_AUTH=example.com:username:s3cr3t
+ROUTEMASTER_DRAIN_TOKENS=d3m0
+

data/.gitignore

@@ -0,0 +1,7 @@
+.bundle/
+doc/
+.yardoc/
+pkg/
+tmp
+*.orig
+tags

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "rebund"]
+	path = rebund
+	url = https://github.com/mezis/rebund.git

data/.rspec

@@ -0,0 +1,3 @@
+-I.
+--color
+--format progress

data/.ruby-version

@@ -0,0 +1 @@
+2.1.2

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+services:
+  - redis-server
+rvm:
+  - 2.1.1
+  - 2.1.2
+script:
+- bundle exec rspec
+install:
+- "./rebund/run download"
+- bundle install --path vendor/bundle
+after_script:
+- "./rebund/run upload"
+env:
+  global:
+    secure: "kMxJcT8xyQ+QyCeKW5lDP44IU99Y8iN1AWZo9Z1BKN6HigGa8Ua8O/aD8AJjrTfRRSjnM402utmW6mk78RYVxJha3+69jJYr25I9EeyYhV2cuRu+5Ictxfcb9R9VhS0b6LDncz9h55xDS1UA35rvj+EOO4/M9moccw7T2qzPa2U="

data/.yardopts

@@ -0,0 +1,6 @@
+--no-private
+--protected
+--markup-provider=redcarpet
+--markup=markdown
+
+lib/**/*.rb - README.md LICENSE.txt

data/Gemfile

@@ -0,0 +1,17 @@
+source ENV.fetch('GEM_SOURCE', 'https://rubygems.org')
+
+# Specify your gem's dependencies in routemaster_client.gemspec
+gemspec
+
+# Just here to avoid a safety warning
+gem 'psych',        require: false
+
+# Used in builds and tests
+gem 'bundler',      require: false
+gem 'rake',         require: false
+gem 'guard-rspec',  require: false
+gem 'webmock',      require: false
+gem 'pry-nav',      require: false
+gem 'rack-test',    require: false
+gem 'dotenv',       require: false
+

data/Gemfile.lock

@@ -0,0 +1,122 @@
+PATH
+  remote: .
+  specs:
+    routemaster-drain (1.0.0)
+      faraday
+      faraday_middleware
+      hashie
+      net-http-persistent
+      rack
+      redis-namespace
+      resque
+      thread
+      wisper
+
+GEM
+  remote: http://eu.yarp.io/
+  specs:
+    addressable (2.3.6)
+    celluloid (0.16.0)
+      timers (~> 4.0.0)
+    coderay (1.1.0)
+    crack (0.4.2)
+      safe_yaml (~> 1.0.0)
+    diff-lcs (1.2.5)
+    dotenv (0.11.1)
+      dotenv-deployment (~> 0.0.2)
+    dotenv-deployment (0.0.2)
+    faraday (0.9.0)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.9.1)
+      faraday (>= 0.7.4, < 0.10)
+    ffi (1.9.3)
+    formatador (0.2.5)
+    guard (2.6.1)
+      formatador (>= 0.2.4)
+      listen (~> 2.7)
+      lumberjack (~> 1.0)
+      pry (>= 0.9.12)
+      thor (>= 0.18.1)
+    guard-rspec (4.3.1)
+      guard (~> 2.1)
+      rspec (>= 2.14, < 4.0)
+    hashie (3.3.1)
+    hitimes (1.2.2)
+    listen (2.7.9)
+      celluloid (>= 0.15.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.9)
+    method_source (0.8.2)
+    mono_logger (1.1.0)
+    multi_json (1.10.1)
+    multipart-post (2.0.0)
+    net-http-persistent (2.9.4)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-nav (0.2.4)
+      pry (>= 0.9.10, < 0.11.0)
+    psych (2.0.5)
+    rack (1.5.2)
+    rack-protection (1.5.3)
+      rack
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rake (10.3.2)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.5)
+      ffi (>= 0.5.0)
+    redis (3.1.0)
+    redis-namespace (1.5.1)
+      redis (~> 3.0, >= 3.0.4)
+    resque (1.25.2)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.3)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.2)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.0)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.0)
+    safe_yaml (1.0.3)
+    sinatra (1.4.5)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    slop (3.6.0)
+    thor (0.19.1)
+    thread (0.1.4)
+    tilt (1.4.1)
+    timers (4.0.0)
+      hitimes
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+    webmock (1.18.0)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+    wisper (1.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler
+  dotenv
+  guard-rspec
+  pry-nav
+  psych
+  rack-test
+  rake
+  routemaster-drain!
+  webmock

data/Guardfile

@@ -0,0 +1,10 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})   { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')        { "spec" }
+  watch(%r{^spec/support/(.+)\.rb$})  { "spec" }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 HouseTrip Ltd.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,261 @@
+# routemaster-drain
+
+A Rack-based event receiver for the
+[Routemaster](https://github.com/HouseTrip/routemaster) event bus.
+
+![Version](https://badge.fury.io/rb/routemaster-drain.svg)
+ 
+![Build](https://travis-ci.org/HouseTrip/routemaster-drain.svg?branch=master)
+ 
+[![](http://img.shields.io/badge/API%20docs-rubydoc.info-blue.svg)](http://rubydoc.info/github/HouseTrip/routemaster-drain/frames/file/README.md)
+
+`routemaster-drain` is a collection of Rack middleware to receive and
+parse Routemaster events, filter them, and preemptively cache the corresponding
+resources.
+
+It provides prebuilt middleware stacks (`Basic`, `Mapping`, and `Caching`) for
+typical use cases, illustrated below, or you can easily roll your own by
+combining middleware.
+
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+  - [Installation](#installation)
+  - [Illustrated use cases](#illustrated-use-cases)
+    - [Simply receive events from Routemaster](#simply-receive-events-from-routemaster)
+    - [Receive change notifications without duplicates](#receive-change-notifications-without-duplicates)
+    - [Cache data for all notified resources](#cache-data-for-all-notified-resources)
+  - [Internals](#internals)
+    - [Dirty map](#dirty-map)
+    - [Filter](#filter)
+  - [Contributing](#contributing)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'routemaster-drain'
+
+**Configuration**
+
+This gem is configured through the environment, making 12factor compliance
+easier.
+
+- `ROUTEMASTER_DRAIN_TOKENS`: a comma-separated list of valid authentication
+  tokens, used by Routemaster to send you events.
+- `ROUTEMASTER_DRAIN_REDIS`: the URL of the Redis instance used for filtering
+  and dirty mapping. Required if you use either feature, ignored otherwise.
+  A namespace can be specified.
+  Example: `redis://user:s3cr3t@myhost:1234/12/ns`.
+- `ROUTEMASTER_CACHE_REDIS`: the URL of the Redis instance used for caching.
+  Required if you use the feature, ignored otherwise. Formatted like
+  `ROUTEMASTER_DRAIN_REDIS`.
+- `ROUTEMASTER_CACHE_EXPIRY`: if using the cache, for how long to cache
+  entries, in seconds. Default 1 year (31,536,000).
+- `ROUTEMASTER_CACHE_AUTH`: if using the cache, specifies what username/password
+  pairs to use to fetch resources. The format is a comma-separated list of
+  colon-separate lists of regexp, username, password values. Example:
+  `server1:user:p4ss,server2:user:p4ass`.
+- `ROUTEMASTER_QUEUE_NAME`: if using the cache, on which Resque queue the cache
+  population jobs should be enqueued.
+
+
+
+## Illustrated use cases
+
+
+### Simply receive events from Routemaster
+
+Provide a listener for events:
+
+```ruby
+class Listener
+  def on_events_received(batch)
+    batch.each do |event|
+      puts event.url
+    end
+  end
+end
+```
+
+Each event is a `Hashie::Mash` and responds to `type` (one of `create`,
+`update`, `delete`, or `noop`), `url` (the resource), and `t` (the event
+timestamp, in milliseconds since the Epoch).
+
+Create the app that will process events:
+
+```ruby
+require 'routemaster/drain/basic'
+$app = Routemaster::Drain::Basic.new
+```
+
+Bind the app to your listener:
+
+```ruby
+$app.subscribe(Listener.new)
+```
+
+And finally, mount your app to your subscription path:
+
+```ruby
+# typically in config.ru
+map '/events' do
+  run $app
+end
+```
+
+This relies on the excellent event bus from the [wisper
+gem](https://github.com/krisleech/wisper#wisper).
+
+
+### Receive change notifications without duplicates
+
+When reacting to changes of some resource, it's common to want to avoid
+receiving further change notifications until you've actually processed that
+resource.
+
+Possibly you'll want to process changes in batches at regular time intervals.
+
+For this purpose, use `Routemaster::Drain::Mapping`:
+
+```ruby
+require 'routemaster/drain/mapping'
+$app = Routemaster::Drain::Mapping.new
+```
+
+And mount it as usual:
+
+```ruby
+# in config.ru
+map('/events') { run $app }
+```
+
+Instead of processing events, you'll check for changes in the dirty map:
+
+```ruby
+require 'routemaster/dirty/map'
+$map = Routemaster::Dirty::Map.new
+
+every_5_minutes do
+  $map.sweep do |url|
+    # do something about this changed resource
+    true
+  end
+end
+```
+
+Until you've called `#sweep` and your block has returned `true`, you won't be
+bugged again — the dirty map acts as a buffer of changes (see below for
+internals).
+
+Notes:
+- You can limit the number of resources to be swept (`$map.sweep(123) { ... }`).
+- You can count the number of resources to be swept with `$map.count`.
+- You're not told _what_ is to be swept; entities won't be swept in the order of
+  events received (much like Routemaster does not guarantee ordering).
+- If your sweeper fails, the dirty map will not be cleaned, so you can have
+  leftovers. It's good practice to regularly run `$map.sweep { ... }` and perform
+  cleanup regularly.
+- The map won't tell you if the resources has been changed, created, or deleted.
+  You'll have to figure it out with an API call.
+- You can still attach a listener to the app to get all events.
+
+
+### Cache data for all notified resources
+
+Another common use case is that you'll actually need the representation of the
+resources Routemaster tells you about.
+
+The `Caching` prebuilt app can do that for you, using Resque to populate the
+cache as events are received.
+
+For this purpose, use `Routemaster::Drain::Caching`:
+
+```ruby
+require 'routemaster/drain/machine'
+$app = Routemaster::Drain:Caching.new
+```
+
+And mount it as usual:
+
+```ruby
+# in config.ru
+map('/events') { run $app }
+```
+
+You can still attach a listenenr if you want the incoming events. Typically,
+what you'll want is the cache:
+
+```ruby
+require 'routemaster/cache'
+$cache = Routemaster::Cache.new
+
+response = @cache.fget('https://example.com/widgets/123')
+puts response.body.id
+```
+
+In this example, is your app was notified by Routemaster about Widget #123, the
+cache will be very likely to be hit; and it will be invalidated automatically
+whenever the drain gets notified about a change on that widget.
+
+Note that `Cache#fget` is a future, so you can efficiently query many resources
+and have any `HTTP GET` requests (and cache queries) happen in parallel.
+
+See
+[rubydoc](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Cache)
+for more details on `Cache`.
+
+
+## Internals
+
+The more elaborate drains are built with two components which can also be used
+independently,
+[`Dirty::Map`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Map)
+and
+[`Dirty::Filter`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Filter).
+
+### Dirty map
+
+A dirty map collects entities that have been created, updated, or deleted (or
+rather, their URLs).  It can be used to delay your service's reaction to events,
+for instance combined with Resque.
+
+A dirty map map gets _marked_ when an event about en entity gets processed that
+indicates a state change, and _swept_ to process those changes.
+
+Practically, instances of
+[`Routemaster::Dirty::Map`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Map)
+will emit a `:dirty_entity` event when a URL is marked as dirty, and can be
+swept when an entity is "cleaned".  If a URL is marked multiple times before
+being swept (e.g. for very volatile entities), the event will only by broadcast
+once.
+
+To sweep the map, you can for instance listen to this event and call
+[`#sweep_one`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Map#sweep_one-instance_method).
+
+If you're not in a hurry and would rather run through batches you can call
+[`#sweep`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Map#sweep-instance_method)
+which will yield URLs until it runs out of dirty resources.
+
+### Filter
+
+[`Routemaster::Dirty::Filter`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Filter) is a simple event filter
+that performs reordering. It ignores events older than the latest known
+information on an entity.
+
+It stores transient state in Redis and will emit `:entity_changed` events
+whenever an entity has changed. This event can usefully be fed into a dirty map,
+as in `Receiver::Filter` for instance.
+
+
+## Contributing
+
+1. Fork it ( http://github.com/HouseTrip/routemaster-drain/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request