routemaster-drain 1.0.5 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (34) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +5 -1
  3. data/.gitmodules +0 -3
  4. data/.ruby-version +1 -1
  5. data/.travis.yml +4 -10
  6. data/Gemfile +14 -10
  7. data/Gemfile.lock +96 -62
  8. data/LICENSE.txt +2 -1
  9. data/README.md +13 -11
  10. data/lib/routemaster/cache.rb +16 -41
  11. data/lib/routemaster/config.rb +4 -0
  12. data/lib/routemaster/dirty/map.rb +1 -1
  13. data/lib/routemaster/drain.rb +1 -1
  14. data/lib/routemaster/fetcher.rb +29 -28
  15. data/lib/routemaster/jobs/backends/resque.rb +25 -0
  16. data/lib/routemaster/jobs/backends/sidekiq.rb +27 -0
  17. data/lib/routemaster/jobs/cache.rb +2 -2
  18. data/lib/routemaster/jobs/cache_and_sweep.rb +1 -1
  19. data/lib/routemaster/jobs/client.rb +30 -0
  20. data/lib/routemaster/jobs/job.rb +23 -0
  21. data/lib/routemaster/middleware/cache.rb +4 -4
  22. data/lib/routemaster/middleware/caching.rb +71 -0
  23. data/routemaster-drain.gemspec +3 -4
  24. data/spec/routemaster/cache_spec.rb +69 -81
  25. data/spec/routemaster/drain/caching_spec.rb +2 -2
  26. data/spec/routemaster/fetcher_spec.rb +6 -2
  27. data/spec/routemaster/integration/cache_spec.rb +68 -0
  28. data/spec/routemaster/jobs/backends/backend_examples.rb +21 -0
  29. data/spec/routemaster/jobs/backends/resque_spec.rb +16 -0
  30. data/spec/routemaster/jobs/backends/sidekiq_spec.rb +11 -0
  31. data/spec/routemaster/jobs/client_spec.rb +46 -0
  32. data/spec/routemaster/middleware/cache_spec.rb +7 -8
  33. data/spec/spec_helper.rb +5 -0
  34. metadata +28 -27
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c0272cbd5f92879aa1a44d0b928d5671db23568d
4
- data.tar.gz: a0c52c2160834a7542817358896b2d25cb6adc6a
3
+ metadata.gz: a38994768fac0a952e0072791c9318b2c87912be
4
+ data.tar.gz: 0e2acec86ae92448297a2ba916ef1ca61bdaefdb
5
5
  SHA512:
6
- metadata.gz: 265ccd4eaa6f7fdb28f59f5a104b631341bf2858cbd940bef9825eab39539f8fcf64dc9600ccc1470f047f5c2585ead66b1d5b5955fe377e3a479d5e38160df7
7
- data.tar.gz: 6c98dbc0dc65777ed88cc223b36b0ffe2017918481cf1a675b696dc9f5331380725a5f05fe1bb4ebfecfd6878e05bdd8fee23a354e629d18deaa4eb01798a508
6
+ metadata.gz: 76718e5f07d9c5616e626eafcddb19323398f756f47f177fc4f0c34fa50e711346cd993bf71c4ec161cce3b4eedbe43868b71e8a89902ebacc91412de4d9bea7
7
+ data.tar.gz: 0fa0392ec5962ffdc66023280e5503427609d9a2c3cefb827022d63acf4b29e1f55a3f838e7452f3bafa47f9dcb53f8555a596d401e469f0d538167fc0a89897
data/.gitignore CHANGED
@@ -4,4 +4,8 @@ doc/
4
4
  pkg/
5
5
  tmp
6
6
  *.orig
7
- tags
7
+ tags
8
+ *.swp
9
+ *.swo
10
+ coverage/
11
+ .byebug_history
data/.gitmodules CHANGED
@@ -1,3 +0,0 @@
1
- [submodule "rebund"]
2
- path = rebund
3
- url = https://github.com/mezis/rebund.git
data/.ruby-version CHANGED
@@ -1 +1 @@
1
- 2.1.2
1
+ 2.3.1
data/.travis.yml CHANGED
@@ -1,16 +1,10 @@
1
1
  language: ruby
2
+ cache: bundler
2
3
  services:
3
4
  - redis-server
4
5
  rvm:
5
- - 2.1.1
6
- - 2.1.2
6
+ - 2.2.5
7
+ - 2.3.1
7
8
  script:
8
9
  - bundle exec rspec
9
- install:
10
- - "./rebund/run download"
11
- - bundle install --path vendor/bundle
12
- after_script:
13
- - "./rebund/run upload"
14
- env:
15
- global:
16
- secure: BOuoRqbmovhZUAs6mpYo8yXEaHwOvloXsCaVKr84Aub5l9GlyoaPeOs2HDGzWMLXVSI+N1Zf4or+tqWwrwnWhrqb2rj4yVd6tp/qm0V8ICw34W7RL30eWYe/Dcw2uGROAqn56xAj2pXZZY4qLb4dgoXitM+riHkq8rPvqohywZ0=
10
+ - bundle exec codeclimate-test-reporter
data/Gemfile CHANGED
@@ -1,17 +1,21 @@
1
- source ENV.fetch('GEM_SOURCE', 'https://rubygems.org')
1
+ source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in routemaster_client.gemspec
4
4
  gemspec
5
5
 
6
6
  # Just here to avoid a safety warning
7
- gem 'psych', require: false
7
+ gem 'psych', require: false
8
8
 
9
9
  # Used in builds and tests
10
- gem 'bundler', require: false
11
- gem 'rake', require: false
12
- gem 'guard-rspec', require: false
13
- gem 'webmock', require: false
14
- gem 'pry-nav', require: false
15
- gem 'rack-test', require: false
16
- gem 'dotenv', require: false
17
-
10
+ gem 'bundler', require: false
11
+ gem 'rake', require: false
12
+ gem 'guard-rspec', require: false
13
+ gem 'webmock', require: false
14
+ gem 'pry-byebug', require: false
15
+ gem 'rack-test', require: false
16
+ gem 'dotenv', require: false
17
+ gem 'codeclimate-test-reporter', require: false
18
+ gem 'simplecov'
19
+ gem 'rspec'
20
+ gem 'resque'
21
+ gem 'sidekiq'
data/Gemfile.lock CHANGED
@@ -1,122 +1,156 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- routemaster-drain (1.0.5)
4
+ routemaster-drain (1.1.0)
5
5
  faraday (>= 0.9.0)
6
6
  faraday_middleware
7
7
  hashie
8
- net-http-persistent
9
- rack
8
+ net-http-persistent (< 3)
9
+ rack (>= 1.6.2)
10
10
  redis-namespace
11
- resque
12
11
  thread
13
- wisper (>= 1.4.0)
12
+ wisper (~> 1.6.1)
14
13
 
15
14
  GEM
16
15
  remote: https://rubygems.org/
17
16
  specs:
18
- addressable (2.3.6)
19
- celluloid (0.16.0)
20
- timers (~> 4.0.0)
21
- coderay (1.1.0)
22
- crack (0.4.2)
17
+ addressable (2.5.0)
18
+ public_suffix (~> 2.0, >= 2.0.2)
19
+ byebug (9.0.6)
20
+ codeclimate-test-reporter (1.0.3)
21
+ simplecov
22
+ coderay (1.1.1)
23
+ concurrent-ruby (1.0.2)
24
+ connection_pool (2.2.1)
25
+ crack (0.4.3)
23
26
  safe_yaml (~> 1.0.0)
24
27
  diff-lcs (1.2.5)
25
- dotenv (0.11.1)
26
- dotenv-deployment (~> 0.0.2)
27
- dotenv-deployment (0.0.2)
28
- faraday (0.9.1)
28
+ docile (1.1.5)
29
+ dotenv (2.1.1)
30
+ faraday (0.10.0)
29
31
  multipart-post (>= 1.2, < 3)
30
- faraday_middleware (0.9.1)
31
- faraday (>= 0.7.4, < 0.10)
32
- ffi (1.9.3)
32
+ faraday_middleware (0.10.1)
33
+ faraday (>= 0.7.4, < 1.0)
34
+ ffi (1.9.14)
33
35
  formatador (0.2.5)
34
- guard (2.6.1)
36
+ guard (2.14.0)
35
37
  formatador (>= 0.2.4)
36
- listen (~> 2.7)
38
+ listen (>= 2.7, < 4.0)
37
39
  lumberjack (~> 1.0)
40
+ nenv (~> 0.1)
41
+ notiffany (~> 0.0)
38
42
  pry (>= 0.9.12)
43
+ shellany (~> 0.0)
39
44
  thor (>= 0.18.1)
40
- guard-rspec (4.3.1)
45
+ guard-compat (1.2.1)
46
+ guard-rspec (4.7.3)
41
47
  guard (~> 2.1)
42
- rspec (>= 2.14, < 4.0)
43
- hashie (3.4.0)
44
- hitimes (1.2.2)
45
- listen (2.7.9)
46
- celluloid (>= 0.15.2)
47
- rb-fsevent (>= 0.9.3)
48
- rb-inotify (>= 0.9)
49
- lumberjack (1.0.9)
48
+ guard-compat (~> 1.1)
49
+ rspec (>= 2.99.0, < 4.0)
50
+ hashdiff (0.3.1)
51
+ hashie (3.4.6)
52
+ json (2.0.2)
53
+ listen (3.1.5)
54
+ rb-fsevent (~> 0.9, >= 0.9.4)
55
+ rb-inotify (~> 0.9, >= 0.9.7)
56
+ ruby_dep (~> 1.2)
57
+ lumberjack (1.0.10)
50
58
  method_source (0.8.2)
51
59
  mono_logger (1.1.0)
52
- multi_json (1.11.1)
60
+ multi_json (1.12.1)
53
61
  multipart-post (2.0.0)
62
+ nenv (0.3.0)
54
63
  net-http-persistent (2.9.4)
55
- pry (0.10.1)
64
+ notiffany (0.1.1)
65
+ nenv (~> 0.1)
66
+ shellany (~> 0.0)
67
+ pry (0.10.4)
56
68
  coderay (~> 1.1.0)
57
69
  method_source (~> 0.8.1)
58
70
  slop (~> 3.4)
59
- pry-nav (0.2.4)
60
- pry (>= 0.9.10, < 0.11.0)
61
- psych (2.0.6)
62
- rack (1.6.2)
71
+ pry-byebug (3.4.1)
72
+ byebug (~> 9.0)
73
+ pry (~> 0.10)
74
+ psych (2.2.1)
75
+ public_suffix (2.0.4)
76
+ rack (1.6.5)
63
77
  rack-protection (1.5.3)
64
78
  rack
65
- rack-test (0.6.2)
79
+ rack-test (0.6.3)
66
80
  rack (>= 1.0)
67
- rake (10.3.2)
68
- rb-fsevent (0.9.4)
69
- rb-inotify (0.9.5)
81
+ rake (11.3.0)
82
+ rb-fsevent (0.9.8)
83
+ rb-inotify (0.9.7)
70
84
  ffi (>= 0.5.0)
71
- redis (3.2.1)
85
+ redis (3.3.2)
72
86
  redis-namespace (1.5.2)
73
87
  redis (~> 3.0, >= 3.0.4)
74
- resque (1.25.2)
88
+ resque (1.26.0)
75
89
  mono_logger (~> 1.0)
76
90
  multi_json (~> 1.0)
77
91
  redis-namespace (~> 1.3)
78
92
  sinatra (>= 0.9.2)
79
93
  vegas (~> 0.1.2)
80
- rspec (3.1.0)
81
- rspec-core (~> 3.1.0)
82
- rspec-expectations (~> 3.1.0)
83
- rspec-mocks (~> 3.1.0)
84
- rspec-core (3.1.4)
85
- rspec-support (~> 3.1.0)
86
- rspec-expectations (3.1.1)
94
+ rspec (3.5.0)
95
+ rspec-core (~> 3.5.0)
96
+ rspec-expectations (~> 3.5.0)
97
+ rspec-mocks (~> 3.5.0)
98
+ rspec-core (3.5.4)
99
+ rspec-support (~> 3.5.0)
100
+ rspec-expectations (3.5.0)
101
+ diff-lcs (>= 1.2.0, < 2.0)
102
+ rspec-support (~> 3.5.0)
103
+ rspec-mocks (3.5.0)
87
104
  diff-lcs (>= 1.2.0, < 2.0)
88
- rspec-support (~> 3.1.0)
89
- rspec-mocks (3.1.1)
90
- rspec-support (~> 3.1.0)
91
- rspec-support (3.1.0)
92
- safe_yaml (1.0.3)
93
- sinatra (1.4.6)
94
- rack (~> 1.4)
105
+ rspec-support (~> 3.5.0)
106
+ rspec-support (3.5.0)
107
+ ruby_dep (1.5.0)
108
+ safe_yaml (1.0.4)
109
+ shellany (0.0.1)
110
+ sidekiq (4.2.7)
111
+ concurrent-ruby (~> 1.0)
112
+ connection_pool (~> 2.2, >= 2.2.0)
113
+ rack-protection (>= 1.5.0)
114
+ redis (~> 3.2, >= 3.2.1)
115
+ simplecov (0.12.0)
116
+ docile (~> 1.1.0)
117
+ json (>= 1.8, < 3)
118
+ simplecov-html (~> 0.10.0)
119
+ simplecov-html (0.10.0)
120
+ sinatra (1.4.7)
121
+ rack (~> 1.5)
95
122
  rack-protection (~> 1.4)
96
123
  tilt (>= 1.3, < 3)
97
124
  slop (3.6.0)
98
- thor (0.19.1)
99
- thread (0.1.4)
100
- tilt (2.0.1)
101
- timers (4.0.1)
102
- hitimes
125
+ thor (0.19.4)
126
+ thread (0.2.2)
127
+ tilt (2.0.5)
103
128
  vegas (0.1.11)
104
129
  rack (>= 1.0.0)
105
- webmock (1.18.0)
130
+ webmock (2.3.1)
106
131
  addressable (>= 2.3.6)
107
132
  crack (>= 0.3.2)
108
- wisper (1.6.0)
133
+ hashdiff
134
+ wisper (1.6.1)
109
135
 
110
136
  PLATFORMS
111
137
  ruby
112
138
 
113
139
  DEPENDENCIES
114
140
  bundler
141
+ codeclimate-test-reporter
115
142
  dotenv
116
143
  guard-rspec
117
- pry-nav
144
+ pry-byebug
118
145
  psych
119
146
  rack-test
120
147
  rake
148
+ resque
121
149
  routemaster-drain!
150
+ rspec
151
+ sidekiq
152
+ simplecov
122
153
  webmock
154
+
155
+ BUNDLED WITH
156
+ 1.13.6
data/LICENSE.txt CHANGED
@@ -1,4 +1,5 @@
1
- Copyright (c) 2014 HouseTrip Ltd.
1
+ Copyright (c) 2014-2016 HouseTrip Ltd.
2
+ Copyright (c) 2016 Deliveroo Ltd.
2
3
 
3
4
  MIT License
4
5
 
data/README.md CHANGED
@@ -1,13 +1,15 @@
1
1
  # routemaster-drain
2
2
 
3
3
  A Rack-based event receiver for the
4
- [Routemaster](https://github.com/HouseTrip/routemaster) event bus.
4
+ [Routemaster](https://github.com/mezis/routemaster) event bus.
5
5
 
6
6
  [![Version](https://badge.fury.io/rb/routemaster-drain.svg)](https://rubygems.org/gems/routemaster-drain)
7
7
  &nbsp;
8
- [![Build](https://travis-ci.org/HouseTrip/routemaster-drain.svg?branch=master)](https://travis-ci.org/HouseTrip/routemaster-drain)
8
+ [![Build](https://travis-ci.org/mezis/routemaster-drain.svg?branch=master)](https://travis-ci.org/mezis/routemaster-drain)
9
9
  &nbsp;
10
- [![Docs](http://img.shields.io/badge/API%20docs-rubydoc.info-blue.svg)](http://rubydoc.info/github/HouseTrip/routemaster-drain/frames/file/README.md)
10
+ [![Code Climate](https://codeclimate.com/github/deliveroo/routemaster-drain/badges/gpa.svg)](https://codeclimate.com/github/deliveroo/routemaster-drain)
11
+ [![Test Coverage](https://codeclimate.com/github/deliveroo/routemaster-drain/badges/coverage.svg)](https://codeclimate.com/github/deliveroo/routemaster-drain/coverage)
12
+ [![Docs](http://img.shields.io/badge/API%20docs-rubydoc.info-blue.svg)](http://rubydoc.info/github/mezis/routemaster-drain/frames/file/README.md)
11
13
 
12
14
  `routemaster-drain` is a collection of Rack middleware to receive and
13
15
  parse Routemaster events, filter them, and preemptively cache the corresponding
@@ -211,7 +213,7 @@ Note that `Cache#fget` is a future, so you can efficiently query many resources
211
213
  and have any `HTTP GET` requests (and cache queries) happen in parallel.
212
214
 
213
215
  See
214
- [rubydoc](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Cache)
216
+ [rubydoc](http://rubydoc.info/github/mezis/routemaster-drain/Routemaster/Cache)
215
217
  for more details on `Cache`.
216
218
 
217
219
 
@@ -219,9 +221,9 @@ for more details on `Cache`.
219
221
 
220
222
  The more elaborate drains are built with two components which can also be used
221
223
  independently,
222
- [`Dirty::Map`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Map)
224
+ [`Dirty::Map`](http://rubydoc.info/github/mezis/routemaster-drain/Routemaster/Dirty/Map)
223
225
  and
224
- [`Dirty::Filter`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Filter).
226
+ [`Dirty::Filter`](http://rubydoc.info/github/mezis/routemaster-drain/Routemaster/Dirty/Filter).
225
227
 
226
228
  ### Dirty map
227
229
 
@@ -233,22 +235,22 @@ A dirty map map gets _marked_ when an event about en entity gets processed that
233
235
  indicates a state change, and _swept_ to process those changes.
234
236
 
235
237
  Practically, instances of
236
- [`Routemaster::Dirty::Map`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Map)
238
+ [`Routemaster::Dirty::Map`](http://rubydoc.info/github/mezis/routemaster-drain/Routemaster/Dirty/Map)
237
239
  will emit a `:dirty_entity` event when a URL is marked as dirty, and can be
238
240
  swept when an entity is "cleaned". If a URL is marked multiple times before
239
241
  being swept (e.g. for very volatile entities), the event will only by broadcast
240
242
  once.
241
243
 
242
244
  To sweep the map, you can for instance listen to this event and call
243
- [`#sweep_one`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Map#sweep_one-instance_method).
245
+ [`#sweep_one`](http://rubydoc.info/github/mezis/routemaster-drain/Routemaster/Dirty/Map#sweep_one-instance_method).
244
246
 
245
247
  If you're not in a hurry and would rather run through batches you can call
246
- [`#sweep`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Map#sweep-instance_method)
248
+ [`#sweep`](http://rubydoc.info/github/mezis/routemaster-drain/Routemaster/Dirty/Map#sweep-instance_method)
247
249
  which will yield URLs until it runs out of dirty resources.
248
250
 
249
251
  ### Filter
250
252
 
251
- [`Routemaster::Dirty::Filter`](http://rubydoc.info/github/HouseTrip/routemaster-drain/Routemaster/Dirty/Filter) is a simple event filter
253
+ [`Routemaster::Dirty::Filter`](http://rubydoc.info/github/mezis/routemaster-drain/Routemaster/Dirty/Filter) is a simple event filter
252
254
  that performs reordering. It ignores events older than the latest known
253
255
  information on an entity.
254
256
 
@@ -259,7 +261,7 @@ as in `Receiver::Filter` for instance.
259
261
 
260
262
  ## Contributing
261
263
 
262
- 1. Fork it ( http://github.com/HouseTrip/routemaster-drain/fork )
264
+ 1. Fork it ( http://github.com/mezis/routemaster-drain/fork )
263
265
  2. Create your feature branch (`git checkout -b my-new-feature`)
264
266
  3. Commit your changes (`git commit -am 'Add some feature'`)
265
267
  4. Push to the branch (`git push origin my-new-feature`)
data/lib/routemaster/cache.rb CHANGED
@@ -38,7 +38,7 @@ module Routemaster
38
38
  # Wraps a future response, so it quacks exactly like an ordinary response.
39
39
  class FutureResponse
40
40
  extend Forwardable
41
-
41
+
42
42
  # The `block` is expected to return a {Response}
43
43
  def initialize(&block)
44
44
  @future = Pool.instance.future(&block)
@@ -47,41 +47,33 @@ module Routemaster
47
47
  # @!attribute status
48
48
  # @return [Integer]
49
49
  # Delegated to the `block`'s return value.
50
-
50
+
51
51
  # @!attribute headers
52
52
  # @return [Hash]
53
53
  # Delegated to the `block`'s return value.
54
-
54
+
55
55
  # @!attribute body
56
- # @return pHashie::Mash]
56
+ # @return [Hashie::Mash]
57
57
  # Delegated to the `block`'s return value.
58
-
58
+
59
59
  delegate :value => :@future
60
60
  delegate %i(status headers body) => :value
61
61
  end
62
62
 
63
- class Response < Hashie::Mash
64
- # @!attribute status
65
- # Integer
66
-
67
- # @!attribute headers
68
- # Hash
69
-
70
- # @!attribute body
71
- # Hashie::Mash
72
- end
73
-
74
-
75
63
  def initialize(redis:nil, fetcher:nil)
76
64
  @redis = redis || Config.cache_redis
77
- @expiry = Config.cache_expiry
78
- @fetcher = fetcher || Fetcher
65
+ @fetcher = fetcher || Fetcher.new(listener: self)
79
66
  end
80
67
 
81
68
  # Bust the cache for a given URL
82
69
  def bust(url)
83
70
  @redis.del("cache:#{url}")
84
- publish(:cache_bust, url)
71
+ _publish(:cache_bust, url)
72
+ end
73
+
74
+ # This is because wisper makes broadcasting methods private
75
+ def _publish(event, url)
76
+ publish(event, url)
85
77
  end
86
78
 
87
79
  # Get the response from a URL, from the cache if possible.
@@ -94,31 +86,14 @@ module Routemaster
94
86
  # header.
95
87
  #
96
88
  # @return [Response], which responds to `status`, `headers`, and `body`.
97
- def get(url, version:nil, locale:nil)
98
- key = "cache:#{url}"
99
- field = "v:#{version},l:#{locale}"
100
-
101
- # check cache
102
- if payload = @redis.hget(key, field)
103
- publish(:cache_hit, url)
104
- return Response.new(JSON.parse(payload))
105
- end
106
-
107
- # fetch data
89
+ def get(url, version: nil, locale: nil)
108
90
  headers = {
109
91
  'Accept' => version ?
110
92
  "application/json;v=#{version}" :
111
93
  "application/json"
112
94
  }
113
95
  headers['Accept-Language'] = locale if locale
114
- response = @fetcher.get(url, headers: headers)
115
-
116
- # store in redis
117
- @redis.hset(key, field, response.to_json)
118
- @redis.expire(key, @expiry)
119
-
120
- publish(:cache_miss, url)
121
- Response.new(response)
96
+ @fetcher.get(url, headers: headers)
122
97
  end
123
98
 
124
99
  # Like {#get}, but schedules any request in the background using a thread
@@ -126,8 +101,8 @@ module Routemaster
126
101
  #
127
102
  # @return [FutureResponse], which responds to `status`, `headers`, and `body`
128
103
  # like [Response].
129
- def fget(*args)
130
- FutureResponse.new { get(*args) }
104
+ def fget(url, version: nil, locale: nil)
105
+ FutureResponse.new { get(url, version: version, locale: locale) }
131
106
  end
132
107
  end
133
108
  end