routemaster-drain 3.0.1 → 3.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 193d7dd526f02d0397233f0c2868e74aedb41a23
-  data.tar.gz: 9a3c37a909a17321b305cd9c5d0bf916b71b722b
+  metadata.gz: 2d3a7e45eaf612d87b49876ad21714abaf7df4ca
+  data.tar.gz: 86494043fed1e03fe0b99c3377b454defc814621
 SHA512:
-  metadata.gz: d9dfc1c8da243ee0e56a30f77f5af28ddc374200c94de8341b77cf5f69a4a8a892426b310b59c0b5dfa8fe2763aa2bb59435e229201319a6dff94319d4de7737
-  data.tar.gz: a09d4315e6d6b9d0993aa8c35c93a5a5e2790339180159bc5346b497dc64b38b9617abe3ab8e0d2d4d771c0a4cd96e127df3ae961feb2140e35cd22a88067b13
+  metadata.gz: 18fae881aefa55f9a9b7807d07b7d9b8c1db5438cbc4b60d912de6925f874837266e5ab1e732f5ca9b769f21a81e7a6efc58dacf1f3ee0826ec0b22bb314e569
+  data.tar.gz: d86089d7ac3130d8dae61c623a30e65a374f434d95360e6c77fc13d44160f6bc0e36784437e7eed8eb7d23c55f5fbbb4fa26ed776c07ffa73f2e93399458ba3c

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+### 3.0.2 (2017-09-05)
+
+Bug fixes:
+
+- Fixes a condition where cache keys in Redis would not expire (#63)
+
+
 ### 3.0.1 (2017-08-08)
 
 Bug fixes:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    routemaster-drain (3.0.1)
+    routemaster-drain (3.0.2)
       addressable
       concurrent-ruby
       faraday (>= 0.9.0)
@@ -37,7 +37,7 @@ GEM
     dotenv (2.1.1)
     ethon (0.10.1)
       ffi (>= 1.3.0)
-    faraday (0.12.1)
+    faraday (0.12.2)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.12.2)
       faraday (>= 0.7.4, < 1.0)
@@ -133,7 +133,7 @@ GEM
     slop (3.6.0)
     thor (0.19.4)
     tilt (2.0.5)
-    typhoeus (1.1.2)
+    typhoeus (1.3.0)
       ethon (>= 0.9.0)
     url (0.3.2)
     vegas (0.1.11)
@@ -168,4 +168,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   1.15.3
+   1.15.4

data/README.md

@@ -17,6 +17,7 @@ combining middleware.
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 
   - [Installation](#installation)
+  - [Upgrading](#upgrading)
   - [Illustrated use cases](#illustrated-use-cases)
     - [Simply receive events from Routemaster](#simply-receive-events-from-routemaster)
     - [Receive change notifications without duplicates](#receive-change-notifications-without-duplicates)
@@ -77,6 +78,24 @@ Routemaster::RedisBroker.instance.inject(
 )
 ```
 
+## Upgrading
+
+
+If upgrading from any version between 2.4.0 and 3.0.1, and are using caching,
+your cache may be corrupted by entries that lack a TTL (which will eventually
+cause your Redis
+storage to blow up).
+
+We provide a tool to fix your data. With your environment loaded and configured
+(e.g. from a Rails console), run:
+
+```ruby
+Routemaster::Tasks::FixCacheTTL.new.call
+```
+
+This will scan your cache and add TTLs where missing.
+
+
 ## Illustrated use cases
 
 

data/gemfiles/rails_3.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    routemaster-drain (3.0.1)
+    routemaster-drain (3.0.2)
       addressable
       concurrent-ruby
       faraday (>= 0.9.0)
@@ -67,7 +67,7 @@ GEM
     erubis (2.7.0)
     ethon (0.10.1)
       ffi (>= 1.3.0)
-    faraday (0.12.1)
+    faraday (0.12.2)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.12.2)
       faraday (>= 0.7.4, < 1.0)
@@ -200,7 +200,7 @@ GEM
     treetop (1.4.15)
       polyglot
       polyglot (>= 0.3.1)
-    typhoeus (1.1.2)
+    typhoeus (1.3.0)
       ethon (>= 0.9.0)
     tzinfo (0.3.53)
     url (0.3.2)
@@ -237,4 +237,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   1.15.3
+   1.15.4

data/gemfiles/rails_4.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    routemaster-drain (3.0.1)
+    routemaster-drain (3.0.2)
       addressable
       concurrent-ruby
       faraday (>= 0.9.0)
@@ -74,7 +74,7 @@ GEM
     erubis (2.7.0)
     ethon (0.10.1)
       ffi (>= 1.3.0)
-    faraday (0.12.1)
+    faraday (0.12.2)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.12.2)
       faraday (>= 0.7.4, < 1.0)
@@ -216,7 +216,7 @@ GEM
     thor (0.19.4)
     thread_safe (0.3.6)
     tilt (2.0.7)
-    typhoeus (1.1.2)
+    typhoeus (1.3.0)
       ethon (>= 0.9.0)
     tzinfo (1.2.3)
       thread_safe (~> 0.1)
@@ -254,4 +254,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   1.15.3
+   1.15.4

data/gemfiles/rails_5.gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: ..
   specs:
-    routemaster-drain (3.0.1)
+    routemaster-drain (3.0.2)
       addressable
       concurrent-ruby
       faraday (>= 0.9.0)
@@ -77,7 +77,7 @@ GEM
     erubis (2.7.0)
     ethon (0.10.1)
       ffi (>= 1.3.0)
-    faraday (0.12.1)
+    faraday (0.12.2)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.12.2)
       faraday (>= 0.7.4, < 1.0)
@@ -216,7 +216,7 @@ GEM
       sprockets (>= 3.0.0)
     thor (0.19.4)
     thread_safe (0.3.6)
-    typhoeus (1.1.2)
+    typhoeus (1.3.0)
       ethon (>= 0.9.0)
     tzinfo (1.2.3)
       thread_safe (~> 0.1)
@@ -257,4 +257,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   1.15.3
+   1.15.4

data/lib/routemaster/cache_key.rb

@@ -1,7 +1,9 @@
 module Routemaster
   class CacheKey
+    PREFIX = 'cache:'.freeze
+
     def self.url_key(url)
-      "cache:#{url}"
+      "#{PREFIX}#{url}"
     end
   end
 end

data/lib/routemaster/drain.rb

@@ -1,5 +1,5 @@
 module Routemaster
   module Drain
-    VERSION = '3.0.1'
+    VERSION = '3.0.2'.freeze
   end
 end

data/lib/routemaster/event_index.rb

@@ -1,21 +1,34 @@
 require 'routemaster/cache_key'
 module Routemaster
   class EventIndex
-    attr_reader :url, :cache
-
     def initialize(url, cache: Config.cache_redis)
       @url = url
       @cache = cache
     end
 
     def increment
-      i = cache.hincrby(CacheKey.url_key(url), 'current_index', 1).to_i
-      Config.logger.debug("DRAIN: Increment #{@url} to #{i}") if Config.logger.debug?
-      i
+      _node do |cache, key|
+        cache.multi do |m|
+          m.hincrby(key, 'current_index', 1)
+          m.expire(key, Config.cache_expiry)
+        end
+      end
+      self
     end
 
     def current
-      (cache.hget(CacheKey.url_key(url), 'current_index') || 0).to_i
+      (@cache.hget(_key, 'current_index') || 0).to_i
+    end
+
+    private
+
+    def _node
+      namespaced_key = "#{@cache.namespace}:#{_key}"
+      yield @cache.redis.node_for(namespaced_key), namespaced_key
+    end
+
+    def _key
+      @_key ||= CacheKey.url_key(@url)
     end
   end
 end

data/lib/routemaster/tasks/fix_cache_ttl.rb

@@ -0,0 +1,45 @@
+require 'routemaster/cache_key'
+
+module Routemaster
+  module Tasks
+    class FixCacheTTL
+      def initialize(cache: Config.cache_redis, batch_size: 100)
+        @cache = cache
+        @batch_size = batch_size
+      end
+
+      def call
+        pattern = "#{@cache.namespace}:#{CacheKey::PREFIX}*"
+        _each_key_batch(pattern) do |node, keys|
+          _fix_keys(node, keys)
+        end
+      end
+
+      private
+
+      def _each_key_batch(pattern)
+        @cache.redis.nodes.each do |node|
+          cursor = 0
+          loop do
+            cursor, keys = node.scan(cursor, count: @batch_size, match: pattern)
+            yield node, keys
+            break if cursor.to_i == 0
+          end
+        end
+      end
+
+      def _fix_keys(node, keys)
+        ttls = node.pipelined do |p|
+          keys.each { |k| p.ttl(k) }
+        end
+
+        node.pipelined do |p|
+          keys.zip(ttls).each do |k,ttl|
+            next unless ttl < 0
+            p.expire(k, Config.cache_expiry)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/routemaster/event_index_spec.rb

@@ -0,0 +1,29 @@
+require 'routemaster/event_index'
+require 'spec/support/uses_redis'
+
+describe Routemaster::EventIndex do
+  uses_redis
+
+  let(:cache) { Routemaster::Config.cache_redis }
+  let(:url) { 'https://example.com/widgets/1234' }
+  subject { described_class.new(url, cache: cache) }
+
+  describe '#increment' do
+    it 'increases #current' do
+      expect {
+        subject.increment
+      }.to change {
+        subject.current
+      }.from(0).to(1)
+    end
+
+    it 'leaves all keys with TTLs' do
+      subject.increment
+      cache.redis.nodes.each do |node|
+        node.keys.each do |key|
+          expect(node.ttl(key)).to be > 0
+        end
+      end
+    end
+  end
+end

data/spec/routemaster/tasks/fix_cache_ttl_spec.rb

@@ -0,0 +1,41 @@
+require 'routemaster/tasks/fix_cache_ttl'
+require 'spec/support/uses_redis'
+
+describe Routemaster::Tasks::FixCacheTTL do
+  uses_redis
+
+  let(:cache) { Routemaster::Config.cache_redis }
+  subject { described_class.new(cache: cache, batch_size: 5) }
+
+  before do
+    # add keys without a TTL, that aren't cache keys
+    @ignored_keys = (1..20).map { |n| "fubar:#{n}".tap { |k| cache.set(k, n) } }
+
+    # add cache keys with a pre-existing TTL
+    @good_keys = (1..20).map { |n| "cache:good-#{n}".tap { |k| cache.set(k, n, ex: n + 1_000) } }
+    
+    # add cache keys without a TTL
+    @bad_keys = (1..20).map { |n| "cache:bad-#{n}".tap { |k| cache.set("cache:bad-#{n}", n) } }
+  end
+
+  it 'leaves non-cache keys alone' do
+    subject.call
+    @ignored_keys.each { |k|
+      expect(cache.ttl(k)).to eq -1
+    }
+  end
+
+  it 'adds a TTL to broken cache keys' do
+    subject.call
+    @bad_keys.each { |k|
+      expect(cache.ttl(k)).to be > 100_000
+    }
+  end
+
+  it 'leaves the TTL of good cache keys alone' do
+    subject.call
+    @good_keys.each { |k|
+      expect(0..2_000).to include cache.ttl(k)
+    }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: routemaster-drain
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.2
 platform: ruby
 authors:
 - Julien Letessier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-08 00:00:00.000000000 Z
+date: 2017-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -209,6 +209,7 @@ files:
 - lib/routemaster/responses/hateoas_enumerable_response.rb
 - lib/routemaster/responses/hateoas_response.rb
 - lib/routemaster/responses/response_promise.rb
+- lib/routemaster/tasks/fix_cache_ttl.rb
 - routemaster-drain.gemspec
 - spec/routemaster/api_client_spec.rb
 - spec/routemaster/cache_spec.rb
@@ -221,6 +222,7 @@ files:
 - spec/routemaster/drain/caching_spec.rb
 - spec/routemaster/drain/mapping_spec.rb
 - spec/routemaster/drain/terminator_spec.rb
+- spec/routemaster/event_index_spec.rb
 - spec/routemaster/integration/api_client_spec.rb
 - spec/routemaster/integration/cache_spec.rb
 - spec/routemaster/jobs/backends/backend_examples.rb
@@ -240,6 +242,7 @@ files:
 - spec/routemaster/responses/hateoas_enumerable_response_spec.rb
 - spec/routemaster/responses/hateoas_response_spec.rb
 - spec/routemaster/responses/response_promise_spec.rb
+- spec/routemaster/tasks/fix_cache_ttl_spec.rb
 - spec/spec_helper.rb
 - spec/support/breakpoint_class.rb
 - spec/support/events.rb
@@ -269,7 +272,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Event receiver for the Routemaster bus
@@ -285,6 +288,7 @@ test_files:
 - spec/routemaster/drain/caching_spec.rb
 - spec/routemaster/drain/mapping_spec.rb
 - spec/routemaster/drain/terminator_spec.rb
+- spec/routemaster/event_index_spec.rb
 - spec/routemaster/integration/api_client_spec.rb
 - spec/routemaster/integration/cache_spec.rb
 - spec/routemaster/jobs/backends/backend_examples.rb
@@ -304,6 +308,7 @@ test_files:
 - spec/routemaster/responses/hateoas_enumerable_response_spec.rb
 - spec/routemaster/responses/hateoas_response_spec.rb
 - spec/routemaster/responses/response_promise_spec.rb
+- spec/routemaster/tasks/fix_cache_ttl_spec.rb
 - spec/spec_helper.rb
 - spec/support/breakpoint_class.rb
 - spec/support/events.rb