derrick 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3337d0adef7d3a74367fd32cd6432e10746d9b50
-  data.tar.gz: 23b953e5d08de9fefcc73ac06217d64a39253af1
+  metadata.gz: 85d156880226b9165b0f32522de5eaf60e1a63ac
+  data.tar.gz: f5d4ea05933b71e2cabfb72a55683bc81d79b063
 SHA512:
-  metadata.gz: 9ffb3ec31001802893c3de556751ecbc086387957c5028895281541b127948c1b528a45c6c6c003494a03ed3500a4d847bcde49dc81e90babf75cfd18f67e04c
-  data.tar.gz: b61b22c42e72d3b7d3dd83daeb8eed86c552d5b5628bc71131a446b53e26f5f314443ebcb3e14377fb9b03dea9d7c3672e6059fe77d873e0ceee856fbb9b6433
+  metadata.gz: 8f6d15213b7598333fb606230287f9ca9cd1f2276b04e5a5da1a4ba020dc9622aac58caeab1b8950f19d30f62a3717fd6e57486b6d659a68398ad693eab3e809
+  data.tar.gz: 72b95d56cb1b4b52f6ab680d45c76016570f9b9b44008dd0506a37ed2ac215686aa0dce61b9c6b0286235fbcf558cec4cb360870dc50635c510b759851e5fcbf

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--warnings
+--require spec_helper

data/derrick.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', '~> 1.6'
   spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
 
   spec.add_dependency 'redis'
 end

data/lib/derrick/aggregator.rb

@@ -0,0 +1,51 @@
+module Derrick
+  class Aggregator
+    ANY = '*'.freeze
+
+    attr_reader :patterns
+    def initialize(queue, context)
+      @queue = queue
+      @patterns = {}
+      @context = context
+    end
+
+    def run
+      fetcher_count = @context.concurrency
+      loop do
+        keys = @queue.pop
+        if keys == :stop
+          fetcher_count -= 1
+          break if fetcher_count == 0
+        else
+          keys.each { |k| aggregate(k) }
+        end
+      end
+      self
+    end
+
+    def aggregate(key)
+      pattern = pattern_from(key)
+      pattern.aggregate(key)
+
+      if patterns.size > @context.max_patterns
+        compact_uniques!
+      end
+    end
+
+    def pattern_from(key)
+      @patterns[Pattern.extract(key.name)] ||= Pattern.new
+    end
+
+    def compact_uniques!
+      any = @patterns.delete(ANY) || Pattern.new
+      @patterns.each do |key, aggregate|
+        if aggregate.count == 1
+          any.merge!(@patterns.delete(key))
+        end
+      end
+      @patterns[ANY] = any
+      nil
+    end
+
+  end
+end

data/lib/derrick/cli.rb

@@ -41,11 +41,12 @@ module Derrick
     end
 
     class Context
-      attr_accessor :concurrency, :batch_size
+      attr_accessor :concurrency, :batch_size, :max_patterns
 
       def initialize
         @concurrency = 2
         @batch_size = 10_000
+        @max_patterns = 1_000
       end
     end
 
@@ -140,6 +141,9 @@ module Derrick
         opts.on('-b', '--batch-size BATCH_SIZE') do |batch_size|
           @context.batch_size = Integer(batch_size)
         end
+        opts.on('-C', '--max-patterns MAX_PATTERNS') do |max_patterns|
+          @context.max_patterns = Integer(max_patterns)
+        end
       end
     end
 

data/lib/derrick/collector.rb

@@ -0,0 +1,25 @@
+module Derrick
+  class Collector
+    def initialize(redis, queue, progress, context)
+      @redis = redis
+      @queue = queue
+      @progress = progress
+      @context = context
+    end
+
+    def run
+      collect_keys
+      @context.concurrency.times { @queue.push(:stop) }
+    end
+
+    def collect_keys
+      cursor = '0'
+      loop do
+        cursor, keys = @redis.scan(cursor, count: @context.batch_size)
+        @queue.push(keys)
+        @progress.increment_collected(keys.size)
+        return if cursor == '0'
+      end
+    end
+  end
+end

data/lib/derrick/fetcher.rb

@@ -0,0 +1,39 @@
+module Derrick
+  Key = Struct.new(:name, :type, :ttl)
+
+  class Fetcher
+    def initialize(redis, input, output, progress)
+      @redis = redis
+      @input = input
+      @output = output
+      @progress = progress
+    end
+
+    def run
+      while (keys = @input.pop) != :stop
+        @output.push(stats(keys))
+      end
+      @output.push(:stop)
+    end
+
+    def stats(keys)
+      types = @redis.pipelined do
+        keys.each do |key|
+          @redis.type(key)
+        end
+      end
+
+      ttls = @redis.pipelined do
+        keys.each do |key|
+          @redis.ttl(key)
+        end
+      end
+
+      @progress.increment_fetched(keys.size)
+
+      keys.map.with_index do |key, index|
+        Key.new(key, types[index], ttls[index])
+      end
+    end
+  end
+end

data/lib/derrick/inspector.rb

@@ -3,161 +3,6 @@ require 'thread'
 Thread.abort_on_exception = true
 
 module Derrick
-  class Collector
-    def initialize(redis, queue, progress, context)
-      @redis = redis
-      @queue = queue
-      @progress = progress
-      @context = context
-    end
-
-    def run
-      collect_keys
-      @context.concurrency.times { @queue.push(:stop) }
-    end
-
-    def collect_keys
-      cursor = '0'
-      loop do
-        cursor, keys = @redis.scan(cursor, count: @context.batch_size)
-        @queue.push(keys)
-        @progress.increment_collected(keys.size)
-        return if cursor == '0'
-      end
-    end
-  end
-
-  Key = Struct.new(:name, :type, :ttl)
-
-  class Fetcher
-
-    def initialize(redis, input, output, progress)
-      @redis = redis
-      @input = input
-      @output = output
-      @progress = progress
-    end
-
-    def run
-      while (keys = @input.pop) != :stop
-        @output.push(stats(keys))
-      end
-      @output.push(:stop)
-    end
-
-    def stats(keys)
-      types = @redis.pipelined do
-        keys.each do |key|
-          @redis.type(key)
-        end
-      end
-
-      ttls = @redis.pipelined do
-        keys.each do |key|
-          @redis.ttl(key)
-        end
-      end
-
-      @progress.increment_fetched(keys.size)
-
-      keys.map.with_index do |key, index|
-        Key.new(key, types[index], ttls[index])
-      end
-    end
-
-  end
-
-  class Pattern
-    attr_reader :pattern, :count, :expirable_count, :persisted_count, :types_count
-
-    def initialize
-      @count = 0
-      @expirable_count = 0
-      @persisted_count = 0
-      @types_count = Hash.new(0)
-    end
-
-    def expirable_ratio
-      return 1 if count == 0
-      expirable_count.to_f / count
-    end
-
-    def types_ratio
-      Hash[@types_count.map do |type, sub_count|
-        [type, sub_count.to_f / count]
-      end]
-    end
-
-    def aggregate(key)
-      @count += 1
-
-      if key.ttl == -1
-        @persisted_count += 1
-      else
-        @expirable_count += 1
-      end
-
-      @types_count[key.type] += 1
-    end
-
-  end
-
-  class Progress
-
-    attr_reader :total, :collected, :fetched
-
-    def initialize(total)
-      @total = total
-      @mutex = Mutex.new
-
-      @collected = 0
-      @fetched = 0
-    end
-
-    def increment_collected(count)
-      @mutex.synchronize { @collected += count }
-    end
-
-    def increment_fetched(count)
-      @mutex.synchronize { @fetched += count }
-    end
-
-  end
-
-  class Aggregator
-    attr_reader :patterns
-    def initialize(queue, context)
-      @queue = queue
-      @patterns = {}
-      @context = context
-    end
-
-    def run
-      fetcher_count = @context.concurrency
-      loop do
-        keys = @queue.pop
-        if keys == :stop
-          fetcher_count -= 1
-          break if fetcher_count == 0
-        else
-          keys.each { |k| aggregate(k) }
-        end
-      end
-      self
-    end
-
-    def aggregate(key)
-      pattern = pattern_from(key)
-      pattern.aggregate(key)
-    end
-
-    def pattern_from(key)
-      canonical_key = key.name.inspect.gsub(/(^|:)(\d+|[0-9a-f]{32,40})($|:)/, '\1*\3')
-      @patterns[canonical_key] ||= Pattern.new
-    end
-
-  end
-
   class Inspector
     attr_reader :redis, :progress
 

data/lib/derrick/pattern.rb

@@ -0,0 +1,62 @@
+module Derrick
+  class Pattern
+    SEGMENT_SEPARATORS = %w(: _ /).freeze
+    FIRST_SEGMENT_PATTERN = /.*(#{SEGMENT_SEPARATORS.map { |s| Regexp.escape(s) }.join('|')})/
+    IDENTIFIER_PATTERNS = '\d+|[0-9a-f]{32,40}'
+    SEGMENT_PATTERNS = SEGMENT_SEPARATORS.map do |separator|
+      /(^|#{Regexp.escape(separator)})(#{IDENTIFIER_PATTERNS})($|#{Regexp.escape(separator)})/
+    end
+
+    attr_reader :pattern, :count, :expirable_count, :persisted_count, :types_count
+
+    def self.extract(key_name)
+      key_pattern = SEGMENT_PATTERNS.inject(key_name.inspect[1..-2]) do |key, pattern|
+        key.gsub(pattern, '\1*\3')
+      end
+
+      return "#{key_name[FIRST_SEGMENT_PATTERN]}*" if key_pattern == key_name
+
+      key_pattern
+    end
+
+    def initialize
+      @count = 0
+      @expirable_count = 0
+      @persisted_count = 0
+      @types_count = Hash.new(0)
+    end
+
+    def merge!(other)
+      @count += other.count
+      @expirable_count += other.expirable_count
+      @persisted_count += other.persisted_count
+      other.types_count.each do |type, count|
+        @types_count[type] += count
+      end
+      self
+    end
+
+    def expirable_ratio
+      return 1 if count == 0
+      expirable_count.to_f / count
+    end
+
+    def types_ratio
+      Hash[@types_count.map do |type, sub_count|
+        [type, sub_count.to_f / count]
+      end]
+    end
+
+    def aggregate(key)
+      @count += 1
+
+      if key.ttl == -1
+        @persisted_count += 1
+      else
+        @expirable_count += 1
+      end
+
+      @types_count[key.type] += 1
+    end
+  end
+end

data/lib/derrick/progress.rb

@@ -0,0 +1,21 @@
+module Derrick
+  class Progress
+    attr_reader :total, :collected, :fetched
+
+    def initialize(total)
+      @total = total
+      @mutex = Mutex.new
+
+      @collected = 0
+      @fetched = 0
+    end
+
+    def increment_collected(count)
+      @mutex.synchronize { @collected += count }
+    end
+
+    def increment_fetched(count)
+      @mutex.synchronize { @fetched += count }
+    end
+  end
+end

data/lib/derrick/version.rb

@@ -1,3 +1,3 @@
 module Derrick
-  VERSION = "0.0.2"
+  VERSION = '0.1.0'
 end

data/lib/derrick.rb

@@ -1,5 +1,10 @@
 require 'derrick/version'
 require 'derrick/inspector'
+require 'derrick/aggregator'
+require 'derrick/collector'
+require 'derrick/fetcher'
+require 'derrick/pattern'
+require 'derrick/progress'
 
 module Derrick
 end

data/spec/pattern_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+
+describe Derrick::Pattern do
+
+  describe '.extract' do
+
+    it 'canonicalizes integers' do
+      expect('foo:42:bar').to match_pattern('foo:*:bar')
+    end
+
+    it 'canonicalizes MD5s' do
+      expect('foo:258622b1688250cb619f3c9ccaefb7eb:bar').to match_pattern('foo:*:bar')
+    end
+
+    it 'canonicalizes SHA1s' do
+      expect('foo:792eaaec6718c335d63cde10d7281baa4132ba78:bar').to match_pattern('foo:*:bar')
+    end
+
+    it 'escapes strings with invalid characters' do
+      expect("hi \xAD").to match_pattern('hi \xAD')
+    end
+
+    it 'accepts `_` as segment separator' do
+      expect('foo_42_bar').to match_pattern('foo_*_bar')
+    end
+
+    it 'accepts `/` as segment separator' do
+      expect('foo/42/bar').to match_pattern('foo/*/bar')
+    end
+
+    context 'when no identifier is found' do
+
+      it 'assumes the first segment is shared' do
+        expect('foo:bar').to match_pattern('foo:*')
+      end
+
+      context 'and no segment serparator either' do
+
+        it 'falls in the `*` pattern' do
+          expect('foobar').to match_pattern('*')
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,86 @@
+lib = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'derrick'
+
+
+Dir[File.join(File.expand_path('../support', __FILE__), '**/*.rb')].each { |f| require f }
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+=end
+end

data/spec/support/pattern_matcher.rb

@@ -0,0 +1,9 @@
+RSpec::Matchers.define :match_pattern do |expected_pattern|
+  match do |actual|
+    Derrick::Pattern.extract(actual) == expected_pattern
+  end
+
+  failure_message do |actual|
+    "expected `#{actual}` to be canonicalized as `#{expected_pattern}` but was `#{Derrick::Pattern.extract(actual)}`"
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: derrick
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-17 00:00:00.000000000 Z
+date: 2014-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
@@ -61,6 +75,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -68,9 +83,17 @@ files:
 - bin/derrick
 - derrick.gemspec
 - lib/derrick.rb
+- lib/derrick/aggregator.rb
 - lib/derrick/cli.rb
+- lib/derrick/collector.rb
+- lib/derrick/fetcher.rb
 - lib/derrick/inspector.rb
+- lib/derrick/pattern.rb
+- lib/derrick/progress.rb
 - lib/derrick/version.rb
+- spec/pattern_spec.rb
+- spec/spec_helper.rb
+- spec/support/pattern_matcher.rb
 homepage: ''
 licenses:
 - MIT
@@ -95,4 +118,7 @@ rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Inspect Redis databases and print statistics about the keys
-test_files: []
+test_files:
+- spec/pattern_spec.rb
+- spec/spec_helper.rb
+- spec/support/pattern_matcher.rb