redis_scanner 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a0480eb1d36893c2716f1e22908890f0c8f9b21
-  data.tar.gz: e700af17696a802085fcc0ef0f5abe775d7f4f11
+  metadata.gz: f4d4379646a6674439ca7ad275828c5881c97901
+  data.tar.gz: 36e42b13cab65c468e71dded6a2edfafeaba3671
 SHA512:
-  metadata.gz: 17992d3625d7aa84c4c6f632249d1d9fc0a669531112d9b43b386b3093908ce7febb5430ee438ac34fd5c15950408340ef2547578c8689856fcfd1f3302c5e4d
-  data.tar.gz: 6bca4cbc90c4716e0858f24cc6dacd60887f45142430bbd6343e60e72f6d599146b053bed86d60cb375c8d008343bdfd8d546f3c86d0cc1ecaf9ebd1117776ea
+  metadata.gz: 1d4180f672190a047a5a39269b1d726ad8b26c3dfe9d640d8839006c0be2050d96a185bbbaf16c883f62cc05afa6387f00ba62de2c8c325aab542431c8cbc21e
+  data.tar.gz: 83599028ff33ec24a5f573a5332aedb1db25a03c9a7e3daebd364210cffc34e52e6d39af65052a7ef0aaecbc5cf90ff0df278b8d5ad27b1acc5d5fb4a0193fd6

data/CHANGELOG.md

@@ -1,12 +1,19 @@
+## v0.1.3
+
+ * Provide key's detail information which includes type and size.
+ * Provide table format output. And you can use simpe output also.
+ * Add progress bar.
+ * Support limit, format and detail options.
+ * Refactor codes.Add more auto test cases.
 
 ## v0.1.2
 
- * Support date pattern
- * Add auto test cases
+ * Support date pattern.
+ * Add auto test cases.
 
 ## v0.1.1
 
- * Add redis_scanner to executable file
+ * Add redis_scanner to executable file.
 
 ## v0.1.0
 

data/README.md

@@ -4,6 +4,8 @@ RedisScanner is a tiny tool for scanning all redis keys and creating statistic r
 
 * Scans keys using *scan* replacing *keys* command.
 * Creates statistic result by key's pattern.
+* Provide key's detail information which includes type and size.
+* Provide table format output..
 
 ## Installation
 
@@ -30,19 +32,40 @@ redis_scanner
 The Output is like this:
 
 ```shell
-=========result is=========
-demo:user:<id>:counter 10000
-u:<uuid>:pf 52
-sidekiq_demo:stat:failed:<date> 4
-sidekiq_demo:stat:processed:<date> 4
-_sp_one:queue:default 1
-bh:queues 1
-bh:retry 1
-bh:stat:failed 1
-bh:stat:processed 1
-bus_app:app_two 1
++------------------------------------+-------+
+| Key                                | Count |
++------------------------------------+-------+
+| demo:user:<id>:counter             | 10000 |
+| u:<uuid>:pf                        |    52 |
+| sidekiq_demo:stat:failed:<date>    |     4 |
+| sidekiq_demo:stat:processed:<date> |     4 |
+| _sp_one:queue:default              |     1 |
+| bh:queues                          |     1 |
 ...
-===========================
++------------------------------------+-------+
+```
+
+```shell
+redis_scanner -d
+```
+
+The Output is like this:
+
+```shell
++------------------------------------+-------+--------+---------+
+| Key                                | Count | Size   | AvgSize |
++------------------------------------+-------+--------+---------+
+| demo:user:<id>:counter             | 10000 |        |         |
+|  > string                          | 10000 | 927510 |   92.75 |
+| u:<uuid>:pf                        |    52 |        |         |
+|  > hash                            |    52 |    108 |    2.08 |
+| sidekiq_demo:stat:failed:<date>    |     4 |        |         |
+|  > string                          |     4 |      5 |    1.25 |
+| sidekiq_demo:stat:processed:<date> |     4 |        |         |
+|  > string                          |     4 |      6 |     1.5 |
+| _sp_one:queue:default              |     1 |        |         |
+|  > list                            |     1 |      1 |     1.0 |
++------------------------------------+-------+--------+---------+
 ```
 
 * Scan keys with some pattern

data/bin/redis_scanner

@@ -9,14 +9,28 @@ options = {}
 OptionParser.new do |opts|
   opts.banner = "Usage: redis_scanner [options]"
 
-  opts.on("-f FILE", "--file FILE", "Output file") do |v|
+  opts.on("-f FILE", "--file FILE", "Output to file") do |v|
     options[:file] = v
   end
 
-  opts.on("-m MATCH", "--match MATCH", "Scan pattern") do |v|
+  opts.on("-m MATCH", "--match MATCH", "Only scan the pattern") do |v|
     options[:match] = v
   end
 
+  opts.on("-l LIMIT", "--limit LIMIT", "Only show top <limit> keys") do |v|
+    options[:limit] = v.to_i
+  end
+
+  options[:detail] = false
+  opts.on("-d", "--detail", "Show detail info(type & size)") do |v|
+    options[:detail] = v
+  end
+
+  options[:format] = 'table'
+  opts.on("-t FORMAT", "--format FORMAT", "Formt(simple or talbe. default is table)") do |v|
+    options[:format] = v
+  end
+
   # redis client options
   # -h <hostname>      Server hostname (default: 127.0.0.1).
   # -p <port>          Server port (default: 6379).

data/lib/redis_scanner/engine.rb

@@ -1,29 +1,37 @@
+require "ruby-progressbar"
+
 module RedisScanner
   class Engine
     def initialize(redis, options)
       @redis = redis
       @options = options
+      @rule = Rule.new
     end
 
     def run
-      convert_to_sorted_array scan
+      result = scan
+      result.values.sort
     end
 
     private
 
-    def convert_to_sorted_array(stat)
-      stat.to_a.sort do |x, y|
-        if x[1] == y[1]
-          x[0] <=> y[0]
-        else
-          y[1] <=> x[1]
-        end
-      end
+    def create_progress_bar
+      total = @redis.total_keys
+      bar = ProgressBar.create(
+        title: "Keys",
+        format: '%a %bᗧ%i %p%% %t',
+        progress_mark: ' ',
+        remainder_mark: '･',
+        total: total)
+      bar.log "total keys is #{total}"
+      bar
     end
 
     def scan
       cursor = 0
-      stat = Hash.new(0)
+      stat = Hash.new {|hash, key| hash[key] = Pattern.new(key) }
+
+      bar = create_progress_bar
       while true
         if @options[:match]
           cursor, result = @redis.scan cursor, match: @options[:match]
@@ -31,32 +39,22 @@ module RedisScanner
           cursor, result = @redis.scan cursor
         end
         result.each do |key|
-          pattern = resolve_pattern(key)
-          stat[pattern] += 1
+          pattern = @rule.extract_pattern(key)
+          if @options[:detail]
+            type, size = @redis.get_type_and_size(key)
+            stat[pattern].increment type, size
+          else
+            stat[pattern].increment
+          end
+          bar.increment
         end
         cursor = cursor.to_i
         break if cursor == 0
       end
+      bar.finish
+
       stat
     end
 
-    PATTERNS = [
-      [/(:\d+:)/, ":<id>:"],
-      [/(:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}:)/, ":<uuid>:"],
-      [/(:\d{4}-\d{2}-\d{2}:)/, ":<date>:"],
-      [/(:\d+)$/, ":<id>"],
-      [/(:\w{8}-\w{4}-\w{4}-\w{4}-\w{12})$/, ":<uuid>"],
-      [/(:\d{4}-\d{2}-\d{2})$/, ":<date>"]
-    ]
-
-    def resolve_pattern(key)
-      PATTERNS.each do |pattern, replacer|
-        if m = pattern.match(key)
-          key = key.sub(m[1], replacer)
-          break
-        end
-      end
-      key
-    end
   end
 end

data/lib/redis_scanner/formatter.rb

@@ -0,0 +1,71 @@
+require "terminal-table"
+
+module RedisScanner
+  class Formatter
+    def initialize(options)
+      @options = options
+    end
+
+    def format(patterns)
+      if @options[:format] == "simple"
+        simple patterns
+      else
+        table patterns
+      end
+    end
+
+    private
+
+    def touch_limit?(count)
+      @options[:limit] &&
+        @options[:limit].to_i > 0 &&
+        count >= @options[:limit].to_i
+    end
+
+    def simple(patterns)
+      ret = ""
+      count = 0
+      patterns.each do |pattern|
+        ret << pattern.to_s
+        ret << "\n"
+        count += 1
+        break if touch_limit?(count)
+      end
+      ret
+    end
+
+    def table(patterns)
+      with_detail = @options[:detail]
+      rows = []
+      count = 0
+
+      patterns.each do |pattern|
+        if with_detail
+          rows << [pattern.name, pattern.total, "", ""]
+          pattern.sorted_items.each do |item|
+            rows << [" > #{item.type}", item.count, item.size, item.avg_size]
+          end
+        else
+          rows << [pattern.name, pattern.total]
+        end
+        count += 1
+        break if touch_limit?(count)
+      end
+
+      headings = ['Key', 'Count']
+      if with_detail
+        headings << "Size"
+        headings << "AvgSize"
+      end
+
+      table = Terminal::Table.new headings: headings, rows: rows
+      table.align_column(1, :right)
+      if with_detail
+        table.align_column(2, :right)
+        table.align_column(3, :right)
+      end
+
+      table.to_s
+    end
+  end
+end

data/lib/redis_scanner/pattern.rb

@@ -0,0 +1,76 @@
+module RedisScanner
+  class PatternItem
+    attr_reader :type, :count, :size
+
+    def initialize(type)
+      @type = type
+      @count = 0
+      @size = 0
+    end
+
+    def increment(size)
+      @count += 1
+      @size += size
+    end
+
+    def <=>(other)
+      if @count == other.count
+        @type <=> other.type
+      else
+        other.count <=> @count
+      end
+    end
+
+    def avg_size
+      @count > 0 ? (@size * 1.0 / @count).round(2) : nil
+    end
+
+    def to_s
+      "#{type} #{count} #{size} #{avg_size}"
+    end
+  end
+
+  class Pattern
+    attr_reader :name
+    attr_accessor :total
+
+    def initialize(name)
+      @name = name
+      @total = 0
+      @items = Hash.new {|hash, key| hash[key] = PatternItem.new(key) }
+    end
+
+    def increment(type = nil, size = nil)
+      @total += 1
+      if type && size
+        @items[type].increment(size)
+      end
+    end
+
+    def <=>(other)
+      if @total == other.total
+        @name <=> other.name
+      else
+        other.total <=> @total
+      end
+    end
+
+    def to_a
+      [name, total]
+    end
+
+    def to_s
+      ret = "#{name} #{total}"
+      if @items.size > 0
+        sorted_items.each do |item|
+          ret << "\n  #{item}"
+        end
+      end
+      ret
+    end
+
+    def sorted_items
+      @items.values.sort
+    end
+  end
+end

data/lib/redis_scanner/redis.rb

@@ -0,0 +1,54 @@
+module RedisScanner
+  class Redis
+    attr_reader :client
+
+    def initialize(options)
+      @options = options
+      @client = ::Redis.new extract_redis_options(options)
+    end
+
+    def scan(*args)
+      @client.scan *args
+    end
+
+    # total keys for given db(default 0)
+    def total_keys
+      ret = 0
+      if (info = @client.info) && (str = info["db#{@options[:db].to_i}"])
+        if m = str.scan(/keys=(\d+)/)
+          ret = m.flatten.first.to_i
+        end
+      end
+      ret
+    end
+
+    def get_type_and_size(key)
+      type = @client.type key
+      size = case type
+        when "string"
+          @client.strlen key
+        when "list"
+          @client.llen key
+        when "hash"
+          @client.hlen key
+        when "set"
+          @client.scard key
+        when "zset"
+          @client.zcard key
+        else
+          1
+      end
+      [type, size.to_i]
+    end
+
+    private
+
+    def extract_redis_options(options)
+      result = {}
+      [:host, :port, :socket, :password, :db].each do |key|
+        result[key] = options[key]
+      end
+      result
+    end
+  end
+end

data/lib/redis_scanner/rule.rb

@@ -0,0 +1,26 @@
+module RedisScanner
+  class Rule
+    PRESET_RULES = [
+      [/(:\d+:)/, ":<id>:"],
+      [/(:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}:)/, ":<uuid>:"],
+      [/(:\d{4}-\d{2}-\d{2}:)/, ":<date>:"],
+      [/(:\d+)$/, ":<id>"],
+      [/(:\w{8}-\w{4}-\w{4}-\w{4}-\w{12})$/, ":<uuid>"],
+      [/(:\d{4}-\d{2}-\d{2})$/, ":<date>"]
+    ]
+
+    def initialize
+      @rules = PRESET_RULES
+    end
+
+    def extract_pattern(key)
+      @rules.each do |rule, replacer|
+        if m = rule.match(key)
+          key = key.sub(m[1], replacer)
+          break
+        end
+      end
+      key
+    end
+  end
+end

data/lib/redis_scanner/version.rb

@@ -1,3 +1,3 @@
 module RedisScanner
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/lib/redis_scanner.rb

@@ -1,32 +1,29 @@
 require "redis_scanner/version"
+require "redis_scanner/rule"
+require "redis_scanner/pattern"
+require "redis_scanner/redis"
 require "redis_scanner/engine"
+require "redis_scanner/formatter"
 require "redis"
 
 module RedisScanner
   def self.scan(options)
-    redis = Redis.new extract_redis_options(options)
+    redis = Redis.new options
     engine = Engine.new redis, options
-    result = engine.run
-    output_result(result, options)
+    patterns = engine.run
+    output_result(patterns, options)
   end
 
-  def self.output_result(result, options)
+  def self.output_result(patterns, options)
+    formatter = Formatter.new(options)
+    result = formatter.format patterns
     if options[:file]
       File.open(options[:file], "w") do |file|
-        result.each { |key, count| file.puts "#{key} #{count}" }
+        file.puts result
       end
     else
-      puts "=========result is========="
-      result.each { |key, count| puts "#{key} #{count}" }
-      puts "==========================="
+      puts result
     end
   end
 
-  def self.extract_redis_options(options)
-    result = {}
-    [:host, :port, :socket, :password, :db].each do |key|
-      result[key] = options[key]
-    end
-    result
-  end
 end

data/redis_scanner.gemspec

@@ -19,6 +19,8 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "redis"
+  spec.add_dependency "ruby-progressbar"
+  spec.add_dependency "terminal-table"
 
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redis_scanner
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Vincent Xie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-27 00:00:00.000000000 Z
+date: 2016-06-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -24,6 +24,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: terminal-table
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -86,6 +114,10 @@ files:
 - bin/redis_scanner
 - lib/redis_scanner.rb
 - lib/redis_scanner/engine.rb
+- lib/redis_scanner/formatter.rb
+- lib/redis_scanner/pattern.rb
+- lib/redis_scanner/redis.rb
+- lib/redis_scanner/rule.rb
 - lib/redis_scanner/version.rb
 - redis_scanner.gemspec
 homepage: http://github.com/xiewenwei/redis_scanner.git