redis_scanner 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -3
- data/README.md +35 -12
- data/bin/redis_scanner +16 -2
- data/lib/redis_scanner/engine.rb +28 -30
- data/lib/redis_scanner/formatter.rb +71 -0
- data/lib/redis_scanner/pattern.rb +76 -0
- data/lib/redis_scanner/redis.rb +54 -0
- data/lib/redis_scanner/rule.rb +26 -0
- data/lib/redis_scanner/version.rb +1 -1
- data/lib/redis_scanner.rb +12 -15
- data/redis_scanner.gemspec +2 -0
- metadata +34 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f4d4379646a6674439ca7ad275828c5881c97901
|
4
|
+
data.tar.gz: 36e42b13cab65c468e71dded6a2edfafeaba3671
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1d4180f672190a047a5a39269b1d726ad8b26c3dfe9d640d8839006c0be2050d96a185bbbaf16c883f62cc05afa6387f00ba62de2c8c325aab542431c8cbc21e
|
7
|
+
data.tar.gz: 83599028ff33ec24a5f573a5332aedb1db25a03c9a7e3daebd364210cffc34e52e6d39af65052a7ef0aaecbc5cf90ff0df278b8d5ad27b1acc5d5fb4a0193fd6
|
data/CHANGELOG.md
CHANGED
@@ -1,12 +1,19 @@
|
|
1
|
+
## v0.1.3
|
2
|
+
|
3
|
+
* Provide key's detail information which includes type and size.
|
4
|
+
* Provide table format output. And you can use simpe output also.
|
5
|
+
* Add progress bar.
|
6
|
+
* Support limit, format and detail options.
|
7
|
+
* Refactor codes.Add more auto test cases.
|
1
8
|
|
2
9
|
## v0.1.2
|
3
10
|
|
4
|
-
* Support date pattern
|
5
|
-
* Add auto test cases
|
11
|
+
* Support date pattern.
|
12
|
+
* Add auto test cases.
|
6
13
|
|
7
14
|
## v0.1.1
|
8
15
|
|
9
|
-
* Add redis_scanner to executable file
|
16
|
+
* Add redis_scanner to executable file.
|
10
17
|
|
11
18
|
## v0.1.0
|
12
19
|
|
data/README.md
CHANGED
@@ -4,6 +4,8 @@ RedisScanner is a tiny tool for scanning all redis keys and creating statistic r
|
|
4
4
|
|
5
5
|
* Scans keys using *scan* replacing *keys* command.
|
6
6
|
* Creates statistic result by key's pattern.
|
7
|
+
* Provide key's detail information which includes type and size.
|
8
|
+
* Provide table format output..
|
7
9
|
|
8
10
|
## Installation
|
9
11
|
|
@@ -30,19 +32,40 @@ redis_scanner
|
|
30
32
|
The Output is like this:
|
31
33
|
|
32
34
|
```shell
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
bh:
|
42
|
-
bh:stat:processed 1
|
43
|
-
bus_app:app_two 1
|
35
|
+
+------------------------------------+-------+
|
36
|
+
| Key | Count |
|
37
|
+
+------------------------------------+-------+
|
38
|
+
| demo:user:<id>:counter | 10000 |
|
39
|
+
| u:<uuid>:pf | 52 |
|
40
|
+
| sidekiq_demo:stat:failed:<date> | 4 |
|
41
|
+
| sidekiq_demo:stat:processed:<date> | 4 |
|
42
|
+
| _sp_one:queue:default | 1 |
|
43
|
+
| bh:queues | 1 |
|
44
44
|
...
|
45
|
-
|
45
|
+
+------------------------------------+-------+
|
46
|
+
```
|
47
|
+
|
48
|
+
```shell
|
49
|
+
redis_scanner -d
|
50
|
+
```
|
51
|
+
|
52
|
+
The Output is like this:
|
53
|
+
|
54
|
+
```shell
|
55
|
+
+------------------------------------+-------+--------+---------+
|
56
|
+
| Key | Count | Size | AvgSize |
|
57
|
+
+------------------------------------+-------+--------+---------+
|
58
|
+
| demo:user:<id>:counter | 10000 | | |
|
59
|
+
| > string | 10000 | 927510 | 92.75 |
|
60
|
+
| u:<uuid>:pf | 52 | | |
|
61
|
+
| > hash | 52 | 108 | 2.08 |
|
62
|
+
| sidekiq_demo:stat:failed:<date> | 4 | | |
|
63
|
+
| > string | 4 | 5 | 1.25 |
|
64
|
+
| sidekiq_demo:stat:processed:<date> | 4 | | |
|
65
|
+
| > string | 4 | 6 | 1.5 |
|
66
|
+
| _sp_one:queue:default | 1 | | |
|
67
|
+
| > list | 1 | 1 | 1.0 |
|
68
|
+
+------------------------------------+-------+--------+---------+
|
46
69
|
```
|
47
70
|
|
48
71
|
* Scan keys with some pattern
|
data/bin/redis_scanner
CHANGED
@@ -9,14 +9,28 @@ options = {}
|
|
9
9
|
OptionParser.new do |opts|
|
10
10
|
opts.banner = "Usage: redis_scanner [options]"
|
11
11
|
|
12
|
-
opts.on("-f FILE", "--file FILE", "Output file") do |v|
|
12
|
+
opts.on("-f FILE", "--file FILE", "Output to file") do |v|
|
13
13
|
options[:file] = v
|
14
14
|
end
|
15
15
|
|
16
|
-
opts.on("-m MATCH", "--match MATCH", "
|
16
|
+
opts.on("-m MATCH", "--match MATCH", "Only scan the pattern") do |v|
|
17
17
|
options[:match] = v
|
18
18
|
end
|
19
19
|
|
20
|
+
opts.on("-l LIMIT", "--limit LIMIT", "Only show top <limit> keys") do |v|
|
21
|
+
options[:limit] = v.to_i
|
22
|
+
end
|
23
|
+
|
24
|
+
options[:detail] = false
|
25
|
+
opts.on("-d", "--detail", "Show detail info(type & size)") do |v|
|
26
|
+
options[:detail] = v
|
27
|
+
end
|
28
|
+
|
29
|
+
options[:format] = 'table'
|
30
|
+
opts.on("-t FORMAT", "--format FORMAT", "Formt(simple or talbe. default is table)") do |v|
|
31
|
+
options[:format] = v
|
32
|
+
end
|
33
|
+
|
20
34
|
# redis client options
|
21
35
|
# -h <hostname> Server hostname (default: 127.0.0.1).
|
22
36
|
# -p <port> Server port (default: 6379).
|
data/lib/redis_scanner/engine.rb
CHANGED
@@ -1,29 +1,37 @@
|
|
1
|
+
require "ruby-progressbar"
|
2
|
+
|
1
3
|
module RedisScanner
|
2
4
|
class Engine
|
3
5
|
def initialize(redis, options)
|
4
6
|
@redis = redis
|
5
7
|
@options = options
|
8
|
+
@rule = Rule.new
|
6
9
|
end
|
7
10
|
|
8
11
|
def run
|
9
|
-
|
12
|
+
result = scan
|
13
|
+
result.values.sort
|
10
14
|
end
|
11
15
|
|
12
16
|
private
|
13
17
|
|
14
|
-
def
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
18
|
+
def create_progress_bar
|
19
|
+
total = @redis.total_keys
|
20
|
+
bar = ProgressBar.create(
|
21
|
+
title: "Keys",
|
22
|
+
format: '%a %bᗧ%i %p%% %t',
|
23
|
+
progress_mark: ' ',
|
24
|
+
remainder_mark: '・',
|
25
|
+
total: total)
|
26
|
+
bar.log "total keys is #{total}"
|
27
|
+
bar
|
22
28
|
end
|
23
29
|
|
24
30
|
def scan
|
25
31
|
cursor = 0
|
26
|
-
stat = Hash.new(
|
32
|
+
stat = Hash.new {|hash, key| hash[key] = Pattern.new(key) }
|
33
|
+
|
34
|
+
bar = create_progress_bar
|
27
35
|
while true
|
28
36
|
if @options[:match]
|
29
37
|
cursor, result = @redis.scan cursor, match: @options[:match]
|
@@ -31,32 +39,22 @@ module RedisScanner
|
|
31
39
|
cursor, result = @redis.scan cursor
|
32
40
|
end
|
33
41
|
result.each do |key|
|
34
|
-
pattern =
|
35
|
-
|
42
|
+
pattern = @rule.extract_pattern(key)
|
43
|
+
if @options[:detail]
|
44
|
+
type, size = @redis.get_type_and_size(key)
|
45
|
+
stat[pattern].increment type, size
|
46
|
+
else
|
47
|
+
stat[pattern].increment
|
48
|
+
end
|
49
|
+
bar.increment
|
36
50
|
end
|
37
51
|
cursor = cursor.to_i
|
38
52
|
break if cursor == 0
|
39
53
|
end
|
54
|
+
bar.finish
|
55
|
+
|
40
56
|
stat
|
41
57
|
end
|
42
58
|
|
43
|
-
PATTERNS = [
|
44
|
-
[/(:\d+:)/, ":<id>:"],
|
45
|
-
[/(:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}:)/, ":<uuid>:"],
|
46
|
-
[/(:\d{4}-\d{2}-\d{2}:)/, ":<date>:"],
|
47
|
-
[/(:\d+)$/, ":<id>"],
|
48
|
-
[/(:\w{8}-\w{4}-\w{4}-\w{4}-\w{12})$/, ":<uuid>"],
|
49
|
-
[/(:\d{4}-\d{2}-\d{2})$/, ":<date>"]
|
50
|
-
]
|
51
|
-
|
52
|
-
def resolve_pattern(key)
|
53
|
-
PATTERNS.each do |pattern, replacer|
|
54
|
-
if m = pattern.match(key)
|
55
|
-
key = key.sub(m[1], replacer)
|
56
|
-
break
|
57
|
-
end
|
58
|
-
end
|
59
|
-
key
|
60
|
-
end
|
61
59
|
end
|
62
60
|
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require "terminal-table"
|
2
|
+
|
3
|
+
module RedisScanner
|
4
|
+
class Formatter
|
5
|
+
def initialize(options)
|
6
|
+
@options = options
|
7
|
+
end
|
8
|
+
|
9
|
+
def format(patterns)
|
10
|
+
if @options[:format] == "simple"
|
11
|
+
simple patterns
|
12
|
+
else
|
13
|
+
table patterns
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def touch_limit?(count)
|
20
|
+
@options[:limit] &&
|
21
|
+
@options[:limit].to_i > 0 &&
|
22
|
+
count >= @options[:limit].to_i
|
23
|
+
end
|
24
|
+
|
25
|
+
def simple(patterns)
|
26
|
+
ret = ""
|
27
|
+
count = 0
|
28
|
+
patterns.each do |pattern|
|
29
|
+
ret << pattern.to_s
|
30
|
+
ret << "\n"
|
31
|
+
count += 1
|
32
|
+
break if touch_limit?(count)
|
33
|
+
end
|
34
|
+
ret
|
35
|
+
end
|
36
|
+
|
37
|
+
def table(patterns)
|
38
|
+
with_detail = @options[:detail]
|
39
|
+
rows = []
|
40
|
+
count = 0
|
41
|
+
|
42
|
+
patterns.each do |pattern|
|
43
|
+
if with_detail
|
44
|
+
rows << [pattern.name, pattern.total, "", ""]
|
45
|
+
pattern.sorted_items.each do |item|
|
46
|
+
rows << [" > #{item.type}", item.count, item.size, item.avg_size]
|
47
|
+
end
|
48
|
+
else
|
49
|
+
rows << [pattern.name, pattern.total]
|
50
|
+
end
|
51
|
+
count += 1
|
52
|
+
break if touch_limit?(count)
|
53
|
+
end
|
54
|
+
|
55
|
+
headings = ['Key', 'Count']
|
56
|
+
if with_detail
|
57
|
+
headings << "Size"
|
58
|
+
headings << "AvgSize"
|
59
|
+
end
|
60
|
+
|
61
|
+
table = Terminal::Table.new headings: headings, rows: rows
|
62
|
+
table.align_column(1, :right)
|
63
|
+
if with_detail
|
64
|
+
table.align_column(2, :right)
|
65
|
+
table.align_column(3, :right)
|
66
|
+
end
|
67
|
+
|
68
|
+
table.to_s
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module RedisScanner
|
2
|
+
class PatternItem
|
3
|
+
attr_reader :type, :count, :size
|
4
|
+
|
5
|
+
def initialize(type)
|
6
|
+
@type = type
|
7
|
+
@count = 0
|
8
|
+
@size = 0
|
9
|
+
end
|
10
|
+
|
11
|
+
def increment(size)
|
12
|
+
@count += 1
|
13
|
+
@size += size
|
14
|
+
end
|
15
|
+
|
16
|
+
def <=>(other)
|
17
|
+
if @count == other.count
|
18
|
+
@type <=> other.type
|
19
|
+
else
|
20
|
+
other.count <=> @count
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def avg_size
|
25
|
+
@count > 0 ? (@size * 1.0 / @count).round(2) : nil
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_s
|
29
|
+
"#{type} #{count} #{size} #{avg_size}"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
class Pattern
|
34
|
+
attr_reader :name
|
35
|
+
attr_accessor :total
|
36
|
+
|
37
|
+
def initialize(name)
|
38
|
+
@name = name
|
39
|
+
@total = 0
|
40
|
+
@items = Hash.new {|hash, key| hash[key] = PatternItem.new(key) }
|
41
|
+
end
|
42
|
+
|
43
|
+
def increment(type = nil, size = nil)
|
44
|
+
@total += 1
|
45
|
+
if type && size
|
46
|
+
@items[type].increment(size)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def <=>(other)
|
51
|
+
if @total == other.total
|
52
|
+
@name <=> other.name
|
53
|
+
else
|
54
|
+
other.total <=> @total
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def to_a
|
59
|
+
[name, total]
|
60
|
+
end
|
61
|
+
|
62
|
+
def to_s
|
63
|
+
ret = "#{name} #{total}"
|
64
|
+
if @items.size > 0
|
65
|
+
sorted_items.each do |item|
|
66
|
+
ret << "\n #{item}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
ret
|
70
|
+
end
|
71
|
+
|
72
|
+
def sorted_items
|
73
|
+
@items.values.sort
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module RedisScanner
|
2
|
+
class Redis
|
3
|
+
attr_reader :client
|
4
|
+
|
5
|
+
def initialize(options)
|
6
|
+
@options = options
|
7
|
+
@client = ::Redis.new extract_redis_options(options)
|
8
|
+
end
|
9
|
+
|
10
|
+
def scan(*args)
|
11
|
+
@client.scan *args
|
12
|
+
end
|
13
|
+
|
14
|
+
# total keys for given db(default 0)
|
15
|
+
def total_keys
|
16
|
+
ret = 0
|
17
|
+
if (info = @client.info) && (str = info["db#{@options[:db].to_i}"])
|
18
|
+
if m = str.scan(/keys=(\d+)/)
|
19
|
+
ret = m.flatten.first.to_i
|
20
|
+
end
|
21
|
+
end
|
22
|
+
ret
|
23
|
+
end
|
24
|
+
|
25
|
+
def get_type_and_size(key)
|
26
|
+
type = @client.type key
|
27
|
+
size = case type
|
28
|
+
when "string"
|
29
|
+
@client.strlen key
|
30
|
+
when "list"
|
31
|
+
@client.llen key
|
32
|
+
when "hash"
|
33
|
+
@client.hlen key
|
34
|
+
when "set"
|
35
|
+
@client.scard key
|
36
|
+
when "zset"
|
37
|
+
@client.zcard key
|
38
|
+
else
|
39
|
+
1
|
40
|
+
end
|
41
|
+
[type, size.to_i]
|
42
|
+
end
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def extract_redis_options(options)
|
47
|
+
result = {}
|
48
|
+
[:host, :port, :socket, :password, :db].each do |key|
|
49
|
+
result[key] = options[key]
|
50
|
+
end
|
51
|
+
result
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
module RedisScanner
|
2
|
+
class Rule
|
3
|
+
PRESET_RULES = [
|
4
|
+
[/(:\d+:)/, ":<id>:"],
|
5
|
+
[/(:\w{8}-\w{4}-\w{4}-\w{4}-\w{12}:)/, ":<uuid>:"],
|
6
|
+
[/(:\d{4}-\d{2}-\d{2}:)/, ":<date>:"],
|
7
|
+
[/(:\d+)$/, ":<id>"],
|
8
|
+
[/(:\w{8}-\w{4}-\w{4}-\w{4}-\w{12})$/, ":<uuid>"],
|
9
|
+
[/(:\d{4}-\d{2}-\d{2})$/, ":<date>"]
|
10
|
+
]
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@rules = PRESET_RULES
|
14
|
+
end
|
15
|
+
|
16
|
+
def extract_pattern(key)
|
17
|
+
@rules.each do |rule, replacer|
|
18
|
+
if m = rule.match(key)
|
19
|
+
key = key.sub(m[1], replacer)
|
20
|
+
break
|
21
|
+
end
|
22
|
+
end
|
23
|
+
key
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/redis_scanner.rb
CHANGED
@@ -1,32 +1,29 @@
|
|
1
1
|
require "redis_scanner/version"
|
2
|
+
require "redis_scanner/rule"
|
3
|
+
require "redis_scanner/pattern"
|
4
|
+
require "redis_scanner/redis"
|
2
5
|
require "redis_scanner/engine"
|
6
|
+
require "redis_scanner/formatter"
|
3
7
|
require "redis"
|
4
8
|
|
5
9
|
module RedisScanner
|
6
10
|
def self.scan(options)
|
7
|
-
redis = Redis.new
|
11
|
+
redis = Redis.new options
|
8
12
|
engine = Engine.new redis, options
|
9
|
-
|
10
|
-
output_result(
|
13
|
+
patterns = engine.run
|
14
|
+
output_result(patterns, options)
|
11
15
|
end
|
12
16
|
|
13
|
-
def self.output_result(
|
17
|
+
def self.output_result(patterns, options)
|
18
|
+
formatter = Formatter.new(options)
|
19
|
+
result = formatter.format patterns
|
14
20
|
if options[:file]
|
15
21
|
File.open(options[:file], "w") do |file|
|
16
|
-
|
22
|
+
file.puts result
|
17
23
|
end
|
18
24
|
else
|
19
|
-
puts
|
20
|
-
result.each { |key, count| puts "#{key} #{count}" }
|
21
|
-
puts "==========================="
|
25
|
+
puts result
|
22
26
|
end
|
23
27
|
end
|
24
28
|
|
25
|
-
def self.extract_redis_options(options)
|
26
|
-
result = {}
|
27
|
-
[:host, :port, :socket, :password, :db].each do |key|
|
28
|
-
result[key] = options[key]
|
29
|
-
end
|
30
|
-
result
|
31
|
-
end
|
32
29
|
end
|
data/redis_scanner.gemspec
CHANGED
@@ -19,6 +19,8 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
21
|
spec.add_dependency "redis"
|
22
|
+
spec.add_dependency "ruby-progressbar"
|
23
|
+
spec.add_dependency "terminal-table"
|
22
24
|
|
23
25
|
spec.add_development_dependency "bundler", "~> 1.11"
|
24
26
|
spec.add_development_dependency "rake", "~> 10.0"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redis_scanner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Vincent Xie
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06-
|
11
|
+
date: 2016-06-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -24,6 +24,34 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: ruby-progressbar
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: terminal-table
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
27
55
|
- !ruby/object:Gem::Dependency
|
28
56
|
name: bundler
|
29
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -86,6 +114,10 @@ files:
|
|
86
114
|
- bin/redis_scanner
|
87
115
|
- lib/redis_scanner.rb
|
88
116
|
- lib/redis_scanner/engine.rb
|
117
|
+
- lib/redis_scanner/formatter.rb
|
118
|
+
- lib/redis_scanner/pattern.rb
|
119
|
+
- lib/redis_scanner/redis.rb
|
120
|
+
- lib/redis_scanner/rule.rb
|
89
121
|
- lib/redis_scanner/version.rb
|
90
122
|
- redis_scanner.gemspec
|
91
123
|
homepage: http://github.com/xiewenwei/redis_scanner.git
|