msgknife 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/mdot +120 -0
- data/bin/medit +15 -0
- data/bin/mhead +30 -0
- data/bin/mheat +29 -11
- data/bin/msplit +37 -0
- data/bin/mtext +19 -0
- data/lib/msgknife/version.rb +1 -1
- data/lib/msgknife.rb +17 -14
- data/msgknife.gemspec +2 -0
- metadata +40 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4fd10df9695dc9f79dacf92ee420f6982a2036b4
|
4
|
+
data.tar.gz: 3996af2ae260d1bf6903b6bddee0446dde52a4b9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4fa624d4c51d6660a8fc842f094ca428dacbf15a0461e9e501197d48eed07515f9e4b93d1f45f7028ab77428ac358a054c169b22b54590d3f88c491d910eb037
|
7
|
+
data.tar.gz: d9ec2ad336ca7213b56b4dae51cfc95bfcc0771296bc8a7959f10ad07a2620c6c1968871a08f4c48f8c269b6c93d3d6b6a42d842062c43d5f65d0318a592229c
|
data/bin/mdot
ADDED
@@ -0,0 +1,120 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'msgknife'
|
4
|
+
require 'pp'
|
5
|
+
require 'set'
|
6
|
+
require 'digest/md5'
|
7
|
+
|
8
|
+
# MsgPack Prety Print
|
9
|
+
class MDot < Msgknife::Stream
|
10
|
+
class Node
|
11
|
+
attr_reader :key, :hkey, :fwd_link, :fwd_count, :back_link, :category
|
12
|
+
|
13
|
+
def initialize(label)
|
14
|
+
label = '(nil)' if label.nil?
|
15
|
+
@key = label
|
16
|
+
@hkey = Digest::MD5.hexdigest(label)
|
17
|
+
@fwd_link = {}
|
18
|
+
@fwd_count = Hash.new{|h,k| h[k] = 0}
|
19
|
+
@back_link = {}
|
20
|
+
end
|
21
|
+
|
22
|
+
def set_cat(cat)
|
23
|
+
@category = cat
|
24
|
+
end
|
25
|
+
|
26
|
+
def attach(tgt)
|
27
|
+
@fwd_link[tgt.key] = tgt
|
28
|
+
@fwd_count[tgt.key] += 1
|
29
|
+
tgt.attach_back(self)
|
30
|
+
end
|
31
|
+
|
32
|
+
def attach_back(tgt)
|
33
|
+
@back_link[tgt.key] = tgt
|
34
|
+
end
|
35
|
+
|
36
|
+
def set(memo=Set.new)
|
37
|
+
if memo.include?(@key)
|
38
|
+
return memo
|
39
|
+
else
|
40
|
+
memo.add(@key)
|
41
|
+
@fwd_link.each do |k,n|
|
42
|
+
memo = n.set(memo) unless memo.include?(k)
|
43
|
+
end
|
44
|
+
@back_link.each do |k,n|
|
45
|
+
memo = n.set(memo) unless memo.include?(k)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
return memo
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def initialize
|
53
|
+
super
|
54
|
+
@node_map = Hash.new{|h,k| h[k] = Node.new(k)}
|
55
|
+
@optpsr.on('-M val', 'size of minimum graph size') {|v| @min_size = v.to_i}
|
56
|
+
@optpsr.on('-N', 'skip if value is nil') {|v| @skip_nil = v }
|
57
|
+
@optpsr.on('-d', 'output as dot format') {|v| @out_dot = v }
|
58
|
+
end
|
59
|
+
|
60
|
+
def setup(argv)
|
61
|
+
@key1 = argv[0]
|
62
|
+
@key2 = argv[1]
|
63
|
+
end
|
64
|
+
|
65
|
+
def recv(obj, ts, tag)
|
66
|
+
if obj.key?(@key1) and obj.key?(@key2)
|
67
|
+
return if @skip_nil and (obj[@key1].nil? or obj[@key2].nil?)
|
68
|
+
n1 = @node_map[obj[@key1]]
|
69
|
+
n2 = @node_map[obj[@key2]]
|
70
|
+
n1.set_cat(0)
|
71
|
+
n2.set_cat(1)
|
72
|
+
n1.attach(n2)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def teardown
|
77
|
+
ignore_set = Set.new
|
78
|
+
unless @min_size.nil?
|
79
|
+
@node_map.each do |key, node|
|
80
|
+
next if ignore_set.include?(key)
|
81
|
+
m = node.set
|
82
|
+
m.each {|n| ignore_set.add(n)} if m.size < @min_size
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
if @out_dot
|
87
|
+
puts "digraph mdot {"
|
88
|
+
@node_map.each do |key, node|
|
89
|
+
next if ignore_set.include?(node.key)
|
90
|
+
key = node.key.gsub('"', '\"')
|
91
|
+
case node.category
|
92
|
+
when 0; color = '#fbeac1'
|
93
|
+
when 1; color = '#ecac12'
|
94
|
+
end
|
95
|
+
puts " \"#{node.hkey}\" [label=\"#{key}\" color=\"#{color}\" style=\"filled\"];"
|
96
|
+
end
|
97
|
+
|
98
|
+
@node_map.each do |key1, node1|
|
99
|
+
next if ignore_set.include?(key1)
|
100
|
+
node1.fwd_link.each do |key2, node2|
|
101
|
+
count = node1.fwd_count[key2]
|
102
|
+
puts " \"#{node1.hkey}\" -> \"#{node2.hkey}\" [label=\"#{count}\" len=2.5];"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
puts "}"
|
106
|
+
|
107
|
+
else
|
108
|
+
@node_map.each do |key1, node1|
|
109
|
+
next if ignore_set.include?(key1)
|
110
|
+
node1.fwd_link.each do |key2, node2|
|
111
|
+
count = node1.fwd_count[key2]
|
112
|
+
write_stream({'src'=>key1, 'dst'=>key2, 'count'=>count})
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
ins = MDot.new()
|
120
|
+
ins.run(ARGV, 0..1)
|
data/bin/medit
ADDED
data/bin/mhead
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'msgknife'
|
4
|
+
require 'pp'
|
5
|
+
|
6
|
+
class MHead < Msgknife::Stream
|
7
|
+
def initialize
|
8
|
+
super
|
9
|
+
@num = 20
|
10
|
+
@json = false
|
11
|
+
@optpsr.on('-n VAL', 'number of message to show') {|v| @num = v.to_i }
|
12
|
+
end
|
13
|
+
|
14
|
+
def setup(argv)
|
15
|
+
@count = 0
|
16
|
+
end
|
17
|
+
def recv(obj, ts, tag)
|
18
|
+
if @count < @num
|
19
|
+
write_stream(obj, ts, tag)
|
20
|
+
else
|
21
|
+
return false
|
22
|
+
end
|
23
|
+
@count += 1
|
24
|
+
end
|
25
|
+
def teardown
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
MHead.new.run(ARGV)
|
30
|
+
|
data/bin/mheat
CHANGED
@@ -6,6 +6,7 @@ require 'pp'
|
|
6
6
|
require 'time'
|
7
7
|
require 'msgpack'
|
8
8
|
require 'json'
|
9
|
+
require 'terminfo'
|
9
10
|
|
10
11
|
# MsgPack Prety Print
|
11
12
|
class MHeatMap < Msgknife::Stream
|
@@ -19,19 +20,20 @@ class MHeatMap < Msgknife::Stream
|
|
19
20
|
@ts_key = 'ts'
|
20
21
|
@optpsr.on('-t VAL', 'timestamp key name') { |v| @ts_key = v }
|
21
22
|
|
23
|
+
@ts_start = nil
|
24
|
+
@ts_end = nil
|
25
|
+
@optpsr.on('-s VAL', 'timestamp of start') {|v| @ts_start = v.to_i }
|
26
|
+
@optpsr.on('-e VAL', 'timestamp of end') {|v| @ts_end = v.to_i }
|
27
|
+
|
22
28
|
@count_key = nil
|
23
29
|
@optpsr.on('-c VAL', 'key name to sum numbers') { |v| @count_key = v }
|
24
30
|
|
25
|
-
@width =
|
31
|
+
@width = TermInfo.screen_size[1] - 35
|
26
32
|
@optpsr.on('-w VAL', 'width') { |v| @width = v.to_i }
|
27
33
|
|
28
34
|
@relative = false
|
29
35
|
@optpsr.on('-r', 'relative time') { |v| @relative = true }
|
30
36
|
|
31
|
-
@output_fmt = 'console'
|
32
|
-
@optpsr.on('-j', 'output as json') { |v| @output_fmt = 'json' }
|
33
|
-
@optpsr.on('-m', 'output as msgpack') { |v| @output_fmt = 'msgpack' }
|
34
|
-
|
35
37
|
# Set internval parameters
|
36
38
|
@key = nil
|
37
39
|
@ts_min = nil
|
@@ -68,7 +70,17 @@ class MHeatMap < Msgknife::Stream
|
|
68
70
|
end
|
69
71
|
|
70
72
|
def output_console(obj)
|
71
|
-
|
73
|
+
ep = obj['cols'].size - 1
|
74
|
+
dt1_len = obj['cols'][0].size
|
75
|
+
dt2_len = obj['cols'][ep].size
|
76
|
+
sp_len = @width - (dt1_len + dt2_len) - 2
|
77
|
+
if sp_len > 0
|
78
|
+
STDOUT.write(sprintf("%30s | %s %s %s|" , "", obj['cols'][0], " " * sp_len,
|
79
|
+
obj['cols'][ep]))
|
80
|
+
else
|
81
|
+
STDOUT.write(sprintf("%30s | %s" , "", obj['cols'][0]))
|
82
|
+
end
|
83
|
+
|
72
84
|
puts
|
73
85
|
STDOUT.write(sprintf("%30s | %s|", "-" * 28,
|
74
86
|
"-" * @width))
|
@@ -95,6 +107,10 @@ class MHeatMap < Msgknife::Stream
|
|
95
107
|
end
|
96
108
|
|
97
109
|
def teardown
|
110
|
+
@ts_max = @ts_end unless @ts_end.nil?
|
111
|
+
@ts_min = @ts_start unless @ts_start.nil?
|
112
|
+
|
113
|
+
raise 'start timestamp should be less than end timestamp' if @ts_max < @ts_min
|
98
114
|
dur_ts = @ts_max - @ts_min
|
99
115
|
ts_unit = dur_ts / @width
|
100
116
|
|
@@ -117,11 +133,13 @@ class MHeatMap < Msgknife::Stream
|
|
117
133
|
}}
|
118
134
|
@rows.each do |k, ts_list|
|
119
135
|
ts_list.each do |ts, val|
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
data_map[k]['max']
|
136
|
+
if @ts_min <= ts and ts <= @ts_max
|
137
|
+
idx = ((ts - @ts_min) / ts_unit).to_i
|
138
|
+
idx = @width - 1 if idx >= @width
|
139
|
+
data_map[k]['row'][idx] += val
|
140
|
+
if data_map[k]['max'] < data_map[k]['row'][idx]
|
141
|
+
data_map[k]['max'] = data_map[k]['row'][idx]
|
142
|
+
end
|
125
143
|
end
|
126
144
|
end
|
127
145
|
end
|
data/bin/msplit
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'msgknife'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
# MsgPack Prety Print
|
7
|
+
class MGrep < Msgknife::Stream
|
8
|
+
def initialize
|
9
|
+
super
|
10
|
+
@optpsr.on('-a', 'append mode') { |v| @append = v }
|
11
|
+
@optpsr.on('-o VAL', 'output directory') { |v| @outdir = v }
|
12
|
+
end
|
13
|
+
|
14
|
+
def setup(argv)
|
15
|
+
@key = argv[0]
|
16
|
+
@fd_map = Hash.new
|
17
|
+
if !(@outdir.nil?) and !(File.exists?(@outdir))
|
18
|
+
FileUtils.mkdir_p @outdir
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def recv(obj, ts, tag)
|
23
|
+
val = obj.has_key?(@key) ? obj[@key] : 'unknown'
|
24
|
+
unless @fd_map.has_key?(val)
|
25
|
+
mode = @append ? 'wb+' : 'wb'
|
26
|
+
fpath = (val.nil?) ? 'nil.msg' : val + '.msg'
|
27
|
+
puts fpath
|
28
|
+
fpath = File.join(@outdir, fpath) unless @outdir.nil?
|
29
|
+
@fd_map[val] = File.open(fpath, mode)
|
30
|
+
end
|
31
|
+
|
32
|
+
write_stream(obj, ts, tag, @fd_map[val])
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
ins = MGrep.new()
|
37
|
+
ins.run(ARGV, 0..0)
|
data/bin/mtext
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'msgknife'
|
4
|
+
require 'pp'
|
5
|
+
|
6
|
+
# MsgPack Prety Print
|
7
|
+
class MPP < Msgknife::Stream
|
8
|
+
def setup(argv)
|
9
|
+
@key = argv[0]
|
10
|
+
end
|
11
|
+
def recv(obj, ts, tag)
|
12
|
+
if obj.key?(@key) and (!@ignore_nil or !(obj[@key].nil?))
|
13
|
+
puts obj[@key]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
ins = MPP.new()
|
19
|
+
ins.run(ARGV, 0..0)
|
data/lib/msgknife/version.rb
CHANGED
data/lib/msgknife.rb
CHANGED
@@ -1,7 +1,11 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
|
1
3
|
require "msgknife/version"
|
2
4
|
require 'msgpack'
|
3
5
|
require 'optparse'
|
4
6
|
require 'time'
|
7
|
+
require 'pp'
|
8
|
+
require 'json'
|
5
9
|
|
6
10
|
module Msgknife
|
7
11
|
class Stream
|
@@ -10,11 +14,14 @@ module Msgknife
|
|
10
14
|
def initialize
|
11
15
|
@out_encode = false
|
12
16
|
@optpsr = OptionParser.new
|
13
|
-
@optpsr.on('-m', 'output as msgpack encode') { |v| @out_encode = v }
|
14
17
|
@optpsr.on('-F', 'input as fluentd format') { |v| @in_fluentd = v }
|
15
18
|
@optpsr.on('-T VAL', 'key of timestamp in message') { |v| @ts_key = v }
|
16
19
|
@optpsr.on('-N', 'ignore if value is nil') {|v| @ignore_nil = v }
|
17
20
|
@argv = []
|
21
|
+
|
22
|
+
@output_fmt = 'console'
|
23
|
+
@optpsr.on('-j', 'output as json') { |v| @output_fmt = 'json' }
|
24
|
+
@optpsr.on('-m', 'output as msgpack') { |v| @output_fmt = 'msgpack' }
|
18
25
|
end
|
19
26
|
|
20
27
|
def run(cmd_argv, range = nil)
|
@@ -88,21 +95,17 @@ module Msgknife
|
|
88
95
|
end
|
89
96
|
end
|
90
97
|
|
91
|
-
def write_stream(obj, ts=nil, tag=nil)
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
if ts.nil? and tag.nil?
|
100
|
-
STDOUT.write(obj.to_msgpack)
|
101
|
-
else
|
102
|
-
STDOUT.write([tag, ts, obj].to_msgpack)
|
98
|
+
def write_stream(obj, ts=nil, tag=nil, io=STDOUT)
|
99
|
+
msg =(ts.nil? and tag.nil?) ? obj : [tag, ts, obj]
|
100
|
+
|
101
|
+
begin
|
102
|
+
case @output_fmt
|
103
|
+
when 'console'; PP.pp(msg, io);
|
104
|
+
when 'json'; JSON.dump(msg, io);
|
105
|
+
when 'msgpack'; io.write(msg.to_msgpack);
|
103
106
|
end
|
107
|
+
rescue Errno::EPIPE => e ;
|
104
108
|
end
|
105
|
-
rescue Errno::EPIPE => e ;
|
106
109
|
end
|
107
110
|
|
108
111
|
|
data/msgknife.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: msgknife
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masayoshi Mizutani
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-04-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -52,15 +52,48 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0.5'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: ruby-terminfo
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - ">="
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ">="
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: diff-lcs
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
55
83
|
description: MsgKnife makes easy to view, check and analyze MessagePack format file
|
56
84
|
and stream such as Fluentd data
|
57
85
|
email:
|
58
86
|
- mizutani@sfc.wide.ad.jp
|
59
87
|
executables:
|
88
|
+
- mdot
|
89
|
+
- medit
|
60
90
|
- mentropy
|
91
|
+
- mhead
|
61
92
|
- mheat
|
62
93
|
- mkeys
|
63
94
|
- mpp
|
95
|
+
- msplit
|
96
|
+
- mtext
|
64
97
|
- mval
|
65
98
|
extensions: []
|
66
99
|
extra_rdoc_files: []
|
@@ -70,10 +103,15 @@ files:
|
|
70
103
|
- LICENSE.txt
|
71
104
|
- README.md
|
72
105
|
- Rakefile
|
106
|
+
- bin/mdot
|
107
|
+
- bin/medit
|
73
108
|
- bin/mentropy
|
109
|
+
- bin/mhead
|
74
110
|
- bin/mheat
|
75
111
|
- bin/mkeys
|
76
112
|
- bin/mpp
|
113
|
+
- bin/msplit
|
114
|
+
- bin/mtext
|
77
115
|
- bin/mval
|
78
116
|
- lib/msgknife.rb
|
79
117
|
- lib/msgknife/version.rb
|