gri 10.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +7 -0
  3. data/MIT-LICENSE.txt +20 -0
  4. data/README.md +132 -0
  5. data/Rakefile +5 -0
  6. data/bin/grapher +17 -0
  7. data/bin/gri +5 -0
  8. data/bin/gricast +17 -0
  9. data/bin/grispark +5 -0
  10. data/bin/griwalk +9 -0
  11. data/bin/trad +19 -0
  12. data/bin/trad-http +17 -0
  13. data/gri.gemspec +21 -0
  14. data/lib/fluent/plugin/out_gri.rb +56 -0
  15. data/lib/gri/api.rb +28 -0
  16. data/lib/gri/app_collector.rb +154 -0
  17. data/lib/gri/app_walker.rb +23 -0
  18. data/lib/gri/blank.rb +49 -0
  19. data/lib/gri/builtindefs.rb +211 -0
  20. data/lib/gri/cast.rb +84 -0
  21. data/lib/gri/cgraph.rb +37 -0
  22. data/lib/gri/clist.rb +163 -0
  23. data/lib/gri/collector.rb +284 -0
  24. data/lib/gri/config.rb +134 -0
  25. data/lib/gri/ds_list.rb +166 -0
  26. data/lib/gri/format_helper.rb +112 -0
  27. data/lib/gri/gparams.rb +43 -0
  28. data/lib/gri/graph.rb +239 -0
  29. data/lib/gri/grapher.rb +70 -0
  30. data/lib/gri/ldb.rb +160 -0
  31. data/lib/gri/list.rb +242 -0
  32. data/lib/gri/log.rb +140 -0
  33. data/lib/gri/loop.rb +109 -0
  34. data/lib/gri/ltsv.rb +58 -0
  35. data/lib/gri/main.rb +107 -0
  36. data/lib/gri/mlog.rb +22 -0
  37. data/lib/gri/mmsgpack.rb +57 -0
  38. data/lib/gri/msnmp.rb +601 -0
  39. data/lib/gri/page.rb +235 -0
  40. data/lib/gri/pcollector.rb +209 -0
  41. data/lib/gri/plugin.rb +75 -0
  42. data/lib/gri/plugin/bootstrap.rb +65 -0
  43. data/lib/gri/plugin/cisco.rb +98 -0
  44. data/lib/gri/plugin/exec_collector.rb +89 -0
  45. data/lib/gri/plugin/juniper.rb +5 -0
  46. data/lib/gri/plugin/netsnmp.rb +8 -0
  47. data/lib/gri/plugin/ucdavis.rb +176 -0
  48. data/lib/gri/plugin/writer_fluentd.rb +26 -0
  49. data/lib/gri/polling_unit.rb +88 -0
  50. data/lib/gri/q.rb +5 -0
  51. data/lib/gri/request.rb +29 -0
  52. data/lib/gri/rrd.rb +438 -0
  53. data/lib/gri/scheduler.rb +68 -0
  54. data/lib/gri/sgraph.rb +147 -0
  55. data/lib/gri/spark.rb +94 -0
  56. data/lib/gri/tra_collector.rb +80 -0
  57. data/lib/gri/trad.rb +170 -0
  58. data/lib/gri/updater.rb +201 -0
  59. data/lib/gri/util_daemon.rb +19 -0
  60. data/lib/gri/util_marshal.rb +13 -0
  61. data/lib/gri/utils.rb +63 -0
  62. data/lib/gri/vendor.rb +76 -0
  63. data/lib/gri/version.rb +3 -0
  64. data/lib/gri/wmain.rb +67 -0
  65. data/lib/gri/writer.rb +184 -0
  66. data/mcollector +47 -0
  67. data/test/mock.rb +60 -0
  68. data/test/root/gra/.sysdb/sysdb.txt +3 -0
  69. data/test/root/gra/testhost/.records.txt +2 -0
  70. data/test/root/if.def +2 -0
  71. data/test/root/test.conf +4 -0
  72. data/test/root/testtab +9 -0
  73. data/test/root/testtab2 +2 -0
  74. data/test/root/tra/testhost/_/20130614 +20 -0
  75. data/test/test_app.rb +58 -0
  76. data/test/test_builtindefs.rb +24 -0
  77. data/test/test_collector.rb +58 -0
  78. data/test/test_config.rb +62 -0
  79. data/test/test_ds_list.rb +48 -0
  80. data/test/test_exec_collector.rb +33 -0
  81. data/test/test_format_helper.rb +68 -0
  82. data/test/test_graph.rb +69 -0
  83. data/test/test_ldb.rb +29 -0
  84. data/test/test_list.rb +65 -0
  85. data/test/test_log.rb +16 -0
  86. data/test/test_loop.rb +35 -0
  87. data/test/test_ltsv.rb +38 -0
  88. data/test/test_main.rb +19 -0
  89. data/test/test_mmsgpack.rb +27 -0
  90. data/test/test_msnmp.rb +147 -0
  91. data/test/test_page.rb +51 -0
  92. data/test/test_pcollector.rb +71 -0
  93. data/test/test_plugin.rb +62 -0
  94. data/test/test_plugin_cisco.rb +23 -0
  95. data/test/test_polling_unit.rb +58 -0
  96. data/test/test_request.rb +26 -0
  97. data/test/test_rrd.rb +53 -0
  98. data/test/test_rrd_updater.rb +139 -0
  99. data/test/test_scheduler.rb +31 -0
  100. data/test/test_tra_collector.rb +40 -0
  101. data/test/test_trad.rb +33 -0
  102. data/test/test_util_marshal.rb +17 -0
  103. data/test/test_utils.rb +15 -0
  104. data/test/test_vendor.rb +40 -0
  105. data/test/test_writer.rb +33 -0
  106. data/test/unittest_helper.rb +27 -0
  107. metadata +208 -0
@@ -0,0 +1,112 @@
1
+ require 'erb'
2
+ require 'rack/utils'
3
+
4
+ module GRI
5
+ module FormatHelper
6
+ HTML_ESCAPE = {'&' => '&amp;', '>' => '&gt;', '<' => '&lt;',
7
+ '"' => '&quot;', "'" => '&#39;'}
8
+ HTML_ESCAPE_ONCE_REGEXP = /["><']|&(?!([a-zA-Z]+|(#\d+));)/
9
+
10
+ def h obj
11
+ case obj
12
+ when String
13
+ Rack::Utils.escape_html obj
14
+ else
15
+ Rack::Utils.escape_html obj.inspect
16
+ end
17
+ end
18
+
19
+ def escape_once s
20
+ s.to_s.gsub(HTML_ESCAPE_ONCE_REGEXP) {HTML_ESCAPE[$&]}
21
+ end
22
+
23
+ def u str
24
+ Rack::Utils.escape str
25
+ end
26
+
27
+ def url_to arg, q={}
28
+ script_name = ENV['SCRIPT_NAME'] || ''
29
+ if String === arg
30
+ (arg[0] == ??) ? script_name + arg :
31
+ (script_name + '/' + arg + mk_query(q))
32
+ else
33
+ ''
34
+ end
35
+ end
36
+
37
+ def mk_query h
38
+ return '' unless Hash === h and !h.empty?
39
+ '?' + h.map {|k, v| v && "#{k}=#{u v}"}.compact.join('&')
40
+ end
41
+
42
+ def td arg, options={}
43
+ tag = options[:head] ? 'th' : 'td'
44
+ args = (Array === arg) ? arg : [arg]
45
+ res = []
46
+ for arg in args
47
+ s = [:class, :colspan].map {|s|
48
+ options[s] ? "#{s}=#{options[s]}" : nil}.compact.join ' '
49
+ res.push "<#{tag}#{s.empty? ? '' : ' '+s}>#{arg}</#{tag}>"
50
+ end
51
+ res.join ''
52
+ end
53
+
54
+ def mk_tag tag, attrs, body=nil
55
+ "<#{tag}" + attrs.map {|k, v|
56
+ v ? ((v == true) ? ' ' + k : " #{k}=\"#{h v}\"") : nil
57
+ }.join('') + (body ? ">#{body}</#{tag}>" : "/>")
58
+ end
59
+
60
+ def text_field name='', value=nil, size=40, maxlength=nil, cl='form-control'
61
+ attrs = {'type'=>'text', 'name'=>name, 'value'=>value, 'size'=>size.to_s,
62
+ 'class'=>cl}
63
+ attrs['maxlength'] = maxlength.to_s if maxlength
64
+ mk_tag 'input', attrs
65
+ end
66
+
67
+ def hidden name, value
68
+ mk_tag 'input', [['name', name], ['value', value], ['type', 'hidden']]
69
+ end
70
+
71
+ def radio_button name='', value=nil, checked=nil
72
+ mk_tag 'input', 'type'=>'radio',
73
+ 'name'=>name, 'value'=>value, 'checked'=>checked
74
+ end
75
+
76
+ def check_box name='', value=nil, checked=nil
77
+ mk_tag 'input', 'type'=>'checkbox',
78
+ 'name'=>name, 'value'=>value, 'checked'=>checked
79
+ end
80
+
81
+ def popup_menu name, cl, *ary
82
+ body = ary.map {|value, s, selected_p|
83
+ attrs = [['value', value]]
84
+ attrs.push ['selected', true] if selected_p
85
+ mk_tag 'option', attrs, s
86
+ }.join ''
87
+ mk_tag 'select', [['name', name], ['class', cl]], body
88
+ end
89
+
90
+ def render template, b=nil
91
+ ERB.new(template, nil, '-').result(b || binding)
92
+ end
93
+
94
+ def to_scalestr v, base=1000
95
+ if v == nil or base == nil or base == 0
96
+ return(v.to_i == v ? v.to_i : v)
97
+ end
98
+ v = v.to_f
99
+ if v >= base ** 4
100
+ "%gT" % (v / (base ** 4))
101
+ elsif v >= base ** 3
102
+ "%gG" % (v / (base ** 3))
103
+ elsif v >= base ** 2
104
+ "%gM" % (v / (base ** 2))
105
+ elsif v >= base
106
+ "%gK" % (v / base)
107
+ else
108
+ v.to_s
109
+ end
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,43 @@
1
+ module GRI
2
+ class GParams
3
+ include Enumerable
4
+
5
+ def initialize
6
+ @params = {}
7
+ end
8
+
9
+ def setvar key, value
10
+ @params[key] = value
11
+ end
12
+
13
+ def getvar key
14
+ @params[key]
15
+ end
16
+
17
+ def []=(key, value)
18
+ (@params[key] ||= []).push value
19
+ end
20
+
21
+ def [](key)
22
+ (Array === (v = @params[key])) ? v.last : v
23
+ end
24
+
25
+ def each
26
+ @params.each {|k, v| yield k, v}
27
+ end
28
+
29
+ def update hash
30
+ hash.each {|key, value|
31
+ @params[key] = [value]
32
+ }
33
+ end
34
+
35
+ def merge cgi_params
36
+ for key, values in cgi_params
37
+ for value in values
38
+ self[key] = value
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
data/lib/gri/graph.rb ADDED
@@ -0,0 +1,239 @@
1
+ require 'gri/builtindefs'
2
+ require 'gri/blank'
3
+ require 'gri/request'
4
+ require 'gri/utils'
5
+ require 'gri/rrd'
6
+
7
+ module GRI
8
+ class Graph
9
+ include Utils
10
+
11
+ SIZE = {'ss'=>[145, 80], 's'=>[400, 80], 'l'=>[640, 300],
12
+ 'll'=>[860, 500], 'xl'=>[1200,700], 'gf'=>[387, 110]}
13
+ SIZE.default = [560, 120]
14
+ SCOLORS = ['3366ff', 'ff6666', '00cc00', 'ffccff', 'ff0000', 'ffff00',
15
+ '00ff00', '009900', '33ccff', '990000', 'cc99ff', '330099', 'ff33ff',
16
+ '33ffff', 'ffcc33', '33cc99', 'cccccc', '9999ff', '000000', 'eeee66',
17
+ '3f3fff', '3faf3f', 'ff3f3f', 'ffaf3f', '990033', '003399', '3fff3f',
18
+ 'ffff3f', '3f667f', '00cc00', 'ff9966', '3366cc', '666666', '99ff99']
19
+
20
+ def initialize options={}
21
+ @options = options
22
+ @hprops = {}
23
+ @hosthash = {}
24
+ end
25
+
26
+ def get_rrdpaths_and_ub rnames
27
+ total_ub = 0
28
+ rrdpaths = []
29
+ dirs = @options[:dirs]
30
+ for rname in rnames
31
+ host, key = rname.split('_', 2)
32
+ @hosthash[host] = true
33
+ dir, h = search_records dirs, host
34
+ if h
35
+ @hprops[host] ||= h
36
+ if h[key]
37
+ prop = get_prop h[key]
38
+ total_ub += prop[:ub].to_i
39
+ end
40
+ path = "#{dir}/#{host}/#{Rack::Utils.escape rname}.rrd"
41
+ rrdpaths.push path if File.exist? path
42
+ end
43
+ end
44
+ return rrdpaths, total_ub
45
+ end
46
+
47
+ def mk_graph_title rs, params
48
+ if params['p'] == 't'
49
+ r, = rs
50
+ host, key = r.split('_', 2)
51
+ dir, h = search_records @options[:dirs], host
52
+ if h
53
+ @hprops[host] ||= h
54
+ if h[key]
55
+ prop = get_prop h[key]
56
+ t = %Q{"#{host} #{prop[:name]} #{prop[:description]}"}
57
+ end
58
+ end
59
+ elsif (params['p'] == 's' or params['p'] == 'v') and
60
+ (params['ds'] and params['ds'] != '')
61
+ t = params['ds']
62
+ end
63
+ t
64
+ end
65
+
66
+ def mk_label path
67
+ dir = File.dirname path
68
+ label = File.basename path, '.rrd'
69
+ host, key = label.split('_', 2)
70
+ h = (@hprops[host] ||= (load_records "#{dir}/#{host}"))
71
+ if h and h[key]
72
+ prop = get_prop h[key]
73
+ name = prop[:legend] || prop[:name] || key
74
+ label = (@hosthash.size > 1) ? "#{host} #{name}" : name
75
+ end
76
+ label
77
+ end
78
+
79
+ def mk_graph_args specs, rrdpaths, params
80
+ defs = []
81
+
82
+ gidx = params['g'].to_i
83
+ gr_spec = (specs[:graph] || [])[gidx]
84
+ vlabel, base, limit, pickup_re, opts = gr_spec
85
+ defs.push %Q{-v "#{vlabel.gsub(/\"/, '')}"} if vlabel
86
+ if Numeric === base
87
+ defs.push(base.zero? ? '--units-exponent 0 --alt-y-grid' :
88
+ "--base #{base}")
89
+ end
90
+ defs.push "--lower-limit 0"
91
+
92
+ ds_specs = specs[:ds] || []
93
+ ds_specs = ds_specs.grep pickup_re if pickup_re
94
+ defs += mk_defstr ds_specs, rrdpaths, params, limit
95
+ defs.flatten
96
+ end
97
+
98
+ def mk_defstr ds_specs, rrdpaths, params, limit
99
+ mdefs = []
100
+ ds_param = params['ds'].blank? ? nil : params['ds']
101
+ if limit and limit.size == 2
102
+ lower, upper = limit
103
+ end
104
+ x_p = (params['fmt'] == 'json')
105
+ for ds_spec in ds_specs
106
+ mname, dsname, dst, cf, gline, gcolor, legend, mag = ds_spec.split(',')
107
+ next if gline == '-' or (ds_param and ds_param != dsname)
108
+ cf = 'MAX' if cf.blank?
109
+ gline = 'LINE1' if gline.blank?
110
+ gcolor = '#0000ff' if gcolor.blank?
111
+ legend = dsname if legend.blank?
112
+ mag ||= 1
113
+ gattr = [gline, gcolor, legend, params['cl'], params['hw']]
114
+ sdefs = mk_comp_defs(rrdpaths, params['p'], gattr) {
115
+ |paths, xcf, ind, gattr|
116
+ xgline, xgcolor, xlegend = gattr
117
+ xgline = 'XPORT' if x_p
118
+ RRD.defstr paths, dsname, cf, "#{dsname}#{ind}",
119
+ xgline, xgcolor, xlegend, mag, lower, upper
120
+ }
121
+ mdefs.push sdefs
122
+ end
123
+ mdefs
124
+ end
125
+
126
+ def mk_comp_defs rrdpaths, comp_t, gattr, &block
127
+ case comp_t
128
+ when 's'; mk_stack_defs rrdpaths, gattr, &block
129
+ when 'v'; mk_ov_defs rrdpaths, gattr, &block
130
+ else mk_sum_defs rrdpaths, gattr, &block
131
+ end
132
+ end
133
+
134
+ def mk_sum_defs rrdpaths, gattr, &block
135
+ if (legend = gattr[2]) == ':index'
136
+ path, = rrdpaths
137
+ host, dn, gattr[2] = File.basename(path, '.rrd').split('_', 3)
138
+ end
139
+ block.call(rrdpaths, 'MAX', 0, gattr)
140
+ end
141
+
142
+ def mk_stack_defs rrdpaths, gattr, &block
143
+ defs = []
144
+ gline = 'AREA'
145
+ rrdpaths.each_with_index {|path, ind|
146
+ gcolor = '#' + SCOLORS[ind % SCOLORS.size]
147
+ legend = mk_label path
148
+ gattr = [gline, gcolor, legend]
149
+ defs += [block.call([path], 'MAX', ind, gattr)]
150
+ gline = 'STACK'
151
+ }
152
+ defs
153
+ end
154
+
155
+ def mk_ov_defs rrdpaths, gattr, &block
156
+ defs = []
157
+ rrdpaths.each_with_index {|path, ind|
158
+ gcolor = '#' + SCOLORS[ind % SCOLORS.size]
159
+ legend = mk_label path
160
+ gattr = ['LINE1', gcolor, legend]
161
+ defs += [block.call([path], 'MAX', ind, gattr)]
162
+ }
163
+ defs
164
+ end
165
+
166
+ def call env
167
+ req = GRI::Request.new env
168
+ params = req.gparams
169
+ ENV['TZ'] = params['tz'].to_s unless params['tz'].blank?
170
+ stime, etime = req.params['stime'].to_i, req.params['etime'].to_i
171
+ etime = (Time.now + etime).to_i if etime <= 0
172
+ stime = (etime + stime).to_i if stime <= 0
173
+
174
+ if req.params['fmt'] == 'json'
175
+ str = xport stime, etime, params
176
+ [200, {'Content-type'=>'text/plain'}, [str]]
177
+ else
178
+ img = graph stime, etime, params
179
+ [200, {'Content-type'=>'image/png'}, [img]]
180
+ end
181
+ rescue
182
+ Log.error "#{$!}: #{$@.first}"
183
+ [500, {}, []]
184
+ end
185
+
186
+ def xport stime, etime, params
187
+ rnames = params.getvar 'r'
188
+ r = rnames.first
189
+ host, data_name, index = parse_host_key r
190
+ rrdpaths, total_ub = get_rrdpaths_and_ub rnames
191
+
192
+ specs = DEFS.get_specs data_name
193
+ gidx = params['g'].to_i
194
+ gr_spec = (specs[:graph] || [])[gidx]
195
+ vlabel, base, limit, pickup_re, opts = gr_spec
196
+
197
+ ds_specs = specs[:ds] || []
198
+ ds_specs = ds_specs.grep pickup_re if pickup_re
199
+ args = []
200
+ if params['maxrows'].present?
201
+ args.push "--maxrows #{params['maxrows'].to_i}"
202
+ end
203
+ args += mk_defstr(ds_specs, rrdpaths, params, limit).flatten
204
+ rrd = RRD.new Config['rrdcached-address']
205
+ str = rrd.xport stime, etime, *args
206
+ end
207
+
208
+ def graph stime, etime, params
209
+ rnames = params.getvar 'r'
210
+ r = rnames.first
211
+ host, data_name, index = parse_host_key r
212
+ rrdpaths, total_ub = get_rrdpaths_and_ub rnames
213
+
214
+ args = []
215
+ if total_ub > 0
216
+ args.push "-u #{total_ub} --alt-autoscale-max" if params['y'] == 'u'
217
+ args.push "HRULE:#{total_ub}#ff0000"
218
+ end
219
+ if params['title']
220
+ args.push "--title \"#{params['title']}\""
221
+ elsif (title = mk_graph_title rnames, params)
222
+ args.push "--title #{title}"
223
+ end
224
+
225
+ specs = DEFS.get_specs data_name
226
+ args += mk_graph_args specs, rrdpaths, params
227
+
228
+ rrd = RRD.new Config['rrdcached-address']
229
+ # size
230
+ rrd.width, rrd.height = SIZE[params['z']]
231
+ # font
232
+ if Config['font']
233
+ Config.getvar('font').each {|fstr| rrd.fonts.push fstr}
234
+ end
235
+
236
+ img = rrd.graphgen stime, etime, args
237
+ end
238
+ end
239
+ end
@@ -0,0 +1,70 @@
1
+ require 'gri/builtindefs'
2
+ require 'gri/config'
3
+ require 'gri/log'
4
+ require 'gri/utils'
5
+ require 'gri/wmain'
6
+ require 'gri/request'
7
+ require 'gri/list'
8
+ require 'gri/ds_list'
9
+ require 'gri/page'
10
+ require 'gri/graph'
11
+
12
+ module GRI
13
+ class Grapher
14
+ def initialize
15
+ root_dir = Config['root-dir'] ||= Config::ROOT_PATH
16
+ log_dir = Config['log-dir'] || Config['root-dir'] + '/log'
17
+ Log.init "#{log_dir}/#{File.basename $0}.log"
18
+ rescue SystemCallError
19
+ Log.init '/tmp/grapher.log'
20
+ end
21
+
22
+ def call env
23
+ req = GRI::Request.new env
24
+ params = req.params
25
+
26
+ gra_dirs = Config.getvar('gra-dir') || [Config::ROOT_PATH + '/gra']
27
+ if (req.query_string =~ /\A(\d+),(\d+)\z/)
28
+ app = Page.new :dirs=>gra_dirs, :clicked=>true, :imgx=>$1, :imgy=>$2
29
+ elsif params['r'] or params['tag']
30
+ if params['stime']
31
+ app = Graph.new :dirs=>gra_dirs
32
+ else
33
+ app = Page.new :dirs=>gra_dirs
34
+ end
35
+ elsif req.path_info =~ %r{^/([-\w][-#\.\w]*)}
36
+ app = DSList.new :dirs=>gra_dirs
37
+ else
38
+ app = List.new :dirs=>gra_dirs, :list_format=>Config['list-format'],
39
+ :use_regexp_search=>Config['use-regexp-search']
40
+ end
41
+ app.call env
42
+ end
43
+
44
+ def public_dir
45
+ File.dirname(__FILE__) + '/../../public'
46
+ end
47
+
48
+ def self.layout
49
+ <<EOS
50
+ <html>
51
+ <head>
52
+ <title><%= @title %></title>
53
+ <style>
54
+ td.text-right {text-align:right;}
55
+ span.large {font-size: x-large;}
56
+ table.ds td {padding:0;background:#f9f9f9;}
57
+ table.ds th {background:#ffd0d0;
58
+ background:linear-gradient(to bottom, #ffd8d8 0%,#ffcccc 45%,#ffc0c0 100%);
59
+ text-align:left;}
60
+ hr {border:none;border-top:1px #cccccc solid;}
61
+ </style>
62
+ </head>
63
+ <body>
64
+ <%= yield %>
65
+ </body>
66
+ </html>
67
+ EOS
68
+ end
69
+ end
70
+ end