log_sense 1.3.5 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.org +46 -0
- data/Gemfile.lock +4 -4
- data/README.org +24 -10
- data/Rakefile +17 -3
- data/exe/log_sense +24 -16
- data/ip_locations/dbip-country-lite.sqlite3 +0 -0
- data/lib/log_sense/apache_data_cruncher.rb +30 -30
- data/lib/log_sense/apache_log_line_parser.rb +12 -13
- data/lib/log_sense/apache_log_parser.rb +44 -36
- data/lib/log_sense/emitter.rb +518 -15
- data/lib/log_sense/ip_locator.rb +26 -19
- data/lib/log_sense/options_parser.rb +35 -30
- data/lib/log_sense/rails_data_cruncher.rb +8 -4
- data/lib/log_sense/rails_log_parser.rb +108 -100
- data/lib/log_sense/templates/_command_invocation.html.erb +0 -4
- data/lib/log_sense/templates/_command_invocation.txt.erb +4 -3
- data/lib/log_sense/templates/_navigation.html.erb +21 -0
- data/lib/log_sense/templates/_output_table.html.erb +2 -7
- data/lib/log_sense/templates/_output_table.txt.erb +14 -0
- data/lib/log_sense/templates/_performance.html.erb +1 -1
- data/lib/log_sense/templates/_performance.txt.erb +8 -5
- data/lib/log_sense/templates/_report_data.html.erb +2 -2
- data/lib/log_sense/templates/_summary.html.erb +6 -1
- data/lib/log_sense/templates/_summary.txt.erb +11 -8
- data/lib/log_sense/templates/_warning.txt.erb +1 -0
- data/lib/log_sense/templates/apache.html.erb +14 -335
- data/lib/log_sense/templates/apache.txt.erb +22 -0
- data/lib/log_sense/templates/rails.html.erb +13 -174
- data/lib/log_sense/templates/rails.txt.erb +10 -60
- data/lib/log_sense/version.rb +1 -1
- metadata +6 -2
data/lib/log_sense/emitter.rb
CHANGED
|
@@ -1,30 +1,29 @@
|
|
|
1
|
+
# coding: utf-8
|
|
1
2
|
require 'terminal-table'
|
|
2
3
|
require 'json'
|
|
3
4
|
require 'erb'
|
|
4
5
|
require 'ostruct'
|
|
5
|
-
|
|
6
6
|
module LogSense
|
|
7
|
+
#
|
|
8
|
+
# Emit Data
|
|
9
|
+
#
|
|
7
10
|
module Emitter
|
|
8
|
-
|
|
9
|
-
#
|
|
10
|
-
# Emit Data
|
|
11
|
-
#
|
|
12
11
|
def self.emit data = {}, options = {}
|
|
13
|
-
@input_format = options[:input_format] ||
|
|
14
|
-
@output_format = options[:output_format] ||
|
|
12
|
+
@input_format = options[:input_format] || 'apache'
|
|
13
|
+
@output_format = options[:output_format] || 'html'
|
|
15
14
|
|
|
16
15
|
# for the ERB binding
|
|
16
|
+
@reports = method("#{@input_format}_report_specification".to_sym).call(data)
|
|
17
17
|
@data = data
|
|
18
18
|
@options = options
|
|
19
19
|
|
|
20
20
|
# determine the main template to read
|
|
21
|
-
@template = File.join(File.dirname(__FILE__),
|
|
21
|
+
@template = File.join(File.dirname(__FILE__), 'templates', "#{@input_format}.#{@output_format}.erb")
|
|
22
22
|
erb_template = File.read @template
|
|
23
|
-
|
|
24
23
|
output = ERB.new(erb_template).result(binding)
|
|
25
24
|
|
|
26
25
|
if options[:output_file]
|
|
27
|
-
file = File.open options[:output_file],
|
|
26
|
+
file = File.open options[:output_file], 'w'
|
|
28
27
|
file.write output
|
|
29
28
|
file.close
|
|
30
29
|
else
|
|
@@ -32,20 +31,524 @@ module LogSense
|
|
|
32
31
|
end
|
|
33
32
|
end
|
|
34
33
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
def self.render(template, vars)
|
|
38
|
-
@template = File.join(File.dirname(__FILE__), "templates", "_#{template}")
|
|
34
|
+
def self.render(template, vars = {})
|
|
35
|
+
@template = File.join(File.dirname(__FILE__), 'templates', "_#{template}")
|
|
39
36
|
erb_template = File.read @template
|
|
40
37
|
ERB.new(erb_template).result(OpenStruct.new(vars).instance_eval { binding })
|
|
41
38
|
end
|
|
42
39
|
|
|
43
40
|
def self.escape_javascript(string)
|
|
44
|
-
js_escape_map = {
|
|
41
|
+
js_escape_map = {
|
|
42
|
+
'<' => '<',
|
|
43
|
+
'</' => '</',
|
|
44
|
+
'\r\n' => '\\r\\n',
|
|
45
|
+
'\n' => '\\n',
|
|
46
|
+
'\r' => '\\r',
|
|
47
|
+
'\\' => ' \\\\',
|
|
48
|
+
'"' => ' \\"',
|
|
49
|
+
"'" => " \\'",
|
|
50
|
+
'`' => ' \\`',
|
|
51
|
+
'$' => ' \\$'
|
|
52
|
+
}
|
|
45
53
|
js_escape_map.each do |k, v|
|
|
46
54
|
string = string.gsub(k, v)
|
|
47
55
|
end
|
|
48
56
|
string
|
|
49
57
|
end
|
|
58
|
+
|
|
59
|
+
def self.slugify(string)
|
|
60
|
+
(string.start_with?(/[0-9]/) ? 'slug-' : '') + string.downcase.gsub(' ', '-')
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def self.process(value)
|
|
64
|
+
klass = value.class
|
|
65
|
+
[Integer, Float].include?(klass) ? value : escape_javascript(value || '')
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# limit width of special columns, that is, URL, Path, and Description
|
|
69
|
+
# - data: array of arrays
|
|
70
|
+
# - heading: array with column names
|
|
71
|
+
# - width width to set
|
|
72
|
+
def self.shorten(data, heading, width)
|
|
73
|
+
# indexes of columns which have to be shortened
|
|
74
|
+
keywords = %w[URL Referers Description Path]
|
|
75
|
+
to_shorten = keywords.map { |x| heading.index x }.compact
|
|
76
|
+
|
|
77
|
+
if width.nil? || to_shorten.empty? || data[0].nil?
|
|
78
|
+
data
|
|
79
|
+
else
|
|
80
|
+
table_columns = data[0].size
|
|
81
|
+
data.map { |x|
|
|
82
|
+
(0..table_columns - 1).each.map { |col|
|
|
83
|
+
should_shorten = x[col] && x[col].size > width - 3 && to_shorten.include?(col)
|
|
84
|
+
should_shorten ? "#{x[col][0..(width - 3)]}..." : x[col]
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
#
|
|
91
|
+
# Specification of the reports to generate
|
|
92
|
+
# Array of hashes with the following information:
|
|
93
|
+
# - title: report_title
|
|
94
|
+
# header: header of tabular data
|
|
95
|
+
# rows: data to show
|
|
96
|
+
# column_alignment: specification of column alignments (works for txt reports)
|
|
97
|
+
# vega_spec: specifications for Vega output
|
|
98
|
+
# datatable_options: specific options for datatable
|
|
99
|
+
def self.apache_report_specification(data = {})
|
|
100
|
+
[
|
|
101
|
+
{ title: 'Daily Distribution',
|
|
102
|
+
header: %w[Day DOW Hits Visits Size],
|
|
103
|
+
column_alignment: %i[left left right right right],
|
|
104
|
+
rows: data[:daily_distribution],
|
|
105
|
+
vega_spec: {
|
|
106
|
+
'layer': [
|
|
107
|
+
{
|
|
108
|
+
'mark': {
|
|
109
|
+
'type': 'line',
|
|
110
|
+
'point': {
|
|
111
|
+
'filled': false,
|
|
112
|
+
'fill': 'white'
|
|
113
|
+
}
|
|
114
|
+
},
|
|
115
|
+
'encoding': {
|
|
116
|
+
'y': {'field': 'Hits', 'type': 'quantitative'}
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
{
|
|
120
|
+
'mark': {
|
|
121
|
+
'type': 'text',
|
|
122
|
+
'color': '#3E5772',
|
|
123
|
+
'align': 'middle',
|
|
124
|
+
'baseline': 'top',
|
|
125
|
+
'dx': -10,
|
|
126
|
+
'yOffset': -15
|
|
127
|
+
},
|
|
128
|
+
'encoding': {
|
|
129
|
+
'text': {'field': 'Hits', 'type': 'quantitative'},
|
|
130
|
+
'y': {'field': 'Hits', 'type': 'quantitative'}
|
|
131
|
+
}
|
|
132
|
+
},
|
|
133
|
+
|
|
134
|
+
{
|
|
135
|
+
'mark': {
|
|
136
|
+
'type': 'line',
|
|
137
|
+
'color': '#A52A2A',
|
|
138
|
+
'point': {
|
|
139
|
+
'color': '#A52A2A',
|
|
140
|
+
'filled': false,
|
|
141
|
+
'fill': 'white',
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
'encoding': {
|
|
145
|
+
'y': {'field': 'Visits', 'type': 'quantitative'}
|
|
146
|
+
}
|
|
147
|
+
},
|
|
148
|
+
|
|
149
|
+
{
|
|
150
|
+
'mark': {
|
|
151
|
+
'type': 'text',
|
|
152
|
+
'color': '#A52A2A',
|
|
153
|
+
'align': 'middle',
|
|
154
|
+
'baseline': 'top',
|
|
155
|
+
'dx': -10,
|
|
156
|
+
'yOffset': -15
|
|
157
|
+
},
|
|
158
|
+
'encoding': {
|
|
159
|
+
'text': {'field': 'Visits', 'type': 'quantitative'},
|
|
160
|
+
'y': {'field': 'Visits', 'type': 'quantitative'}
|
|
161
|
+
}
|
|
162
|
+
},
|
|
163
|
+
|
|
164
|
+
],
|
|
165
|
+
'encoding': {
|
|
166
|
+
'x': {'field': 'Day', 'type': 'temporal'},
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
},
|
|
171
|
+
{ title: 'Time Distribution',
|
|
172
|
+
header: %w[Hour Hits Visits Size],
|
|
173
|
+
column_alignment: %i[left right right right],
|
|
174
|
+
rows: data[:time_distribution],
|
|
175
|
+
vega_spec: {
|
|
176
|
+
'layer': [
|
|
177
|
+
{
|
|
178
|
+
'mark': 'bar'
|
|
179
|
+
},
|
|
180
|
+
{
|
|
181
|
+
'mark': {
|
|
182
|
+
'type': 'text',
|
|
183
|
+
'align': 'middle',
|
|
184
|
+
'baseline': 'top',
|
|
185
|
+
'dx': -10,
|
|
186
|
+
'yOffset': -15
|
|
187
|
+
},
|
|
188
|
+
'encoding': {
|
|
189
|
+
'text': {'field': 'Hits', 'type': 'quantitative'},
|
|
190
|
+
'y': {'field': 'Hits', 'type': 'quantitative'}
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
],
|
|
194
|
+
'encoding': {
|
|
195
|
+
'x': {'field': 'Hour', 'type': 'nominal'},
|
|
196
|
+
'y': {'field': 'Hits', 'type': 'quantitative'}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
},
|
|
200
|
+
{
|
|
201
|
+
title: '20_ and 30_ on HTML pages',
|
|
202
|
+
header: %w[Path Hits Visits Size Status],
|
|
203
|
+
column_alignment: %i[left right right right right],
|
|
204
|
+
rows: data[:most_requested_pages],
|
|
205
|
+
datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 } ]'
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
title: '20_ and 30_ on other resources',
|
|
209
|
+
header: %w[Path Hits Visits Size Status],
|
|
210
|
+
column_alignment: %i[left right right right right],
|
|
211
|
+
rows: data[:most_requested_resources],
|
|
212
|
+
datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 } ]'
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
title: '40_ and 50_x on HTML pages',
|
|
216
|
+
header: %w[Path Hits Visits Status],
|
|
217
|
+
column_alignment: %i[left right right right],
|
|
218
|
+
rows: data[:missed_pages],
|
|
219
|
+
datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 } ]'
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
title: '40_ and 50_ on other resources',
|
|
223
|
+
header: %w[Path Hits Visits Status],
|
|
224
|
+
column_alignment: %i[left right right right],
|
|
225
|
+
rows: data[:missed_resources],
|
|
226
|
+
datatable_options: 'columnDefs: [{ width: \'40%\', targets: 0 } ]'
|
|
227
|
+
},
|
|
228
|
+
{
|
|
229
|
+
title: 'Statuses',
|
|
230
|
+
header: %w[Status Count],
|
|
231
|
+
column_alignment: %i[left right],
|
|
232
|
+
rows: data[:statuses],
|
|
233
|
+
vega_spec: {
|
|
234
|
+
'mark': 'bar',
|
|
235
|
+
'encoding': {
|
|
236
|
+
'x': {'field': 'Status', 'type': 'nominal'},
|
|
237
|
+
'y': {'field': 'Count', 'type': 'quantitative'}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
title: 'Daily Statuses',
|
|
243
|
+
header: %w[Date S_2xx S_3xx S_4xx],
|
|
244
|
+
column_alignment: %i[left right right right],
|
|
245
|
+
rows: data[:statuses_by_day],
|
|
246
|
+
vega_spec: {
|
|
247
|
+
'transform': [ {'fold': ['S_2xx', 'S_3xx', 'S_4xx' ] }],
|
|
248
|
+
'mark': 'bar',
|
|
249
|
+
'encoding': {
|
|
250
|
+
'x': {
|
|
251
|
+
'field': 'Date',
|
|
252
|
+
'type': 'ordinal',
|
|
253
|
+
'timeUnit': 'day',
|
|
254
|
+
},
|
|
255
|
+
'y': {
|
|
256
|
+
'aggregate': 'sum',
|
|
257
|
+
'field': 'value',
|
|
258
|
+
'type': 'quantitative'
|
|
259
|
+
},
|
|
260
|
+
'color': {
|
|
261
|
+
'field': 'key',
|
|
262
|
+
'type': 'nominal',
|
|
263
|
+
'scale': {
|
|
264
|
+
'domain': ['S_2xx', 'S_3xx', 'S_4xx'],
|
|
265
|
+
'range': ['#228b22', '#ff8c00', '#a52a2a']
|
|
266
|
+
},
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
},
|
|
271
|
+
{ title: 'Browsers',
|
|
272
|
+
header: %w[Browser Hits Visits Size],
|
|
273
|
+
column_alignment: %i[left right right right],
|
|
274
|
+
rows: data[:browsers],
|
|
275
|
+
vega_spec: {
|
|
276
|
+
'layer': [
|
|
277
|
+
{ 'mark': 'bar' },
|
|
278
|
+
{
|
|
279
|
+
'mark': {
|
|
280
|
+
'type': 'text',
|
|
281
|
+
'align': 'middle',
|
|
282
|
+
'baseline': 'top',
|
|
283
|
+
'dx': -10,
|
|
284
|
+
'yOffset': -15
|
|
285
|
+
},
|
|
286
|
+
'encoding': {
|
|
287
|
+
'text': {'field': 'Hits', 'type': 'quantitative'},
|
|
288
|
+
}
|
|
289
|
+
},
|
|
290
|
+
],
|
|
291
|
+
'encoding': {
|
|
292
|
+
'x': {'field': 'Browser', 'type': 'nominal'},
|
|
293
|
+
'y': {'field': 'Hits', 'type': 'quantitative'}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
},
|
|
297
|
+
{ title: 'Platforms',
|
|
298
|
+
header: %w[Platform Hits Visits Size],
|
|
299
|
+
column_alignment: %i[left right right right],
|
|
300
|
+
rows: data[:platforms],
|
|
301
|
+
vega_spec: {
|
|
302
|
+
'layer': [
|
|
303
|
+
{ 'mark': 'bar' },
|
|
304
|
+
{
|
|
305
|
+
'mark': {
|
|
306
|
+
'type': 'text',
|
|
307
|
+
'align': 'middle',
|
|
308
|
+
'baseline': 'top',
|
|
309
|
+
'dx': -10,
|
|
310
|
+
'yOffset': -15
|
|
311
|
+
},
|
|
312
|
+
'encoding': {
|
|
313
|
+
'text': {'field': 'Hits', 'type': 'quantitative'},
|
|
314
|
+
}
|
|
315
|
+
},
|
|
316
|
+
],
|
|
317
|
+
'encoding': {
|
|
318
|
+
'x': {'field': 'Platform', 'type': 'nominal'},
|
|
319
|
+
'y': {'field': 'Hits', 'type': 'quantitative'}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
},
|
|
323
|
+
{
|
|
324
|
+
title: 'IPs',
|
|
325
|
+
header: %w[IPs Hits Visits Size Country],
|
|
326
|
+
column_alignment: %i[left right right right left],
|
|
327
|
+
rows: data[:ips]
|
|
328
|
+
},
|
|
329
|
+
{
|
|
330
|
+
title: 'Countries',
|
|
331
|
+
header: %w[Country Hits Visits IPs],
|
|
332
|
+
column_alignment: %i[left right right left],
|
|
333
|
+
rows: data[:countries]&.map do |k, v|
|
|
334
|
+
[
|
|
335
|
+
k,
|
|
336
|
+
v.map { |x| x[1] }.inject(&:+),
|
|
337
|
+
v.map { |x| x[2] }.inject(&:+),
|
|
338
|
+
v.map { |x| x[0] }.join(' ')
|
|
339
|
+
]
|
|
340
|
+
end
|
|
341
|
+
},
|
|
342
|
+
{
|
|
343
|
+
title: 'Referers',
|
|
344
|
+
header: %w[Referers Hits Visits Size],
|
|
345
|
+
column_alignment: %i[left right right right],
|
|
346
|
+
rows: data[:referers],
|
|
347
|
+
col: 'small-12 cell'
|
|
348
|
+
},
|
|
349
|
+
{
|
|
350
|
+
title: 'Streaks',
|
|
351
|
+
report: :html,
|
|
352
|
+
header: ['IP', 'Date', 'Total HTML', 'Total Other', 'HTML', 'Other'],
|
|
353
|
+
column_alignment: %i[left left right right left left],
|
|
354
|
+
rows: data[:streaks]&.group_by { |x| [x[0], x[1]] }&.map do |k, v|
|
|
355
|
+
[
|
|
356
|
+
k[0],
|
|
357
|
+
k[1],
|
|
358
|
+
v.map { |x| x[2] }.compact.select { |x| x.match(/\.html?$/) }.size,
|
|
359
|
+
v.map { |x| x[2] }.compact.reject { |x| x.match(/\.html?$/) }.size,
|
|
360
|
+
v.map { |x| x[2] }.compact.select { |x| x.match(/\.html?$/) }.join(' ■ '),
|
|
361
|
+
v.map { |x| x[2] }.compact.reject { |x| x.match(/\.html?$/) }.join(' ■ ')
|
|
362
|
+
]
|
|
363
|
+
end,
|
|
364
|
+
col: 'small-12 cell'
|
|
365
|
+
}
|
|
366
|
+
]
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
def self.rails_report_specification(data = {})
|
|
370
|
+
[
|
|
371
|
+
{
|
|
372
|
+
title: "Daily Distribution",
|
|
373
|
+
header: %w[Day DOW Hits],
|
|
374
|
+
column_alignment: %i[left left right],
|
|
375
|
+
rows: data[:daily_distribution],
|
|
376
|
+
vega_spec: {
|
|
377
|
+
"encoding": {
|
|
378
|
+
"x": {"field": "Day", "type": "temporal"},
|
|
379
|
+
"y": {"field": "Hits", "type": "quantitative"}
|
|
380
|
+
},
|
|
381
|
+
"layer": [
|
|
382
|
+
{
|
|
383
|
+
"mark": {
|
|
384
|
+
"type": "line",
|
|
385
|
+
"point": {
|
|
386
|
+
"filled": false,
|
|
387
|
+
"fill": "white"
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
},
|
|
391
|
+
{
|
|
392
|
+
"mark": {
|
|
393
|
+
"type": "text",
|
|
394
|
+
"align": "left",
|
|
395
|
+
"baseline": "middle",
|
|
396
|
+
"dx": 5
|
|
397
|
+
},
|
|
398
|
+
"encoding": {
|
|
399
|
+
"text": {"field": "Hits", "type": "quantitative"}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
]
|
|
403
|
+
}
|
|
404
|
+
},
|
|
405
|
+
{
|
|
406
|
+
title: "Time Distribution",
|
|
407
|
+
header: %w[Hour Hits],
|
|
408
|
+
column_alignment: %i[left right],
|
|
409
|
+
rows: data[:time_distribution],
|
|
410
|
+
vega_spec: {
|
|
411
|
+
"layer": [
|
|
412
|
+
{
|
|
413
|
+
"mark": "bar",
|
|
414
|
+
},
|
|
415
|
+
{
|
|
416
|
+
"mark": {
|
|
417
|
+
"type": "text",
|
|
418
|
+
"align": "middle",
|
|
419
|
+
"baseline": "top",
|
|
420
|
+
"dx": -10,
|
|
421
|
+
"yOffset": -15
|
|
422
|
+
},
|
|
423
|
+
"encoding": {
|
|
424
|
+
"text": {"field": "Hits", "type": "quantitative"}
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
],
|
|
428
|
+
"encoding": {
|
|
429
|
+
"x": {"field": "Hour", "type": "nominal"},
|
|
430
|
+
"y": {"field": "Hits", "type": "quantitative"}
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
},
|
|
434
|
+
{
|
|
435
|
+
title: "Statuses",
|
|
436
|
+
header: %w[Status Count],
|
|
437
|
+
column_alignment: %i[left right],
|
|
438
|
+
rows: data[:statuses],
|
|
439
|
+
vega_spec: {
|
|
440
|
+
"layer": [
|
|
441
|
+
{
|
|
442
|
+
"mark": "bar"
|
|
443
|
+
},
|
|
444
|
+
{
|
|
445
|
+
"mark": {
|
|
446
|
+
"type": "text",
|
|
447
|
+
"align": "left",
|
|
448
|
+
"baseline": "top",
|
|
449
|
+
"dx": -10,
|
|
450
|
+
"yOffset": -20
|
|
451
|
+
},
|
|
452
|
+
"encoding": {
|
|
453
|
+
"text": {"field": "Count", "type": "quantitative"}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
],
|
|
457
|
+
"encoding": {
|
|
458
|
+
"x": {"field": "Status", "type": "nominal"},
|
|
459
|
+
"y": {"field": "Count", "type": "quantitative"}
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
title: "Rails Performance",
|
|
465
|
+
header: %w[Controller Hits Min Avg Max],
|
|
466
|
+
column_alignment: %i[left right right right right],
|
|
467
|
+
rows: data[:performance],
|
|
468
|
+
vega_spec: {
|
|
469
|
+
"layer": [
|
|
470
|
+
{
|
|
471
|
+
"mark": {
|
|
472
|
+
"type": "point",
|
|
473
|
+
"name": "data_points"
|
|
474
|
+
}
|
|
475
|
+
},
|
|
476
|
+
{
|
|
477
|
+
"mark": {
|
|
478
|
+
"name": "label",
|
|
479
|
+
"type": "text",
|
|
480
|
+
"align": "left",
|
|
481
|
+
"baseline": "middle",
|
|
482
|
+
"dx": 5,
|
|
483
|
+
"yOffset": 0
|
|
484
|
+
},
|
|
485
|
+
"encoding": {
|
|
486
|
+
"text": {"field": "Controller"},
|
|
487
|
+
"fontSize": {"value": 8}
|
|
488
|
+
},
|
|
489
|
+
},
|
|
490
|
+
],
|
|
491
|
+
"encoding": {
|
|
492
|
+
"x": {"field": "Avg", "type": "quantitative"},
|
|
493
|
+
"y": {"field": "Hits", "type": "quantitative"}
|
|
494
|
+
},
|
|
495
|
+
}
|
|
496
|
+
},
|
|
497
|
+
{
|
|
498
|
+
title: "Fatal Events",
|
|
499
|
+
header: %w[Date IP URL Description Log ID],
|
|
500
|
+
column_alignment: %i[left left left left left],
|
|
501
|
+
rows: data[:fatal],
|
|
502
|
+
col: 'small-12 cell'
|
|
503
|
+
},
|
|
504
|
+
{
|
|
505
|
+
title: 'Internal Server Errors',
|
|
506
|
+
header: %w[Date Status IP URL Description Log ID],
|
|
507
|
+
column_alignment: %i[left left left left left left],
|
|
508
|
+
rows: data[:internal_server_error],
|
|
509
|
+
col: 'small-12 cell'
|
|
510
|
+
},
|
|
511
|
+
{
|
|
512
|
+
title: 'Errors',
|
|
513
|
+
header: %w[Log ID Context Description Count],
|
|
514
|
+
column_alignment: %i[left left left left],
|
|
515
|
+
rows: data[:error],
|
|
516
|
+
col: 'small-12 cell'
|
|
517
|
+
},
|
|
518
|
+
{
|
|
519
|
+
title: 'IPs',
|
|
520
|
+
header: %w[IPs Hits Country],
|
|
521
|
+
column_alignment: %i[left right left],
|
|
522
|
+
rows: data[:ips]
|
|
523
|
+
},
|
|
524
|
+
{
|
|
525
|
+
title: 'Countries',
|
|
526
|
+
header: %w[Country Hits IPs],
|
|
527
|
+
column_alignment: %i[left right left],
|
|
528
|
+
rows: data[:countries]&.map do |k, v|
|
|
529
|
+
[
|
|
530
|
+
k,
|
|
531
|
+
v.map { |x| x[1] }.inject(&:+),
|
|
532
|
+
v.map { |x| x[0] }.join(' ■ ')
|
|
533
|
+
]
|
|
534
|
+
end
|
|
535
|
+
},
|
|
536
|
+
{
|
|
537
|
+
title: 'Streaks',
|
|
538
|
+
report: :html,
|
|
539
|
+
header: %w[IP Date Total Resources],
|
|
540
|
+
column_alignment: %i[left left right right left left],
|
|
541
|
+
rows: data[:streaks]&.group_by { |x| [x[0], x[1]] }&.map do |k, v|
|
|
542
|
+
[
|
|
543
|
+
k[0],
|
|
544
|
+
k[1],
|
|
545
|
+
v.size,
|
|
546
|
+
v.map { |x| x[2] }.join(' ■ ')
|
|
547
|
+
]
|
|
548
|
+
end,
|
|
549
|
+
col: 'small-12 cell'
|
|
550
|
+
}
|
|
551
|
+
]
|
|
552
|
+
end
|
|
50
553
|
end
|
|
51
554
|
end
|
data/lib/log_sense/ip_locator.rb
CHANGED
|
@@ -4,19 +4,22 @@ require 'ipaddr'
|
|
|
4
4
|
require 'iso_country_codes'
|
|
5
5
|
|
|
6
6
|
module LogSense
|
|
7
|
+
#
|
|
8
|
+
# Populate table of IP Locations from dbip-country-lite
|
|
9
|
+
#
|
|
7
10
|
module IpLocator
|
|
8
|
-
DB_FILE = File.join(File.dirname(__FILE__),
|
|
11
|
+
DB_FILE = File.join(File.dirname(__FILE__), '..', '..', 'ip_locations', 'dbip-country-lite.sqlite3')
|
|
9
12
|
|
|
10
|
-
def self.dbip_to_sqlite
|
|
11
|
-
db = SQLite3::Database.new
|
|
12
|
-
db.execute
|
|
13
|
+
def self.dbip_to_sqlite(db_location)
|
|
14
|
+
db = SQLite3::Database.new ':memory:'
|
|
15
|
+
db.execute 'CREATE TABLE ip_location (
|
|
13
16
|
from_ip_n INTEGER,
|
|
14
17
|
from_ip TEXT,
|
|
15
18
|
to_ip TEXT,
|
|
16
19
|
country_code TEXT
|
|
17
|
-
)
|
|
20
|
+
)'
|
|
18
21
|
|
|
19
|
-
ins = db.prepare
|
|
22
|
+
ins = db.prepare 'INSERT INTO ip_location(from_ip_n, from_ip, to_ip, country_code) values (?, ?, ?, ?)'
|
|
20
23
|
CSV.foreach(db_location) do |row|
|
|
21
24
|
ip = IPAddr.new row[0]
|
|
22
25
|
ins.execute(ip.to_i, row[0], row[1], row[2])
|
|
@@ -33,29 +36,33 @@ module LogSense
|
|
|
33
36
|
SQLite3::Database.new DB_FILE
|
|
34
37
|
end
|
|
35
38
|
|
|
36
|
-
def self.locate_ip
|
|
37
|
-
return
|
|
39
|
+
def self.locate_ip(ip, db)
|
|
40
|
+
return unless ip
|
|
38
41
|
|
|
39
|
-
|
|
40
|
-
res = db.execute "SELECT * FROM ip_location where from_ip_n <= #{ip_n} order by from_ip_n desc limit 1"
|
|
42
|
+
query = db.prepare 'SELECT * FROM ip_location where from_ip_n <= ? order by from_ip_n desc limit 1'
|
|
41
43
|
begin
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
44
|
+
ip_n = IPAddr.new(ip).to_i
|
|
45
|
+
result_set = query.execute ip_n
|
|
46
|
+
country_code = result_set.map { |x| x[3] }[0]
|
|
47
|
+
IsoCountryCodes.find(country_code).name
|
|
48
|
+
rescue IPAddr::InvalidAddressError
|
|
49
|
+
'INVALID IP'
|
|
50
|
+
rescue IsoCountryCodes::UnknownCodeError
|
|
51
|
+
country_code
|
|
45
52
|
end
|
|
46
53
|
end
|
|
47
54
|
|
|
48
55
|
#
|
|
49
56
|
# add country code to data[:ips]
|
|
50
57
|
#
|
|
51
|
-
def self.geolocate
|
|
52
|
-
@location_db = IpLocator
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
58
|
+
def self.geolocate(data)
|
|
59
|
+
@location_db = IpLocator.load_db
|
|
60
|
+
|
|
61
|
+
data[:ips].each do |line|
|
|
62
|
+
country_code = IpLocator.locate_ip line[0], @location_db
|
|
63
|
+
line << country_code
|
|
56
64
|
end
|
|
57
65
|
data
|
|
58
66
|
end
|
|
59
|
-
|
|
60
67
|
end
|
|
61
68
|
end
|