log_sense 1.2.1 → 1.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/lib/log_sense/apache_log_line_parser.rb +56 -0
  3. data/lib/log_sense/apache_log_parser.rb +14 -12
  4. data/lib/log_sense/rails_log_parser.rb +2 -2
  5. data/lib/log_sense/templates/_log_structure.html.erb +2 -2
  6. data/lib/log_sense/templates/apache.html.erb +113 -16
  7. data/lib/log_sense/templates/rails.html.erb +78 -18
  8. data/lib/log_sense/version.rb +1 -1
  9. data/lib/log_sense.rb +1 -0
  10. data/log_sense.gemspec +0 -1
  11. metadata +3 -16
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f9d29ac7d799d22088daaf7c0c5598030ab6c3290ef6e4dc2d81dc2b72e5209e
4
- data.tar.gz: 1fc1f04bdca1d271036fdfdbfa0b6160396bf12bdf303c9e3819375aaadf2b6f
3
+ metadata.gz: 7e05abbc9fc93aa9ccd057f3a01f6c93d966e964695d486e3425b3b4bc073c52
4
+ data.tar.gz: 7514a501cf9ac3766b457bca1d31a545a1331e35d5e8591aa81965994943580d
5
5
  SHA512:
6
- metadata.gz: df42cc6bc0a492926115ac9a2b9bbf9c2ed1c37a447f3e5c66d1d8bcc82fca674580607f6b95ce7588467a913725b67cad981508c2ba9d716d595e1a8d1c69fd
7
- data.tar.gz: '091a7e61f5d564f4e3ff1789ef89223339f42da0dd29090de024edcdfcc16e8a9bc0514915434213d806018560e0368fba6f25ef5aaef1a69f9819163f9ed399'
6
+ metadata.gz: 91cbf90786d0c6479b246b9df92f67fcae73491d3d622eb99c364a8d279aa1ca8aac009ba765c1b75734ca9d664d334bd4aed970c02b6e8fda8a3a15f8534c52
7
+ data.tar.gz: 9b41aa16dd595fe664f6a1ba90bbb196dcbc945c760f6ea6ddcb30569ff0767f70f06487790c3eda0f0d8cd35c30cba4bab2982a3825067025e30a5a5dcb56f3
data/lib/log_sense/apache_log_line_parser.rb ADDED
@@ -0,0 +1,56 @@
1
+ module LogSense
2
+ class ApacheLogLineParser
3
+ # parses a query and makes it into an expression which can be evaluated
4
+ # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined
5
+ #
6
+ # %h: IP
7
+ # %l: ident or -
8
+ # %u: userid or -
9
+ # %t: [10/Oct/2000:13:55:36 -0700]
10
+ # day = 2*digit
11
+ # month = 3*letter
12
+ # year = 4*digit
13
+ # hour = 2*digit
14
+ # minute = 2*digit
15
+ # second = 2*digit
16
+ # zone = (`+' | `-') 4*digit
17
+ # %r: GET /apache_pb.gif HTTP/1.0
18
+ # %{User-agent}: "
19
+ #
20
+ # 116.179.32.16 - - [19/Dec/2021:22:35:11 +0100] "GET / HTTP/1.1" 200 135 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
21
+
22
+ DAY = /[0-9]{2}/
23
+ MONTH = /[A-Za-z]{3}/
24
+ YEAR = /[0-9]{4}/
25
+ TIMEC = /[0-9]{2}/
26
+ TIMEZONE = /(\+|-)[0-9]{4}/
27
+
28
+ IP = /(?<ip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|::1)/
29
+ IDENT = /(?<ident>[^ ]+|-)/
30
+ USERID = /(?<userid>[^ ]+|-)/
31
+
32
+ TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
33
+
34
+ HTTP_METHODS=/GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH/
35
+ WEBDAV_METHODS=/COPY|LOCK|MKCOL|MOVE|PROPFIND|PROPPATCH|UNLOCK/
36
+ OTHER_METHODS=/SEARCH|REPORT/
37
+ METHOD=/(?<method>#{HTTP_METHODS}|#{WEBDAV_METHODS}|#{OTHER_METHODS})/
38
+ PROTOCOL=/(?<protocol>HTTP\/[0-9]\.[0-9])/
39
+ URL=/(?<url>[^ ]+)/
40
+ REFERER=/(?<referer>[^ ]+)/
41
+ RETURN_CODE=/(?<status>[1-5][0-9][0-9])/
42
+ SIZE=/(?<size>[0-9]+|-)/
43
+
44
+ USER_AGENT = /(?<user_agent>[^"]+)/
45
+
46
+ attr_reader :format
47
+
48
+ def initialize
49
+ @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "#{METHOD} #{URL} #{PROTOCOL}" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
50
+ end
51
+
52
+ def parse line
53
+ hash = @format.match(line) || raise("Apache LogLine Parser Error: Could not parse #{line}")
54
+ end
55
+ end
56
+ end
data/lib/log_sense/apache_log_parser.rb CHANGED
@@ -1,4 +1,3 @@
1
- require 'apache_log/parser'
2
1
  require 'sqlite3'
3
2
  require 'browser'
4
3
 
@@ -50,21 +49,20 @@ module LogSense
50
49
  platform_version)
51
50
  values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
52
51
 
53
- parser = ApacheLog::Parser.new(options[:format] || 'combined')
52
+ parser = ApacheLogLineParser.new
54
53
 
55
54
  content.each do |line|
56
55
  begin
57
56
  hash = parser.parse line
58
-
59
57
  ua = Browser.new(hash[:user_agent], accept_language: "en-us")
60
58
  ins.execute(
61
- hash[:datetime].iso8601,
62
- hash[:remote_host],
63
- hash[:user],
64
- hash[:datetime].strftime("%Y-%m-%d") + " " + hash[:remote_host] + " " + hash[:user_agent],
65
- hash[:request][:method],
66
- hash[:request][:path],
67
- (hash[:request][:path] ? File.extname(hash[:request][:path]) : ""),
59
+ DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
60
+ hash[:ip],
61
+ hash[:userid],
62
+ unique_visitor_id(hash),
63
+ hash[:method],
64
+ hash[:url],
65
+ (hash[:url] ? File.extname(hash[:url]) : ""),
68
66
  hash[:status],
69
67
  hash[:size].to_i,
70
68
  hash[:referer],
@@ -75,13 +73,17 @@ module LogSense
75
73
  (ua.platform.name || ""),
76
74
  (ua.platform.version || "")
77
75
  )
78
- rescue
79
- STDERR.puts "Apache Log parser error: could not parse #{line}"
76
+ rescue StandardError => e
77
+ STDERR.puts e.message
80
78
  end
81
79
  end
82
80
 
83
81
  db
84
82
  end
85
83
 
84
+ def self.unique_visitor_id hash
85
+ "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
86
+ end
87
+
86
88
  end
87
89
  end
data/lib/log_sense/rails_log_parser.rb CHANGED
@@ -185,8 +185,8 @@ module LogSense
185
185
  # [584cffcc-f1fd-4b5c-bb8b-b89621bd4921] ActionController::RoutingError (No route matches [GET] "/assets/foundation-icons.svg"):
186
186
  # [fd8df8b5-83c9-48b5-a056-e5026e31bd5e] ActionView::Template::Error (undefined method `all_my_ancestor' for nil:NilClass):
187
187
  # [d17ed55c-f5f1-442a-a9d6-3035ab91adf0] ActionView::Template::Error (undefined method `volunteer_for' for #<DonationsController:0x007f4864c564b8>
188
- CONTEXT = /(?<context>[^ ]+Error)/
189
- ERROR_REGEXP = /^\[#{ID}\] #{CONTEXT} \((?<description>.*)\):/
188
+ EXCEPTION = /[A-Za-z_0-9:]+(Error)?/
189
+ ERROR_REGEXP = /^\[#{ID}\] (?<context>#{EXCEPTION}) \((?<description>(#{EXCEPTION})?.*)\):/
190
190
 
191
191
  def self.match_and_process_error line
192
192
  matchdata = ERROR_REGEXP.match line
data/lib/log_sense/templates/_log_structure.html.erb CHANGED
@@ -14,11 +14,11 @@
14
14
  <%= data[:log_size] %> <span class="stats-list-label">Total Entries</span>
15
15
  </li>
16
16
  <li class="stats-list-negative">
17
- <%= data[:selfpolls_size] %> <span class="stats-list-label">Self Polls Entries</span>
17
+ <%= data[:selfpolls_size] %> <span class="stats-list-label">Self Polls</span>
18
18
  </li>
19
19
  <li class="stats-list-negative">
20
20
  <td><%= data[:crawlers_size] %></td>
21
- <span class="stats-list-label">Crawlers Entries</span>
21
+ <span class="stats-list-label">Crawlers</span>
22
22
  </li>
23
23
  </ul>
24
24
 
data/lib/log_sense/templates/apache.html.erb CHANGED
@@ -29,14 +29,14 @@
29
29
  }
30
30
 
31
31
  #offCanvas {
32
- color: white;
33
- background: #0D0630;
32
+ color: #DEDEDE;
33
+ background: #1C1C1C;
34
34
  border-right: none;
35
35
  box-shadow: none;
36
36
  padding: 0.5rem;
37
37
  }
38
38
  #offCanvas a {
39
- color: #E6F9AF;
39
+ color: #FFFFFF;
40
40
  }
41
41
 
42
42
  .contents-button {
@@ -69,7 +69,7 @@
69
69
 
70
70
  .card-divider {
71
71
  padding: 0.2rem 0.4rem 0.2rem 0.4rem;
72
- background: #0d0630;
72
+ background: #1C1C1C;
73
73
  color: white;
74
74
  }
75
75
 
@@ -224,24 +224,93 @@
224
224
  header: ["Day", "DOW", "Hits", "Visits", "Size"],
225
225
  rows: data[:daily_distribution],
226
226
  vega_spec: {
227
- "mark": {
228
- "type": "line",
229
- "point": {
230
- "filled": false,
231
- "fill": "white"
232
- }
233
- },
227
+ "layer": [
228
+ {
229
+ "mark": {
230
+ "type": "line",
231
+ "point": {
232
+ "filled": false,
233
+ "fill": "white"
234
+ }
235
+ },
236
+ "encoding": {
237
+ "y": {"field": "Hits", "type": "quantitative"}
238
+ }
239
+ },
240
+ {
241
+ "mark": {
242
+ "type": "text",
243
+ "color": "#3E5772",
244
+ "align": "middle",
245
+ "baseline": "top",
246
+ "dx": -10,
247
+ "yOffset": -15
248
+ },
249
+ "encoding": {
250
+ "text": {"field": "Hits", "type": "quantitative"},
251
+ "y": {"field": "Hits", "type": "quantitative"}
252
+ }
253
+ },
254
+
255
+ {
256
+ "mark": {
257
+ "type": "line",
258
+ "color": "#A52A2A",
259
+ "point": {
260
+ "color": "#A52A2A",
261
+ "filled": false,
262
+ "fill": "white",
263
+ }
264
+ },
265
+ "encoding": {
266
+ "y": {"field": "Visits", "type": "quantitative"}
267
+ }
268
+ },
269
+
270
+ {
271
+ "mark": {
272
+ "type": "text",
273
+ "color": "#A52A2A",
274
+ "align": "middle",
275
+ "baseline": "top",
276
+ "dx": -10,
277
+ "yOffset": -15
278
+ },
279
+ "encoding": {
280
+ "text": {"field": "Visits", "type": "quantitative"},
281
+ "y": {"field": "Visits", "type": "quantitative"}
282
+ }
283
+ },
284
+
285
+ ],
234
286
  "encoding": {
235
287
  "x": {"field": "Day", "type": "temporal"},
236
- "y": {"field": "Hits", "type": "quantitative"}
237
288
  }
238
289
  }
290
+
239
291
  },
240
292
  { title: "Time Distribution",
241
293
  header: ["Hour", "Hits", "Visits", "Size"],
242
294
  rows: data[:time_distribution],
243
295
  vega_spec: {
244
- "mark": "bar",
296
+ "layer": [
297
+ {
298
+ "mark": "bar"
299
+ },
300
+ {
301
+ "mark": {
302
+ "type": "text",
303
+ "align": "middle",
304
+ "baseline": "top",
305
+ "dx": -10,
306
+ "yOffset": -15
307
+ },
308
+ "encoding": {
309
+ "text": {"field": "Hits", "type": "quantitative"},
310
+ "y": {"field": "Hits", "type": "quantitative"}
311
+ }
312
+ },
313
+ ],
245
314
  "encoding": {
246
315
  "x": {"field": "Hour", "type": "nominal"},
247
316
  "y": {"field": "Hits", "type": "quantitative"}
@@ -299,7 +368,21 @@
299
368
  header: ["Browser", "Hits", "Visits", "Size"],
300
369
  rows: data[:browsers],
301
370
  vega_spec: {
302
- "mark": "bar",
371
+ "layer": [
372
+ { "mark": "bar" },
373
+ {
374
+ "mark": {
375
+ "type": "text",
376
+ "align": "middle",
377
+ "baseline": "top",
378
+ "dx": -10,
379
+ "yOffset": -15
380
+ },
381
+ "encoding": {
382
+ "text": {"field": "Hits", "type": "quantitative"},
383
+ }
384
+ },
385
+ ],
303
386
  "encoding": {
304
387
  "x": {"field": "Browser", "type": "nominal"},
305
388
  "y": {"field": "Hits", "type": "quantitative"}
@@ -310,7 +393,21 @@
310
393
  header: ["Platform", "Hits", "Visits", "Size"],
311
394
  rows: data[:platforms],
312
395
  vega_spec: {
313
- "mark": "bar",
396
+ "layer": [
397
+ { "mark": "bar" },
398
+ {
399
+ "mark": {
400
+ "type": "text",
401
+ "align": "middle",
402
+ "baseline": "top",
403
+ "dx": -10,
404
+ "yOffset": -15
405
+ },
406
+ "encoding": {
407
+ "text": {"field": "Hits", "type": "quantitative"},
408
+ }
409
+ },
410
+ ],
314
411
  "encoding": {
315
412
  "x": {"field": "Platform", "type": "nominal"},
316
413
  "y": {"field": "Hits", "type": "quantitative"}
@@ -325,7 +422,7 @@
325
422
  <% @reports.each_with_index do |report, index| %>
326
423
  <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
327
424
  <div class="card-divider">
328
- <h2>
425
+ <h2 id="<%= report[:title].downcase.gsub(' ', '-') %>">
329
426
  <%= report[:title] %>
330
427
  </h2>
331
428
  </div>
data/lib/log_sense/templates/rails.html.erb CHANGED
@@ -29,14 +29,14 @@
29
29
  }
30
30
 
31
31
  #offCanvas {
32
- color: white;
33
- background: #0D0630;
32
+ color: #CECECE;
33
+ background: #BD000D;
34
34
  border-right: none;
35
35
  box-shadow: none;
36
36
  padding: 0.5rem;
37
37
  }
38
38
  #offCanvas a {
39
- color: #E6F9AF;
39
+ color: #FFFFFF;
40
40
  }
41
41
 
42
42
  .contents-button {
@@ -159,12 +159,14 @@
159
159
  "Statuses",
160
160
  "Rails Performance",
161
161
  "Fatal Events",
162
+ "Internal Server Errrors",
163
+ "Errors",
162
164
  "IPs",
163
165
  "Command Invocation",
164
166
  "Performance"
165
167
  ].each do |item| %>
166
- <li class="nav-item">
167
- <a href="#<%= item.downcase.gsub(' ', '-') %>" data-close><%= item %></a>
168
+ <li class="nav-item">
169
+ <a href="#<%= item.downcase.gsub(' ', '-') %>" data-close><%= item %></a>
168
170
  </li>
169
171
  <% end %>
170
172
  </ul>
@@ -216,24 +218,55 @@
216
218
  header: ["Day", "DOW", "Hits"],
217
219
  rows: data[:daily_distribution],
218
220
  vega_spec: {
219
- "mark": {
220
- "type": "line",
221
- "point": {
222
- "filled": false,
223
- "fill": "white"
224
- }
225
- },
226
221
  "encoding": {
227
222
  "x": {"field": "Day", "type": "temporal"},
228
223
  "y": {"field": "Hits", "type": "quantitative"}
229
- }
224
+ },
225
+ "layer": [
226
+ {
227
+ "mark": {
228
+ "type": "line",
229
+ "point": {
230
+ "filled": false,
231
+ "fill": "white"
232
+ }
233
+ }
234
+ },
235
+ {
236
+ "mark": {
237
+ "type": "text",
238
+ "align": "left",
239
+ "baseline": "middle",
240
+ "dx": 5
241
+ },
242
+ "encoding": {
243
+ "text": {"field": "Hits", "type": "quantitative"}
244
+ }
245
+ }
246
+ ]
230
247
  }
231
248
  },
232
249
  { title: "Time Distribution",
233
250
  header: ["Hour", "Hits"],
234
251
  rows: data[:time_distribution],
235
252
  vega_spec: {
236
- "mark": "bar",
253
+ "layer": [
254
+ {
255
+ "mark": "bar",
256
+ },
257
+ {
258
+ "mark": {
259
+ "type": "text",
260
+ "align": "middle",
261
+ "baseline": "top",
262
+ "dx": -10,
263
+ "yOffset": -15
264
+ },
265
+ "encoding": {
266
+ "text": {"field": "Hits", "type": "quantitative"}
267
+ }
268
+ }
269
+ ],
237
270
  "encoding": {
238
271
  "x": {"field": "Hour", "type": "nominal"},
239
272
  "y": {"field": "Hits", "type": "quantitative"}
@@ -244,7 +277,23 @@
244
277
  header: ["Status", "Count"],
245
278
  rows: data[:statuses],
246
279
  vega_spec: {
247
- "mark": "bar",
280
+ "layer": [
281
+ {
282
+ "mark": "bar"
283
+ },
284
+ {
285
+ "mark": {
286
+ "type": "text",
287
+ "align": "left",
288
+ "baseline": "top",
289
+ "dx": -10,
290
+ "yOffset": -20
291
+ },
292
+ "encoding": {
293
+ "text": {"field": "Count", "type": "quantitative"}
294
+ }
295
+ }
296
+ ],
248
297
  "encoding": {
249
298
  "x": {"field": "Status", "type": "nominal"},
250
299
  "y": {"field": "Count", "type": "quantitative"}
@@ -253,7 +302,18 @@
253
302
  },
254
303
  { title: "Rails Performance",
255
304
  header: ['Controller', 'Hits', 'Min', 'Avg', 'Max'],
256
- rows: @data[:performance]
305
+ rows: @data[:performance],
306
+ vega_spec: {
307
+ "layer": [
308
+ {
309
+ "mark": "point"
310
+ },
311
+ ],
312
+ "encoding": {
313
+ "x": {"field": "Avg", "type": "quantitative"},
314
+ "y": {"field": "Hits", "type": "quantitative"}
315
+ },
316
+ }
257
317
  },
258
318
  { title: "Fatal Events",
259
319
  header: ['Date', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:fatal],
@@ -277,7 +337,7 @@
277
337
  <% @reports.each_with_index do |report, index| %>
278
338
  <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
279
339
  <div class="card-divider">
280
- <h2>
340
+ <h2 id="<%= report[:title].downcase.gsub(' ', '-') %>">
281
341
  <%= report[:title] %>
282
342
  </h2>
283
343
  </div>
@@ -305,7 +365,7 @@
305
365
  vegaEmbed('#<%= "plot-#{index}"%>', plot_spec_<%= index %>);
306
366
  </script>
307
367
  <% end %>
308
- <div class="card-section">
368
+ <div class="card-section">
309
369
  <%= render "output_table.html.erb", report %>
310
370
  </div>
311
371
  </article>
data/lib/log_sense/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module LogSense
2
- VERSION = "1.2.1"
2
+ VERSION = "1.3.1"
3
3
  end
data/lib/log_sense.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  require 'log_sense/version'
2
2
  require 'log_sense/options_parser'
3
+ require 'log_sense/apache_log_line_parser'
3
4
  require 'log_sense/apache_log_parser'
4
5
  require 'log_sense/apache_data_cruncher'
5
6
  require 'log_sense/rails_log_parser'
data/log_sense.gemspec CHANGED
@@ -27,7 +27,6 @@ Gem::Specification.new do |spec|
27
27
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
28
28
  spec.require_paths = ["lib"]
29
29
 
30
- spec.add_dependency "apache_log-parser"
31
30
  spec.add_dependency "browser"
32
31
  spec.add_dependency "ipaddr"
33
32
  spec.add_dependency "iso_country_codes"
metadata CHANGED
@@ -1,29 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: log_sense
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 1.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adolfo Villafiorita
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-12-20 00:00:00.000000000 Z
11
+ date: 2021-12-24 00:00:00.000000000 Z
12
12
  dependencies:
13
- - !ruby/object:Gem::Dependency
14
- name: apache_log-parser
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ">="
18
- - !ruby/object:Gem::Version
19
- version: '0'
20
- type: :runtime
21
- prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - ">="
25
- - !ruby/object:Gem::Version
26
- version: '0'
27
13
  - !ruby/object:Gem::Dependency
28
14
  name: browser
29
15
  requirement: !ruby/object:Gem::Requirement
@@ -144,6 +130,7 @@ files:
144
130
  - ip_locations/dbip-country-lite.sqlite3
145
131
  - lib/log_sense.rb
146
132
  - lib/log_sense/apache_data_cruncher.rb
133
+ - lib/log_sense/apache_log_line_parser.rb
147
134
  - lib/log_sense/apache_log_parser.rb
148
135
  - lib/log_sense/emitter.rb
149
136
  - lib/log_sense/ip_locator.rb