log_sense 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/lib/log_sense/apache_log_line_parser.rb +53 -0
  3. data/lib/log_sense/apache_log_parser.rb +14 -12
  4. data/lib/log_sense/rails_log_parser.rb +2 -3
  5. data/lib/log_sense/templates/_log_structure.html.erb +2 -2
  6. data/lib/log_sense/templates/apache.html.erb +113 -16
  7. data/lib/log_sense/templates/rails.html.erb +78 -18
  8. data/lib/log_sense/version.rb +1 -1
  9. data/lib/log_sense.rb +1 -0
  10. data/log_sense.gemspec +0 -1
  11. metadata +3 -16
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d96a22ce71f0c0266811faa1853981d1211b93687bfd2e3077b91189feb5742b
4
- data.tar.gz: 014ad1230a6a83b379310ba2b93b16bfd956db19a8f087bd2c216b5e77211620
3
+ metadata.gz: e566be6e4288d20605f8dd39aa5fb016534a20af366b4f7ad229af3610397771
4
+ data.tar.gz: cb338c23a5e829876a2649896cacb00f4274f1be7c9a606a1dcd4c3d9f0431b1
5
5
  SHA512:
6
- metadata.gz: 8fe6fb1a329b0cabae06199f13c70936c9c7aeaa6c99142a683414ac380bfc23fecb14d3588803f2f9afc24483bfb8f429f867633ce5029c3dc23df1c713dea4
7
- data.tar.gz: 10fe88f045845a5573f8ba7aedde8d3475a0f390e6025746799525be685c6db11e5eb9148e9de3f8f05a7950ae9c912f2d67e454a285de08143789db374818c7
6
+ metadata.gz: 89ca1ce0379ed828b0a6ea7af79e35626f0b67693f4b3f2ae744df9b6714fff6f02f30c82f78d84fd76e0d947378a0f3fb11fd865cb015cb11964c02694e3e02
7
+ data.tar.gz: ddf3d3a976979e03c9544a613051fab201146db40b6fb0da2e21b521012b6f19413df133d68ae10187baff0a8abef4c43e2a79322a58c851572faa94c792229c
data/lib/log_sense/apache_log_line_parser.rb ADDED
@@ -0,0 +1,53 @@
1
+ module LogSense
2
+ class ApacheLogLineParser
3
+ # parses a query and makes it into an expression which can be evaluated
4
+ # LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" combined
5
+ #
6
+ # %h: IP
7
+ # %l: ident or -
8
+ # %u: userid or -
9
+ # %t: [10/Oct/2000:13:55:36 -0700]
10
+ # day = 2*digit
11
+ # month = 3*letter
12
+ # year = 4*digit
13
+ # hour = 2*digit
14
+ # minute = 2*digit
15
+ # second = 2*digit
16
+ # zone = (`+' | `-') 4*digit
17
+ # %r: GET /apache_pb.gif HTTP/1.0
18
+ # %{User-agent}: "
19
+ #
20
+ # 116.179.32.16 - - [19/Dec/2021:22:35:11 +0100] "GET / HTTP/1.1" 200 135 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)"
21
+
22
+ DAY = /[0-9]{2}/
23
+ MONTH = /[A-Za-z]{3}/
24
+ YEAR = /[0-9]{4}/
25
+ TIMEC = /[0-9]{2}/
26
+ TIMEZONE = /(\+|-)[0-9]{4}/
27
+
28
+ IP = /(?<ip>[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})/
29
+ IDENT = /(?<ident>[^ ]+|-)/
30
+ USERID = /(?<userid>[^ ]+|-)/
31
+
32
+ TIMESTAMP = /(?<date>#{DAY}\/#{MONTH}\/#{YEAR}):(?<time>#{TIMEC}:#{TIMEC}:#{TIMEC} #{TIMEZONE})/
33
+
34
+ VERB=/(?<method>GET|HEAD|POST|PUT|DELETE|CONNECT|OPTIONS|TRACE|PATCH)/
35
+ PROTOCOL=/(?<protocol>HTTP\/[0-9]\.[0-9])/
36
+ URL=/(?<url>[^ ]+)/
37
+ REFERER=/(?<referer>[^ ])+/
38
+ RETURN_CODE=/(?<status>[1-5][0-9][0-9])/
39
+ SIZE=/(?<size>[0-9]+|-)/
40
+
41
+ USER_AGENT = /(?<user_agent>[^"]+)/
42
+
43
+ attr_reader :format
44
+
45
+ def initialize
46
+ @format = /#{IP} #{IDENT} #{USERID} \[#{TIMESTAMP}\] "#{VERB} #{URL} #{PROTOCOL}" #{RETURN_CODE} #{SIZE} "#{REFERER}" "#{USER_AGENT}"/
47
+ end
48
+
49
+ def parse line
50
+ hash = @format.match(line) || raise("Apache LogLine Parser Error: Could not parse #{line}")
51
+ end
52
+ end
53
+ end
data/lib/log_sense/apache_log_parser.rb CHANGED
@@ -1,4 +1,3 @@
1
- require 'apache_log/parser'
2
1
  require 'sqlite3'
3
2
  require 'browser'
4
3
 
@@ -50,21 +49,20 @@ module LogSense
50
49
  platform_version)
51
50
  values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
52
51
 
53
- parser = ApacheLog::Parser.new(options[:format] || 'combined')
52
+ parser = ApacheLogLineParser.new
54
53
 
55
54
  content.each do |line|
56
55
  begin
57
56
  hash = parser.parse line
58
-
59
57
  ua = Browser.new(hash[:user_agent], accept_language: "en-us")
60
58
  ins.execute(
61
- hash[:datetime].iso8601,
62
- hash[:remote_host],
63
- hash[:user],
64
- hash[:datetime].strftime("%Y-%m-%d") + " " + hash[:remote_host] + " " + hash[:user_agent],
65
- hash[:request][:method],
66
- hash[:request][:path],
67
- (hash[:request][:path] ? File.extname(hash[:request][:path]) : ""),
59
+ DateTime.parse("#{hash[:date]}T#{hash[:time]}").iso8601,
60
+ hash[:ip],
61
+ hash[:userid],
62
+ unique_visitor_id(hash),
63
+ hash[:method],
64
+ hash[:url],
65
+ (hash[:url] ? File.extname(hash[:url]) : ""),
68
66
  hash[:status],
69
67
  hash[:size].to_i,
70
68
  hash[:referer],
@@ -75,13 +73,17 @@ module LogSense
75
73
  (ua.platform.name || ""),
76
74
  (ua.platform.version || "")
77
75
  )
78
- rescue
79
- STDERR.puts "Apache Log parser error: could not parse #{line}"
76
+ rescue StandardError => e
77
+ STDERR.puts e.message
80
78
  end
81
79
  end
82
80
 
83
81
  db
84
82
  end
85
83
 
84
+ def self.unique_visitor_id hash
85
+ "#{hash[:date]} #{hash[:ip]} #{hash[:user_agent]}"
86
+ end
87
+
86
88
  end
87
89
  end
data/lib/log_sense/rails_log_parser.rb CHANGED
@@ -1,5 +1,4 @@
1
1
  require 'sqlite3'
2
- require 'byebug'
3
2
 
4
3
  module LogSense
5
4
  module RailsLogParser
@@ -186,8 +185,8 @@ module LogSense
186
185
  # [584cffcc-f1fd-4b5c-bb8b-b89621bd4921] ActionController::RoutingError (No route matches [GET] "/assets/foundation-icons.svg"):
187
186
  # [fd8df8b5-83c9-48b5-a056-e5026e31bd5e] ActionView::Template::Error (undefined method `all_my_ancestor' for nil:NilClass):
188
187
  # [d17ed55c-f5f1-442a-a9d6-3035ab91adf0] ActionView::Template::Error (undefined method `volunteer_for' for #<DonationsController:0x007f4864c564b8>
189
- CONTEXT = /(?<context>[^ ]+Error)/
190
- ERROR_REGEXP = /^\[#{ID}\] #{CONTEXT} \((?<description>.*)\):/
188
+ EXCEPTION = /[A-Za-z_0-9:]+(Error)?/
189
+ ERROR_REGEXP = /^\[#{ID}\] (?<context>#{EXCEPTION}) \((?<description>(#{EXCEPTION})?.*)\):/
191
190
 
192
191
  def self.match_and_process_error line
193
192
  matchdata = ERROR_REGEXP.match line
data/lib/log_sense/templates/_log_structure.html.erb CHANGED
@@ -14,11 +14,11 @@
14
14
  <%= data[:log_size] %> <span class="stats-list-label">Total Entries</span>
15
15
  </li>
16
16
  <li class="stats-list-negative">
17
- <%= data[:selfpolls_size] %> <span class="stats-list-label">Self Polls Entries</span>
17
+ <%= data[:selfpolls_size] %> <span class="stats-list-label">Self Polls</span>
18
18
  </li>
19
19
  <li class="stats-list-negative">
20
20
  <td><%= data[:crawlers_size] %></td>
21
- <span class="stats-list-label">Crawlers Entries</span>
21
+ <span class="stats-list-label">Crawlers</span>
22
22
  </li>
23
23
  </ul>
24
24
 
data/lib/log_sense/templates/apache.html.erb CHANGED
@@ -29,14 +29,14 @@
29
29
  }
30
30
 
31
31
  #offCanvas {
32
- color: white;
33
- background: #0D0630;
32
+ color: #DEDEDE;
33
+ background: #1C1C1C;
34
34
  border-right: none;
35
35
  box-shadow: none;
36
36
  padding: 0.5rem;
37
37
  }
38
38
  #offCanvas a {
39
- color: #E6F9AF;
39
+ color: #FFFFFF;
40
40
  }
41
41
 
42
42
  .contents-button {
@@ -69,7 +69,7 @@
69
69
 
70
70
  .card-divider {
71
71
  padding: 0.2rem 0.4rem 0.2rem 0.4rem;
72
- background: #0d0630;
72
+ background: #1C1C1C;
73
73
  color: white;
74
74
  }
75
75
 
@@ -224,24 +224,93 @@
224
224
  header: ["Day", "DOW", "Hits", "Visits", "Size"],
225
225
  rows: data[:daily_distribution],
226
226
  vega_spec: {
227
- "mark": {
228
- "type": "line",
229
- "point": {
230
- "filled": false,
231
- "fill": "white"
232
- }
233
- },
227
+ "layer": [
228
+ {
229
+ "mark": {
230
+ "type": "line",
231
+ "point": {
232
+ "filled": false,
233
+ "fill": "white"
234
+ }
235
+ },
236
+ "encoding": {
237
+ "y": {"field": "Hits", "type": "quantitative"}
238
+ }
239
+ },
240
+ {
241
+ "mark": {
242
+ "type": "text",
243
+ "color": "#3E5772",
244
+ "align": "middle",
245
+ "baseline": "top",
246
+ "dx": -10,
247
+ "yOffset": -15
248
+ },
249
+ "encoding": {
250
+ "text": {"field": "Hits", "type": "quantitative"},
251
+ "y": {"field": "Hits", "type": "quantitative"}
252
+ }
253
+ },
254
+
255
+ {
256
+ "mark": {
257
+ "type": "line",
258
+ "color": "#A52A2A",
259
+ "point": {
260
+ "color": "#A52A2A",
261
+ "filled": false,
262
+ "fill": "white",
263
+ }
264
+ },
265
+ "encoding": {
266
+ "y": {"field": "Visits", "type": "quantitative"}
267
+ }
268
+ },
269
+
270
+ {
271
+ "mark": {
272
+ "type": "text",
273
+ "color": "#A52A2A",
274
+ "align": "middle",
275
+ "baseline": "top",
276
+ "dx": -10,
277
+ "yOffset": -15
278
+ },
279
+ "encoding": {
280
+ "text": {"field": "Visits", "type": "quantitative"},
281
+ "y": {"field": "Visits", "type": "quantitative"}
282
+ }
283
+ },
284
+
285
+ ],
234
286
  "encoding": {
235
287
  "x": {"field": "Day", "type": "temporal"},
236
- "y": {"field": "Hits", "type": "quantitative"}
237
288
  }
238
289
  }
290
+
239
291
  },
240
292
  { title: "Time Distribution",
241
293
  header: ["Hour", "Hits", "Visits", "Size"],
242
294
  rows: data[:time_distribution],
243
295
  vega_spec: {
244
- "mark": "bar",
296
+ "layer": [
297
+ {
298
+ "mark": "bar"
299
+ },
300
+ {
301
+ "mark": {
302
+ "type": "text",
303
+ "align": "middle",
304
+ "baseline": "top",
305
+ "dx": -10,
306
+ "yOffset": -15
307
+ },
308
+ "encoding": {
309
+ "text": {"field": "Hits", "type": "quantitative"},
310
+ "y": {"field": "Hits", "type": "quantitative"}
311
+ }
312
+ },
313
+ ],
245
314
  "encoding": {
246
315
  "x": {"field": "Hour", "type": "nominal"},
247
316
  "y": {"field": "Hits", "type": "quantitative"}
@@ -299,7 +368,21 @@
299
368
  header: ["Browser", "Hits", "Visits", "Size"],
300
369
  rows: data[:browsers],
301
370
  vega_spec: {
302
- "mark": "bar",
371
+ "layer": [
372
+ { "mark": "bar" },
373
+ {
374
+ "mark": {
375
+ "type": "text",
376
+ "align": "middle",
377
+ "baseline": "top",
378
+ "dx": -10,
379
+ "yOffset": -15
380
+ },
381
+ "encoding": {
382
+ "text": {"field": "Hits", "type": "quantitative"},
383
+ }
384
+ },
385
+ ],
303
386
  "encoding": {
304
387
  "x": {"field": "Browser", "type": "nominal"},
305
388
  "y": {"field": "Hits", "type": "quantitative"}
@@ -310,7 +393,21 @@
310
393
  header: ["Platform", "Hits", "Visits", "Size"],
311
394
  rows: data[:platforms],
312
395
  vega_spec: {
313
- "mark": "bar",
396
+ "layer": [
397
+ { "mark": "bar" },
398
+ {
399
+ "mark": {
400
+ "type": "text",
401
+ "align": "middle",
402
+ "baseline": "top",
403
+ "dx": -10,
404
+ "yOffset": -15
405
+ },
406
+ "encoding": {
407
+ "text": {"field": "Hits", "type": "quantitative"},
408
+ }
409
+ },
410
+ ],
314
411
  "encoding": {
315
412
  "x": {"field": "Platform", "type": "nominal"},
316
413
  "y": {"field": "Hits", "type": "quantitative"}
@@ -325,7 +422,7 @@
325
422
  <% @reports.each_with_index do |report, index| %>
326
423
  <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
327
424
  <div class="card-divider">
328
- <h2>
425
+ <h2 id="<%= report[:title].downcase.gsub(' ', '-') %>">
329
426
  <%= report[:title] %>
330
427
  </h2>
331
428
  </div>
data/lib/log_sense/templates/rails.html.erb CHANGED
@@ -29,14 +29,14 @@
29
29
  }
30
30
 
31
31
  #offCanvas {
32
- color: white;
33
- background: #0D0630;
32
+ color: #CECECE;
33
+ background: #BD000D;
34
34
  border-right: none;
35
35
  box-shadow: none;
36
36
  padding: 0.5rem;
37
37
  }
38
38
  #offCanvas a {
39
- color: #E6F9AF;
39
+ color: #FFFFFF;
40
40
  }
41
41
 
42
42
  .contents-button {
@@ -159,12 +159,14 @@
159
159
  "Statuses",
160
160
  "Rails Performance",
161
161
  "Fatal Events",
162
+ "Internal Server Errrors",
163
+ "Errors",
162
164
  "IPs",
163
165
  "Command Invocation",
164
166
  "Performance"
165
167
  ].each do |item| %>
166
- <li class="nav-item">
167
- <a href="#<%= item.downcase.gsub(' ', '-') %>" data-close><%= item %></a>
168
+ <li class="nav-item">
169
+ <a href="#<%= item.downcase.gsub(' ', '-') %>" data-close><%= item %></a>
168
170
  </li>
169
171
  <% end %>
170
172
  </ul>
@@ -216,24 +218,55 @@
216
218
  header: ["Day", "DOW", "Hits"],
217
219
  rows: data[:daily_distribution],
218
220
  vega_spec: {
219
- "mark": {
220
- "type": "line",
221
- "point": {
222
- "filled": false,
223
- "fill": "white"
224
- }
225
- },
226
221
  "encoding": {
227
222
  "x": {"field": "Day", "type": "temporal"},
228
223
  "y": {"field": "Hits", "type": "quantitative"}
229
- }
224
+ },
225
+ "layer": [
226
+ {
227
+ "mark": {
228
+ "type": "line",
229
+ "point": {
230
+ "filled": false,
231
+ "fill": "white"
232
+ }
233
+ }
234
+ },
235
+ {
236
+ "mark": {
237
+ "type": "text",
238
+ "align": "left",
239
+ "baseline": "middle",
240
+ "dx": 5
241
+ },
242
+ "encoding": {
243
+ "text": {"field": "Hits", "type": "quantitative"}
244
+ }
245
+ }
246
+ ]
230
247
  }
231
248
  },
232
249
  { title: "Time Distribution",
233
250
  header: ["Hour", "Hits"],
234
251
  rows: data[:time_distribution],
235
252
  vega_spec: {
236
- "mark": "bar",
253
+ "layer": [
254
+ {
255
+ "mark": "bar",
256
+ },
257
+ {
258
+ "mark": {
259
+ "type": "text",
260
+ "align": "middle",
261
+ "baseline": "top",
262
+ "dx": -10,
263
+ "yOffset": -15
264
+ },
265
+ "encoding": {
266
+ "text": {"field": "Hits", "type": "quantitative"}
267
+ }
268
+ }
269
+ ],
237
270
  "encoding": {
238
271
  "x": {"field": "Hour", "type": "nominal"},
239
272
  "y": {"field": "Hits", "type": "quantitative"}
@@ -244,7 +277,23 @@
244
277
  header: ["Status", "Count"],
245
278
  rows: data[:statuses],
246
279
  vega_spec: {
247
- "mark": "bar",
280
+ "layer": [
281
+ {
282
+ "mark": "bar"
283
+ },
284
+ {
285
+ "mark": {
286
+ "type": "text",
287
+ "align": "left",
288
+ "baseline": "top",
289
+ "dx": -10,
290
+ "yOffset": -20
291
+ },
292
+ "encoding": {
293
+ "text": {"field": "Count", "type": "quantitative"}
294
+ }
295
+ }
296
+ ],
248
297
  "encoding": {
249
298
  "x": {"field": "Status", "type": "nominal"},
250
299
  "y": {"field": "Count", "type": "quantitative"}
@@ -253,7 +302,18 @@
253
302
  },
254
303
  { title: "Rails Performance",
255
304
  header: ['Controller', 'Hits', 'Min', 'Avg', 'Max'],
256
- rows: @data[:performance]
305
+ rows: @data[:performance],
306
+ vega_spec: {
307
+ "layer": [
308
+ {
309
+ "mark": "point"
310
+ },
311
+ ],
312
+ "encoding": {
313
+ "x": {"field": "Avg", "type": "quantitative"},
314
+ "y": {"field": "Hits", "type": "quantitative"}
315
+ },
316
+ }
257
317
  },
258
318
  { title: "Fatal Events",
259
319
  header: ['Date', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:fatal],
@@ -277,7 +337,7 @@
277
337
  <% @reports.each_with_index do |report, index| %>
278
338
  <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
279
339
  <div class="card-divider">
280
- <h2>
340
+ <h2 id="<%= report[:title].downcase.gsub(' ', '-') %>">
281
341
  <%= report[:title] %>
282
342
  </h2>
283
343
  </div>
@@ -305,7 +365,7 @@
305
365
  vegaEmbed('#<%= "plot-#{index}"%>', plot_spec_<%= index %>);
306
366
  </script>
307
367
  <% end %>
308
- <div class="card-section">
368
+ <div class="card-section">
309
369
  <%= render "output_table.html.erb", report %>
310
370
  </div>
311
371
  </article>
data/lib/log_sense/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module LogSense
2
- VERSION = "1.2.0"
2
+ VERSION = "1.3.0"
3
3
  end
data/lib/log_sense.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  require 'log_sense/version'
2
2
  require 'log_sense/options_parser'
3
+ require 'log_sense/apache_log_line_parser'
3
4
  require 'log_sense/apache_log_parser'
4
5
  require 'log_sense/apache_data_cruncher'
5
6
  require 'log_sense/rails_log_parser'
data/log_sense.gemspec CHANGED
@@ -27,7 +27,6 @@ Gem::Specification.new do |spec|
27
27
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
28
28
  spec.require_paths = ["lib"]
29
29
 
30
- spec.add_dependency "apache_log-parser"
31
30
  spec.add_dependency "browser"
32
31
  spec.add_dependency "ipaddr"
33
32
  spec.add_dependency "iso_country_codes"
metadata CHANGED
@@ -1,29 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: log_sense
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.0
4
+ version: 1.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adolfo Villafiorita
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-12-20 00:00:00.000000000 Z
11
+ date: 2021-12-24 00:00:00.000000000 Z
12
12
  dependencies:
13
- - !ruby/object:Gem::Dependency
14
- name: apache_log-parser
15
- requirement: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ">="
18
- - !ruby/object:Gem::Version
19
- version: '0'
20
- type: :runtime
21
- prerelease: false
22
- version_requirements: !ruby/object:Gem::Requirement
23
- requirements:
24
- - - ">="
25
- - !ruby/object:Gem::Version
26
- version: '0'
27
13
  - !ruby/object:Gem::Dependency
28
14
  name: browser
29
15
  requirement: !ruby/object:Gem::Requirement
@@ -144,6 +130,7 @@ files:
144
130
  - ip_locations/dbip-country-lite.sqlite3
145
131
  - lib/log_sense.rb
146
132
  - lib/log_sense/apache_data_cruncher.rb
133
+ - lib/log_sense/apache_log_line_parser.rb
147
134
  - lib/log_sense/apache_log_parser.rb
148
135
  - lib/log_sense/emitter.rb
149
136
  - lib/log_sense/ip_locator.rb