log_sense 1.3.4 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.org +35 -0
  3. data/README.org +5 -4
  4. data/exe/log_sense +13 -15
  5. data/lib/log_sense/apache_data_cruncher.rb +14 -12
  6. data/lib/log_sense/apache_log_line_parser.rb +11 -12
  7. data/lib/log_sense/apache_log_parser.rb +44 -36
  8. data/lib/log_sense/emitter.rb +464 -11
  9. data/lib/log_sense/options_parser.rb +30 -30
  10. data/lib/log_sense/rails_data_cruncher.rb +5 -4
  11. data/lib/log_sense/rails_log_parser.rb +108 -100
  12. data/lib/log_sense/templates/_command_invocation.html.erb +0 -4
  13. data/lib/log_sense/templates/_command_invocation.txt.erb +4 -3
  14. data/lib/log_sense/templates/_output_table.html.erb +2 -7
  15. data/lib/log_sense/templates/_output_table.txt.erb +13 -0
  16. data/lib/log_sense/templates/_performance.html.erb +1 -1
  17. data/lib/log_sense/templates/_performance.txt.erb +8 -5
  18. data/lib/log_sense/templates/_report_data.html.erb +2 -2
  19. data/lib/log_sense/templates/_summary.html.erb +6 -1
  20. data/lib/log_sense/templates/_summary.txt.erb +11 -8
  21. data/lib/log_sense/templates/apache.html.erb +24 -225
  22. data/lib/log_sense/templates/apache.txt.erb +35 -0
  23. data/lib/log_sense/templates/rails.html.erb +0 -120
  24. data/lib/log_sense/templates/rails.txt.erb +8 -57
  25. data/lib/log_sense/version.rb +1 -1
  26. metadata +4 -2
data/lib/log_sense/templates/apache.html.erb CHANGED
@@ -157,11 +157,10 @@
157
157
  "Log Structure",
158
158
  "Daily Distribution",
159
159
  "Time Distribution",
160
- "Most Requested Pages",
161
- "Most Requested Resources",
162
- "404 on HTML Files",
163
- "404 on other Resources",
164
- "Attacks",
160
+ "20_ and 30_ on HTML pages",
161
+ "20_ and 30_ on other resources",
162
+ "40_ and 50_ on HTML pages",
163
+ "40_ and 50_ on other Resources",
165
164
  "Statuses",
166
165
  "Daily Statuses",
167
166
  "Browsers",
@@ -221,205 +220,6 @@
221
220
  </article>
222
221
  </div>
223
222
 
224
- <% @reports = [
225
- { title: "Daily Distribution",
226
- header: ["Day", "DOW", "Hits", "Visits", "Size"],
227
- rows: data[:daily_distribution],
228
- vega_spec: {
229
- "layer": [
230
- {
231
- "mark": {
232
- "type": "line",
233
- "point": {
234
- "filled": false,
235
- "fill": "white"
236
- }
237
- },
238
- "encoding": {
239
- "y": {"field": "Hits", "type": "quantitative"}
240
- }
241
- },
242
- {
243
- "mark": {
244
- "type": "text",
245
- "color": "#3E5772",
246
- "align": "middle",
247
- "baseline": "top",
248
- "dx": -10,
249
- "yOffset": -15
250
- },
251
- "encoding": {
252
- "text": {"field": "Hits", "type": "quantitative"},
253
- "y": {"field": "Hits", "type": "quantitative"}
254
- }
255
- },
256
-
257
- {
258
- "mark": {
259
- "type": "line",
260
- "color": "#A52A2A",
261
- "point": {
262
- "color": "#A52A2A",
263
- "filled": false,
264
- "fill": "white",
265
- }
266
- },
267
- "encoding": {
268
- "y": {"field": "Visits", "type": "quantitative"}
269
- }
270
- },
271
-
272
- {
273
- "mark": {
274
- "type": "text",
275
- "color": "#A52A2A",
276
- "align": "middle",
277
- "baseline": "top",
278
- "dx": -10,
279
- "yOffset": -15
280
- },
281
- "encoding": {
282
- "text": {"field": "Visits", "type": "quantitative"},
283
- "y": {"field": "Visits", "type": "quantitative"}
284
- }
285
- },
286
-
287
- ],
288
- "encoding": {
289
- "x": {"field": "Day", "type": "temporal"},
290
- }
291
- }
292
-
293
- },
294
- { title: "Time Distribution",
295
- header: ["Hour", "Hits", "Visits", "Size"],
296
- rows: data[:time_distribution],
297
- vega_spec: {
298
- "layer": [
299
- {
300
- "mark": "bar"
301
- },
302
- {
303
- "mark": {
304
- "type": "text",
305
- "align": "middle",
306
- "baseline": "top",
307
- "dx": -10,
308
- "yOffset": -15
309
- },
310
- "encoding": {
311
- "text": {"field": "Hits", "type": "quantitative"},
312
- "y": {"field": "Hits", "type": "quantitative"}
313
- }
314
- },
315
- ],
316
- "encoding": {
317
- "x": {"field": "Hour", "type": "nominal"},
318
- "y": {"field": "Hits", "type": "quantitative"}
319
- }
320
- }
321
- },
322
- { title: "Most Requested Pages",
323
- header: ["Path", "Hits", "Visits", "Size"],
324
- rows: data[:most_requested_pages],
325
- },
326
- { title: "Most Requested Resources", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_resources] },
327
- { title: "404 on HTML Files", header: ["Path", "Hits", "Visits"], rows: data[:missed_pages] },
328
- { title: "404 on other Resources", header: ["Path", "Hits", "Visits"], rows: data[:missed_resources] },
329
- { title: "Attacks", header: ["Path", "Hits", "Visits"], rows: data[:attacks], col: "small-12 cell" },
330
- { title: "Statuses",
331
- header: ["Status", "Count"],
332
- rows: data[:statuses],
333
- vega_spec: {
334
- "mark": "bar",
335
- "encoding": {
336
- "x": {"field": "Status", "type": "nominal"},
337
- "y": {"field": "Count", "type": "quantitative"}
338
- }
339
- }
340
- },
341
- { title: "Daily Statuses",
342
- header: ["Date", "S_2xx", "S_3xx", "S_4xx"],
343
- rows: data[:statuses_by_day],
344
- vega_spec: {
345
- "transform": [ {"fold": ["S_2xx", "S_3xx", "S_4xx" ] }],
346
- "mark": "bar",
347
- "encoding": {
348
- "x": {
349
- "field": "Date",
350
- "type": "ordinal",
351
- "timeUnit": "day",
352
- },
353
- "y": {
354
- "aggregate": "sum",
355
- "field": "value",
356
- "type": "quantitative"
357
- },
358
- "color": {
359
- "field": "key",
360
- "type": "nominal",
361
- "scale": {
362
- "domain": ["S_2xx", "S_3xx", "S_4xx"],
363
- "range": ["#228b22", "#ff8c00", "#a52a2a"]
364
- },
365
- }
366
- }
367
- }
368
- },
369
- { title: "Browsers",
370
- header: ["Browser", "Hits", "Visits", "Size"],
371
- rows: data[:browsers],
372
- vega_spec: {
373
- "layer": [
374
- { "mark": "bar" },
375
- {
376
- "mark": {
377
- "type": "text",
378
- "align": "middle",
379
- "baseline": "top",
380
- "dx": -10,
381
- "yOffset": -15
382
- },
383
- "encoding": {
384
- "text": {"field": "Hits", "type": "quantitative"},
385
- }
386
- },
387
- ],
388
- "encoding": {
389
- "x": {"field": "Browser", "type": "nominal"},
390
- "y": {"field": "Hits", "type": "quantitative"}
391
- }
392
- }
393
- },
394
- { title: "Platforms",
395
- header: ["Platform", "Hits", "Visits", "Size"],
396
- rows: data[:platforms],
397
- vega_spec: {
398
- "layer": [
399
- { "mark": "bar" },
400
- {
401
- "mark": {
402
- "type": "text",
403
- "align": "middle",
404
- "baseline": "top",
405
- "dx": -10,
406
- "yOffset": -15
407
- },
408
- "encoding": {
409
- "text": {"field": "Hits", "type": "quantitative"},
410
- }
411
- },
412
- ],
413
- "encoding": {
414
- "x": {"field": "Platform", "type": "nominal"},
415
- "y": {"field": "Hits", "type": "quantitative"}
416
- }
417
- }
418
- },
419
- { title: "IPs", header: ["IPs", "Hits", "Visits", "Size", "Country"], rows: data[:ips] },
420
- { title: "Referers", header: ["Referers", "Hits", "Visits", "Size"], rows: data[:referers], col: "small-12 cell" },
421
- ]
422
- %>
423
223
  <div class="grid-x grid-margin-x">
424
224
  <% @reports.each_with_index do |report, index| %>
425
225
  <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
@@ -494,10 +294,10 @@
494
294
  <th>IP</th>
495
295
  <th>
496
296
  <div class="grid-x grid-margin-x">
497
- <div class="col-2 cell">
297
+ <div class="small-2 cell">
498
298
  Day
499
299
  </div>
500
- <div class="col-10 cell">
300
+ <div class="small-10 cell">
501
301
  Resources
502
302
  </div>
503
303
  </div>
@@ -513,26 +313,25 @@
513
313
  <td class="streaks">
514
314
  <div class="grid-x grid-margin-x">
515
315
  <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
516
- <div class="col-2 cell">
316
+ <div class="small-12 medium-1 cell">
517
317
  <span class="date"><%= date %></span>
518
318
  </div>
519
- <div class="col-10 cell grid-x">
520
- <div class="small-12 medium-6 cell">
521
- <span class="res-title">HTML:</span>
522
- <ul>
523
- <% urls.map { |x| x[2] }.compact.select { |x| x.match /.*\.html?/ }.each do |url| %>
524
- <li><%= url %></li>
525
- <% end %>
526
- </ul>
527
- </div>
528
- <div class=" small-12 medium-6 cell">
529
- <span class="res-title small-12 medium-6 cell">Other Resources:</span>
530
- <ul>
531
- <% urls.map { |x| x[2] }.compact.sort.select { |x| x and not x.match /.*\.html?/ }.each do |url| %>
532
- <li><%= url %></li>
533
- <% end %>
534
- </ul>
535
- </div>
319
+ <div class="small-12 medium-5 cell">
320
+ <span class="res-title">HTML:</span>
321
+ <% unique_with_count = urls.map { |x| x[2] }.compact.group_by{|e| e}.map{|k, v| [k, v.length]} %>
322
+ <ul class="no-bullet">
323
+ <% unique_with_count.select { |x| x[0].match /.*\.html?/ }.each do |url| %>
324
+ <li>[<%= url[1] %>] <%= Emitter::escape_javascript url[0] %></li>
325
+ <% end %>
326
+ </ul>
327
+ </div>
328
+ <div class=" small-12 medium-5 cell">
329
+ <span class="res-title">Other Resources:</span>
330
+ <ul class="no-bullet">
331
+ <% unique_with_count.select { |x| x[0] and ! x[0].match /.*\.html?/ }.each do |url| %>
332
+ <li>[<%= url[1] %>] <%= Emitter::escape_javascript url[0] %></li>
333
+ <% end %>
334
+ </ul>
536
335
  </div>
537
336
  <% end %>
538
337
  </div>
@@ -555,7 +354,7 @@
555
354
 
556
355
  <div class="small-12 large-6 cell">
557
356
  <article>
558
- <h2 id="performance"> Performance</h2>
357
+ <h2 id="performance">Performance</h2>
559
358
 
560
359
  <%= render "performance.html.erb", data: data %>
561
360
  </article>
data/lib/log_sense/templates/apache.txt.erb ADDED
@@ -0,0 +1,35 @@
1
+ * Apache Log Analysis
2
+
3
+ >>>> URLs NOT SANITIZED. DO NOT CONVERT TO HTML <<<<
4
+ >>>> (USE THE HTML EXPORT FUNCTION INSTEAD) <<<<
5
+
6
+ ** Summary
7
+
8
+ <%= render "summary.txt.erb", data: data %>
9
+
10
+ <% @reports.each do |report| %>
11
+ ** <%= report[:title] %>
12
+
13
+ <%= render "output_table.txt.erb", report: report, data: data %>
14
+ <% end %>
15
+
16
+ ** Geolocation
17
+
18
+ <%=
19
+ ips = data[:ips].group_by { |x| x[4] }.map { |k, v|
20
+ [k, v.map { |x| x[1] }.inject(&:+), v.map { |x| x[2] }.inject(&:+) ]
21
+ }
22
+ table = Terminal::Table.new headings: ["Country", "Hits", "Total Visits"], rows: ips
23
+ table.style = { border_i: "|" }
24
+ (1..2).map { |i| table.align_column(i, :right) }
25
+ table
26
+ %>
27
+
28
+ ** Command Invocation
29
+
30
+ <%= render 'command_invocation.txt.erb', data: data %>
31
+
32
+ ** Performance
33
+
34
+ <%= render 'performance.txt.erb', data: data %>
35
+
data/lib/log_sense/templates/rails.html.erb CHANGED
@@ -215,126 +215,6 @@
215
215
  </article>
216
216
  </div>
217
217
 
218
- <% @reports = [
219
- { title: "Daily Distribution",
220
- header: ["Day", "DOW", "Hits"],
221
- rows: data[:daily_distribution],
222
- vega_spec: {
223
- "encoding": {
224
- "x": {"field": "Day", "type": "temporal"},
225
- "y": {"field": "Hits", "type": "quantitative"}
226
- },
227
- "layer": [
228
- {
229
- "mark": {
230
- "type": "line",
231
- "point": {
232
- "filled": false,
233
- "fill": "white"
234
- }
235
- }
236
- },
237
- {
238
- "mark": {
239
- "type": "text",
240
- "align": "left",
241
- "baseline": "middle",
242
- "dx": 5
243
- },
244
- "encoding": {
245
- "text": {"field": "Hits", "type": "quantitative"}
246
- }
247
- }
248
- ]
249
- }
250
- },
251
- { title: "Time Distribution",
252
- header: ["Hour", "Hits"],
253
- rows: data[:time_distribution],
254
- vega_spec: {
255
- "layer": [
256
- {
257
- "mark": "bar",
258
- },
259
- {
260
- "mark": {
261
- "type": "text",
262
- "align": "middle",
263
- "baseline": "top",
264
- "dx": -10,
265
- "yOffset": -15
266
- },
267
- "encoding": {
268
- "text": {"field": "Hits", "type": "quantitative"}
269
- }
270
- }
271
- ],
272
- "encoding": {
273
- "x": {"field": "Hour", "type": "nominal"},
274
- "y": {"field": "Hits", "type": "quantitative"}
275
- }
276
- }
277
- },
278
- { title: "Statuses",
279
- header: ["Status", "Count"],
280
- rows: data[:statuses],
281
- vega_spec: {
282
- "layer": [
283
- {
284
- "mark": "bar"
285
- },
286
- {
287
- "mark": {
288
- "type": "text",
289
- "align": "left",
290
- "baseline": "top",
291
- "dx": -10,
292
- "yOffset": -20
293
- },
294
- "encoding": {
295
- "text": {"field": "Count", "type": "quantitative"}
296
- }
297
- }
298
- ],
299
- "encoding": {
300
- "x": {"field": "Status", "type": "nominal"},
301
- "y": {"field": "Count", "type": "quantitative"}
302
- }
303
- }
304
- },
305
- { title: "Rails Performance",
306
- header: ['Controller', 'Hits', 'Min', 'Avg', 'Max'],
307
- rows: @data[:performance],
308
- vega_spec: {
309
- "layer": [
310
- {
311
- "mark": "point"
312
- },
313
- ],
314
- "encoding": {
315
- "x": {"field": "Avg", "type": "quantitative"},
316
- "y": {"field": "Hits", "type": "quantitative"}
317
- },
318
- }
319
- },
320
- { title: "Fatal Events",
321
- header: ['Date', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:fatal],
322
- col: "small-12 cell"
323
- },
324
- { title: "Internal Server Errors",
325
- header: ['Date', 'Status', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:internal_server_error],
326
- col: "small-12 cell"
327
- },
328
- { title: "Errors",
329
- header: ['Log ID', 'Context', 'Description', 'Count'], rows: @data[:error],
330
- col: "small-12 cell"
331
- },
332
- { title: "IPs",
333
- header: ["IPs", "Hits", "Country"],
334
- rows: data[:ips]
335
- },
336
- ]
337
- %>
338
218
  <div class="grid-x grid-margin-x">
339
219
  <% @reports.each_with_index do |report, index| %>
340
220
  <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
data/lib/log_sense/templates/rails.txt.erb CHANGED
@@ -1,66 +1,17 @@
1
1
  * Rails Log Analysis
2
2
 
3
- <%= render "summary.txt.erb", data: data %>
4
-
5
- ** Access by Day
6
-
7
- <%=
8
- table = Terminal::Table.new headings: ['Date', 'Day', 'Events'], rows: @data[:daily_distribution]
9
- table.align_column(3, :right)
10
- table
11
- %>
12
-
13
- ** Access by Time
14
-
15
- <%=
16
- table = Terminal::Table.new headings: ['Hour', 'Events'], rows: @data[:time_distribution]
17
- table.align_column(2, :right)
18
- table
19
- %>
20
-
21
- ** Statuses
22
-
23
- <%=
24
- table = Terminal::Table.new headings: ['Status', 'Events'], rows: @data[:statuses]
25
- table.align_column(2, :right)
26
- table
27
- %>
3
+ >>>> URLs NOT SANITIZED. DO NOT CONVERT TO HTML <<<<
4
+ >>>> (USE THE HTML EXPORT FUNCTION INSTEAD) <<<<
28
5
 
29
- ** Rails Performance
6
+ ** Summary
30
7
 
31
- <%= table = Terminal::Table.new headings: ['Controller', 'Hits', 'Min', 'Avg', 'Max'], rows: @data[:performance]
32
- table.align_column(1, :right)
33
- table.align_column(2, :right)
34
- table.align_column(3, :right)
35
- table.align_column(4, :right)
36
- table
37
- %>
38
-
39
- ** Fatal Events
40
-
41
- <%= table = Terminal::Table.new headings: ['Date', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:fatal]
42
- table
43
- %>
44
-
45
- ** Internal Server Errors
46
-
47
- <%= table = Terminal::Table.new headings: ['Date', 'Status', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:internal_server_error]
48
- table
49
- %>
50
-
51
- ** Errors
52
-
53
- <%= table = Terminal::Table.new headings: ['Log ID', 'Context', 'Description', 'Count'], rows: @data[:error]
54
- table
55
- %>
8
+ <%= render "summary.txt.erb", data: data %>
56
9
 
57
- ** IPs
10
+ <% @reports.each do |report| %>
11
+ ** <%= report[:title] %>
58
12
 
59
- <%=
60
- table = Terminal::Table.new headings: ['IP', 'Hits', 'Country'], rows: @data[:ips]
61
- table.align_column(1, :right)
62
- table
63
- %>
13
+ <%= render "output_table.txt.erb", report: report, data: data %>
14
+ <% end %>
64
15
 
65
16
  ** Command Invocation
66
17
 
data/lib/log_sense/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module LogSense
2
- VERSION = "1.3.4"
2
+ VERSION = "1.4.1"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: log_sense
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.3.4
4
+ version: 1.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adolfo Fibrillation
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-01-12 00:00:00.000000000 Z
11
+ date: 2022-03-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: browser
@@ -142,12 +142,14 @@ files:
142
142
  - lib/log_sense/templates/_command_invocation.txt.erb
143
143
  - lib/log_sense/templates/_log_structure.html.erb
144
144
  - lib/log_sense/templates/_output_table.html.erb
145
+ - lib/log_sense/templates/_output_table.txt.erb
145
146
  - lib/log_sense/templates/_performance.html.erb
146
147
  - lib/log_sense/templates/_performance.txt.erb
147
148
  - lib/log_sense/templates/_report_data.html.erb
148
149
  - lib/log_sense/templates/_summary.html.erb
149
150
  - lib/log_sense/templates/_summary.txt.erb
150
151
  - lib/log_sense/templates/apache.html.erb
152
+ - lib/log_sense/templates/apache.txt.erb
151
153
  - lib/log_sense/templates/rails.html.erb
152
154
  - lib/log_sense/templates/rails.txt.erb
153
155
  - lib/log_sense/version.rb