log_sense 1.0.11 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.org +6 -19
  3. data/README.org +23 -20
  4. data/lib/log_sense/emitter.rb +1 -0
  5. data/lib/log_sense/options_parser.rb +4 -0
  6. data/lib/log_sense/rails_data_cruncher.rb +11 -2
  7. data/lib/log_sense/rails_log_parser.rb +54 -61
  8. data/lib/log_sense/templates/_log_structure.html.erb +24 -0
  9. data/lib/log_sense/templates/_performance.html.erb +21 -23
  10. data/lib/log_sense/templates/_summary.html.erb +23 -34
  11. data/lib/log_sense/templates/apache.html.erb +341 -95
  12. data/lib/log_sense/templates/rails.html.erb +351 -0
  13. data/lib/log_sense/templates/rails.txt.erb +21 -9
  14. data/lib/log_sense/version.rb +1 -1
  15. metadata +4 -3
  16. data/lib/log_sense/templates/_total_hits.html.erb +0 -32
data/lib/log_sense/templates/apache.html.erb CHANGED
@@ -1,7 +1,7 @@
1
1
  <!doctype html>
2
2
  <html class="no-js" lang="en">
3
3
  <head>
4
- <title>Log Sense: <%= data[:log_file] %></title>
4
+ <title><%= options[:title] || "Log Sense: #{data[:log_file]}" %></title>
5
5
 
6
6
  <meta charset="utf-8" />
7
7
  <meta http-equiv="x-ua-compatible" content="ie=edge">
@@ -9,20 +9,34 @@
9
9
  <meta name="author" content="Log Sense">
10
10
  <meta name="description" content="Analysis of <%= data[:log_file] %>">
11
11
 
12
+ <link rel="preconnect" href="https://fonts.googleapis.com">
13
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
14
+ <link href="https://fonts.googleapis.com/css2?family=PT+Sans&display=swap" rel="stylesheet">
15
+
12
16
  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/foundicons/3.0.0/foundation-icons.min.css">
13
17
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/css/foundation.min.css">
14
18
  <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.css"/>
15
19
 
20
+
21
+ <script src="https://cdn.jsdelivr.net/npm/vega@5.21.0"></script>
22
+ <script src="https://cdn.jsdelivr.net/npm/vega-lite@5.2.0"></script>
23
+ <script src="https://cdn.jsdelivr.net/npm/vega-embed@6.20.2"></script>
24
+
16
25
  <style>
26
+ body {
27
+ font-family: 'PT Sans', sans-serif;
28
+ font-size: 80%;
29
+ }
30
+
17
31
  #offCanvas {
18
32
  color: white;
19
- background: #333333;
20
- border-right: ;
33
+ background: #0D0630;
34
+ border-right: none;
21
35
  box-shadow: none;
22
36
  padding: 0.5rem;
23
37
  }
24
38
  #offCanvas a {
25
- color: orange;
39
+ color: #E6F9AF;
26
40
  }
27
41
 
28
42
  .contents-button {
@@ -33,6 +47,100 @@
33
47
  margin-left: 45px;
34
48
  }
35
49
 
50
+ h1 {
51
+ font-size: 1.8rem;
52
+ }
53
+
54
+ h2 {
55
+ font-size: 1.2rem;
56
+ }
57
+
58
+ th {
59
+ padding: 0.2rem 1.2rem 0.2rem 0.2rem !important
60
+ }
61
+
62
+ td {
63
+ padding: 0.2rem 1rem 0.2rem 0.2rem !important;
64
+ }
65
+
66
+ .hits, .visits, .size, .count, .s2xx, .s3xx, .so4xx, .total-hits, .total-visits {
67
+ text-align: right !important;
68
+ }
69
+
70
+ .card-divider {
71
+ padding: 0.2rem 0.4rem 0.2rem 0.4rem;
72
+ background: #0d0630;
73
+ color: white;
74
+ }
75
+
76
+ input, select {
77
+ font-size: 0.8rem !important;
78
+ height: 1.5rem !important;
79
+ padding: 0.2rem 0.4rem 0.2rem 0.4rem !important;
80
+ }
81
+
82
+ .dataTables_info {
83
+ font-size: small;
84
+ color: rgb(202, 202, 202);
85
+ }
86
+
87
+ ul.pagination, li.paginate_button {
88
+ font-size: small;
89
+ margin-top: 0px !important;
90
+ margin-bottom: 0px !important;
91
+ padding-top: 0px !important;
92
+ padding-bottom: 0px !important;
93
+ }
94
+
95
+ .stats-list {
96
+ list-style-type: none;
97
+ clear: left;
98
+ margin: 0;
99
+ padding: 0;
100
+ text-align: center;
101
+ margin-bottom: 30px;
102
+ }
103
+
104
+ .stats-list .stats-list-positive {
105
+ color: #228b22;
106
+ }
107
+
108
+ .stats-list .stats-list-negative {
109
+ color: #a52a2a;
110
+ }
111
+
112
+ .stats-list > li {
113
+ display: inline-block;
114
+ margin-right: 10px;
115
+ padding-right: 10px;
116
+ border-right: 1px solid #cacaca;
117
+ text-align: center;
118
+ font-size: 1.1em;
119
+ font-weight: bold;
120
+ }
121
+
122
+ .stats-list > li:last-child {
123
+ border: none;
124
+ margin: 0;
125
+ padding: 0;
126
+ }
127
+
128
+ .stats-list > li .stats-list-label {
129
+ display: block;
130
+ margin-top: 2px;
131
+ font-size: 0.9em;
132
+ font-weight: normal;
133
+ }
134
+
135
+ #streaks-table .ip {
136
+ vertical-align: top;
137
+ }
138
+ #streaks-table .date {
139
+ font-weight: bold;
140
+ }
141
+ #streaks-table .res-title {
142
+ font-decoration: underline;
143
+ }
36
144
  </style>
37
145
 
38
146
  </head>
@@ -87,123 +195,261 @@
87
195
  </div>
88
196
 
89
197
  <section class="main-section">
90
- <h1>Apache Log Analysis: <%= data[:log_file] || "stdin" %></h1>
198
+ <h1><%= options[:title] || "Log Sense: #{data[:log_file]}" %></h1>
199
+
200
+ <p><b>Input File:</b> <%= (data[:log_file] || "stdin") %></p>
91
201
 
92
202
  <div class="grid-x grid-margin-x">
93
- <article class="small-12 large-6 cell">
94
- <h2 id="summary">Summary</h2>
95
- <%= render "summary.html.erb", data: data %>
203
+ <article class="card small-12 large-6 cell">
204
+ <div class="card-divider">
205
+ <h2 id="summary">Summary</h2>
206
+ </div>
207
+ <div class="card-section">
208
+ <%= render "summary.html.erb", data: data %>
209
+ </div>
96
210
  </article>
97
211
 
98
- <article class="cell small-12 large-6">
99
- <h2 id="log-structure">Log Structure</h2>
100
- <%= render "total_hits.html.erb", data: data %>
212
+ <article class="card cell small-12 large-6">
213
+ <div class="card-divider">
214
+ <h2 id="log-structure">Log Structure</h2>
215
+ </div>
216
+ <div class="card-section">
217
+ <%= render "log_structure.html.erb", data: data %>
218
+ </div>
101
219
  </article>
102
220
  </div>
103
221
 
104
222
  <% @reports = [
105
- { title: "Daily Distribution", header: ["Day", "DOW", "Hits", "Visits", "Size"], rows: data[:daily_distribution] },
106
- { title: "Time Distribution", header: ["Hour", "Hits", "Visits", "Size"], rows: data[:time_distribution] },
107
- { title: "Most Requested Pages", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_pages] },
223
+ { title: "Daily Distribution",
224
+ header: ["Day", "DOW", "Hits", "Visits", "Size"],
225
+ rows: data[:daily_distribution],
226
+ vega_spec: {
227
+ "mark": {
228
+ "type": "line",
229
+ "point": {
230
+ "filled": false,
231
+ "fill": "white"
232
+ }
233
+ },
234
+ "encoding": {
235
+ "x": {"field": "Day", "type": "temporal"},
236
+ "y": {"field": "Hits", "type": "quantitative"}
237
+ }
238
+ }
239
+ },
240
+ { title: "Time Distribution",
241
+ header: ["Hour", "Hits", "Visits", "Size"],
242
+ rows: data[:time_distribution],
243
+ vega_spec: {
244
+ "mark": "bar",
245
+ "encoding": {
246
+ "x": {"field": "Hour", "type": "nominal"},
247
+ "y": {"field": "Hits", "type": "quantitative"}
248
+ }
249
+ }
250
+ },
251
+ { title: "Most Requested Pages",
252
+ header: ["Path", "Hits", "Visits", "Size"],
253
+ rows: data[:most_requested_pages],
254
+ },
108
255
  { title: "Most Requested Resources", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_resources] },
109
256
  { title: "404 on HTML Files", header: ["Path", "Hits", "Visits"], rows: data[:missed_pages] },
110
257
  { title: "404 on other Resources", header: ["Path", "Hits", "Visits"], rows: data[:missed_resources] },
111
- { title: "Attacks", header: ["Path", "Hits", "Visits"], rows: data[:attacks] },
112
- { title: "Statuses", header: ["Status", "Count"], rows: data[:statuses] },
113
- { title: "Daily Statuses", header: ["Status", "2xx", "3xx", "4xx"], rows: data[:statuses_by_day] },
114
- { title: "Browsers", header: ["Browser", "Hits", "Visits", "Size"], rows: data[:browsers] },
115
- { title: "Platforms", header: ["Platform", "Hits", "Visits", "Size"], rows: data[:platforms] },
116
- { title: "Referers", header: ["Referers", "Hits", "Visits", "Size"], rows: data[:referers], col: "col-12" },
258
+ { title: "Attacks", header: ["Path", "Hits", "Visits"], rows: data[:attacks], col: "small-12 cell" },
259
+ { title: "Statuses",
260
+ header: ["Status", "Count"],
261
+ rows: data[:statuses],
262
+ vega_spec: {
263
+ "mark": "bar",
264
+ "encoding": {
265
+ "x": {"field": "Status", "type": "nominal"},
266
+ "y": {"field": "Count", "type": "quantitative"}
267
+ }
268
+ }
269
+ },
270
+ { title: "Daily Statuses",
271
+ header: ["Date", "S_2xx", "S_3xx", "S_4xx"],
272
+ rows: data[:statuses_by_day],
273
+ vega_spec: {
274
+ "transform": [ {"fold": ["S_2xx", "S_3xx", "S_4xx" ] }],
275
+ "mark": "bar",
276
+ "encoding": {
277
+ "x": {
278
+ "field": "Date",
279
+ "type": "ordinal",
280
+ "timeUnit": "day",
281
+ },
282
+ "y": {
283
+ "aggregate": "sum",
284
+ "field": "value",
285
+ "type": "quantitative"
286
+ },
287
+ "color": {
288
+ "field": "key",
289
+ "type": "nominal",
290
+ "scale": {
291
+ "domain": ["S_2xx", "S_3xx", "S_4xx"],
292
+ "range": ["#228b22", "#ff8c00", "#a52a2a"]
293
+ },
294
+ }
295
+ }
296
+ }
297
+ },
298
+ { title: "Browsers",
299
+ header: ["Browser", "Hits", "Visits", "Size"],
300
+ rows: data[:browsers],
301
+ vega_spec: {
302
+ "mark": "bar",
303
+ "encoding": {
304
+ "x": {"field": "Browser", "type": "nominal"},
305
+ "y": {"field": "Hits", "type": "quantitative"}
306
+ }
307
+ }
308
+ },
309
+ { title: "Platforms",
310
+ header: ["Platform", "Hits", "Visits", "Size"],
311
+ rows: data[:platforms],
312
+ vega_spec: {
313
+ "mark": "bar",
314
+ "encoding": {
315
+ "x": {"field": "Platform", "type": "nominal"},
316
+ "y": {"field": "Hits", "type": "quantitative"}
317
+ }
318
+ }
319
+ },
117
320
  { title: "IPs", header: ["IPs", "Hits", "Visits", "Size", "Country"], rows: data[:ips] },
321
+ { title: "Referers", header: ["Referers", "Hits", "Visits", "Size"], rows: data[:referers], col: "small-12 cell" },
118
322
  ]
119
323
  %>
120
324
  <div class="grid-x grid-margin-x">
121
- <% @reports.each do |report| %>
122
- <article class="cell <%= report[:col] || "small-12 large-6" %>" >
123
- <h2 id="<%= report[:title].downcase.gsub(/ +/, '-') %>">
124
- <%= report[:title] %>
125
- </h2>
126
- <%= render "output_table.html.erb", report %>
325
+ <% @reports.each_with_index do |report, index| %>
326
+ <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
327
+ <div class="card-divider">
328
+ <h2>
329
+ <%= report[:title] %>
330
+ </h2>
331
+ </div>
332
+
333
+ <% if report[:vega_spec] %>
334
+ <div id="<%= "plot-#{index}" %>"></div>
335
+ <script>
336
+ plot_spec_<%= index %> = Object.assign(
337
+ <%= report[:vega_spec].to_json %>,
338
+ { "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
339
+ width: "container",
340
+ description: "<%= report[:title] %>",
341
+ data: {
342
+ values: [
343
+ <% report[:rows].each do |row| %>
344
+ {
345
+ <% report[:header].each_with_index do |h, i| %>
346
+ "<%= h %>": <%= (row[i].class == Integer or row[i].class == Float) ? row[i] : "\"#{row[i]}\"" %>,
347
+ <% end %>
348
+ },
349
+ <% end %>
350
+ ]
351
+ },
352
+ });
353
+ vegaEmbed('#<%= "plot-#{index}"%>', plot_spec_<%= index %>);
354
+ </script>
355
+ <% end %>
356
+ <div class="card-section">
357
+ <%= render "output_table.html.erb", report %>
358
+ </div>
127
359
  </article>
128
360
  <% end %>
129
361
  </div>
130
362
 
131
- <article>
132
- <h2 id="geolocation">Geolocation</h2>
133
- <table id="geolocation-table" class="table unstriped">
134
- <thead>
135
- <tr>
136
- <th>Country Code</th>
137
- <th>Total Hits</th>
138
- <th>Total Visits</th>
139
- <th>IPs</th>
140
- </tr>
141
- </thead>
142
- <tbody>
143
- <%# IP, Hits, Visits Size, Country%>
144
- <% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
363
+ <article class="card">
364
+ <div class="card-divider">
365
+ <h2 id="geolocation">Geolocation</h2>
366
+ </div>
367
+ <div class="card-section">
368
+ <table id="geolocation-table" class="table unstriped">
369
+ <thead>
145
370
  <tr>
146
- <td><%= k %></td>
147
- <td><%= v.map { |x| x[1] }.inject(&:+) %></td>
148
- <td><%= v.map { |x| x[2] }.inject(&:+) %></td>
149
- <td><%= v.map { |x| x[0] }.join(", ") %></td>
150
- </tr>
151
- <% end %>
152
- </tbody>
153
- </table>
371
+ <th>Country Code</th>
372
+ <th>Total Hits</th>
373
+ <th>Total Visits</th>
374
+ <th>IPs</th>
375
+ </tr>
376
+ </thead>
377
+ <tbody>
378
+ <%# IP, Hits, Visits Size, Country%>
379
+ <% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
380
+ <tr>
381
+ <td class="country"><%= k %></td>
382
+ <td class="total-hits"><%= v.map { |x| x[1] }.inject(&:+) %></td>
383
+ <td class="total-visits"><%= v.map { |x| x[2] }.inject(&:+) %></td>
384
+ <td class="ips">
385
+ <%= v.map { |x| "<a href=\"https://whatismyipaddress.com/ip/#{x[0]}\">#{x[0]}</a>" }.join(", ") %>
386
+ </td>
387
+ </tr>
388
+ <% end %>
389
+ </tbody>
390
+ </table>
391
+ </div>
154
392
  </article>
155
393
 
156
- <article>
157
- <h2 id="streaks">Streaks</h2>
158
-
159
- <table id="streaks-table" class="table data-table streaks">
160
- <thead>
161
- <tr>
162
- <th>IP</th>
163
- <th>
164
- <div class="grid-x grid-margin-x">
165
- <div class="col-2 cell">
166
- Day
167
- </div>
168
- <div class="col-10 cell">
169
- Resources
170
- </div>
171
- </div>
172
- </th>
173
- </tr>
174
- </thead>
175
- <tbody>
176
- <% data[:streaks].group_by(&:first).each do |ip, date_urls| %>
394
+ <article class="card">
395
+ <div class="card-divider">
396
+ <h2 id="streaks">Streaks</h2>
397
+ </div>
398
+ <div class="card-section">
399
+ <table id="streaks-table" class="table data-table streaks">
400
+ <thead>
177
401
  <tr>
178
- <td class="ip"><%= ip %></td>
179
- <td class="streaks">
402
+ <th>IP</th>
403
+ <th>
180
404
  <div class="grid-x grid-margin-x">
181
- <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
182
- <div class="col-2 cell">
183
- <%= date %>
184
- </div>
185
- <div class="col-10 cell">
186
- <span class="res-title">HTML:</span>
187
- <ul>
188
- <% urls.map { |x| x[2] }.compact.select { |x| x.match /.*\.html?/ }.each do |url| %>
189
- <li><%= url %></li>
190
- <% end %>
191
- </ul>
192
-
193
- <span class="res-title">Other Resources:</span>
194
- <ul>
195
- <% urls.map { |x| x[2] }.compact.sort.select { |x| x and not x.match /.*\.html?/ }.each do |url| %>
196
- <li><%= url %></li>
197
- <% end %>
198
- </ul>
199
- </div>
200
- <% end %>
405
+ <div class="col-2 cell">
406
+ Day
407
+ </div>
408
+ <div class="col-10 cell">
409
+ Resources
410
+ </div>
201
411
  </div>
202
- </td>
412
+ </th>
203
413
  </tr>
204
- <% end %>
205
- </tbody>
206
- </table>
414
+ </thead>
415
+ <tbody>
416
+ <% data[:streaks].group_by(&:first).each do |ip, date_urls| %>
417
+ <tr>
418
+ <td class="ip">
419
+ <a href="https://whatismyipaddress.com/ip/<%= ip %>"><%= ip %></a>
420
+ </td>
421
+ <td class="streaks">
422
+ <div class="grid-x grid-margin-x">
423
+ <% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
424
+ <div class="col-2 cell">
425
+ <span class="date"><%= date %></span>
426
+ </div>
427
+ <div class="col-10 cell grid-x">
428
+ <div class="small-12 medium-6 cell">
429
+ <span class="res-title">HTML:</span>
430
+ <ul>
431
+ <% urls.map { |x| x[2] }.compact.select { |x| x.match /.*\.html?/ }.each do |url| %>
432
+ <li><%= url %></li>
433
+ <% end %>
434
+ </ul>
435
+ </div>
436
+ <div class=" small-12 medium-6 cell">
437
+ <span class="res-title small-12 medium-6 cell">Other Resources:</span>
438
+ <ul>
439
+ <% urls.map { |x| x[2] }.compact.sort.select { |x| x and not x.match /.*\.html?/ }.each do |url| %>
440
+ <li><%= url %></li>
441
+ <% end %>
442
+ </ul>
443
+ </div>
444
+ </div>
445
+ <% end %>
446
+ </div>
447
+ </td>
448
+ </tr>
449
+ <% end %>
450
+ </tbody>
451
+ </table>
452
+ </div>
207
453
  </article>
208
454
 
209
455
  <div class="grid-x grid-margin-x">