log_sense 1.7.0 → 1.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.org +5 -0
  3. data/README.org +46 -46
  4. data/lib/log_sense/apache_report_shaper.rb +267 -163
  5. data/lib/log_sense/emitter.rb +1 -3
  6. data/lib/log_sense/rails/log_parser.rb +74 -14
  7. data/lib/log_sense/rails_aggregator.rb +74 -1
  8. data/lib/log_sense/rails_report_shaper.rb +330 -107
  9. data/lib/log_sense/templates/_log_structure.html.erb +6 -6
  10. data/lib/log_sense/templates/_performance.html.erb +6 -5
  11. data/lib/log_sense/templates/_rails.css.erb +14 -2
  12. data/lib/log_sense/templates/_stylesheet.css +26 -8
  13. data/lib/log_sense/templates/_summary.html.erb +8 -8
  14. data/lib/log_sense/templates/report_html.erb +18 -2
  15. data/lib/log_sense/version.rb +1 -1
  16. metadata +2 -2
data/lib/log_sense/rails_report_shaper.rb CHANGED
@@ -7,133 +7,227 @@ module LogSense
7
7
  header: %w[Day DOW Hits],
8
8
  column_alignment: %i[left left right],
9
9
  rows: data[:daily_distribution],
10
- vega_spec: {
11
- "encoding": {
12
- "x": {"field": "Day", "type": "temporal"},
13
- "y": {"field": "Hits", "type": "quantitative"}
14
- },
15
- "layer": [
16
- {
17
- "mark": {
18
- "type": "line",
19
- "point": {
20
- "filled": false,
21
- "fill": "white"
22
- }
23
- }
24
- },
25
- {
26
- "mark": {
27
- "type": "text",
28
- "align": "left",
29
- "baseline": "middle",
30
- "dx": 5
31
- },
32
- "encoding": {
33
- "text": {"field": "Hits", "type": "quantitative"}
34
- }
35
- }
36
- ]
37
- }
10
+ echarts_spec: "{
11
+ toolbox: {
12
+ feature: {
13
+ saveAsImage: {},
14
+ }
15
+ },
16
+ tooltip: {
17
+ trigger: 'axis'
18
+ },
19
+ xAxis: {
20
+ type: 'category',
21
+ data: SERIES_DATA.filter(row => row['Day'] != '').map(row => row['Day']),
22
+ showGrid: true,
23
+ },
24
+ yAxis: {
25
+ type: 'value',
26
+ name: 'Hits',
27
+ showGrid: true,
28
+ },
29
+ series: [
30
+ {
31
+ data: SERIES_DATA.filter(row => row['Day'] != '').map(row => row['Hits']),
32
+ type: 'line',
33
+ color: '#D30001',
34
+ label: {
35
+ show: true,
36
+ position: 'top'
37
+ },
38
+ },
39
+ ]
40
+ };",
38
41
  },
39
42
  {
40
43
  title: "Time Distribution",
41
44
  header: %w[Hour Hits],
42
45
  column_alignment: %i[left right],
43
46
  rows: data[:time_distribution],
44
- vega_spec: {
45
- "layer": [
46
- {
47
- "mark": "bar",
48
- },
49
- {
50
- "mark": {
51
- "type": "text",
52
- "align": "middle",
53
- "baseline": "top",
54
- "dx": -10,
55
- "yOffset": -15
56
- },
57
- "encoding": {
58
- "text": {"field": "Hits", "type": "quantitative"}
59
- }
60
- }
61
- ],
62
- "encoding": {
63
- "x": {"field": "Hour", "type": "nominal"},
64
- "y": {"field": "Hits", "type": "quantitative"}
65
- }
66
- }
47
+ echarts_spec: "{
48
+ xAxis: {
49
+ type: 'category',
50
+ data: SERIES_DATA.map(row => row['Hour'])
51
+ /* data: ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09',
52
+ '10', '11', '12', '13', '14', '15', '16', '17', '18', '19',
53
+ '20', '21', '22', '23', '24'] */
54
+ },
55
+ yAxis: {
56
+ type: 'value'
57
+ },
58
+ tooltip: {
59
+ trigger: 'axis'
60
+ },
61
+ series: [
62
+ {
63
+ data: SERIES_DATA.map(row => row['Hits']),
64
+ type: 'bar',
65
+ color: '#D30001',
66
+ label: {
67
+ show: true,
68
+ position: 'top'
69
+ },
70
+ }
71
+ ]
72
+ }",
67
73
  },
68
74
  {
69
75
  title: "Statuses",
70
76
  header: %w[Status Count],
71
77
  column_alignment: %i[left right],
72
78
  rows: data[:statuses],
73
- vega_spec: {
74
- "layer": [
75
- {
76
- "mark": "bar"
77
- },
78
- {
79
- "mark": {
80
- "type": "text",
81
- "align": "left",
82
- "baseline": "top",
83
- "dx": -10,
84
- "yOffset": -20
85
- },
86
- "encoding": {
87
- "text": {"field": "Count", "type": "quantitative"}
88
- }
89
- }
90
- ],
91
- "encoding": {
92
- "x": {"field": "Status", "type": "nominal"},
93
- "y": {"field": "Count", "type": "quantitative"}
94
- }
95
- }
79
+ col: "small-12 cell",
80
+ echarts_spec: "{
81
+ xAxis: {
82
+ type: 'category',
83
+ data: SERIES_DATA.map(row => row['Status'])
84
+ /* data: ['100', '101', '102', '103',
85
+ '200', '201', '202', '203', '204', '205', '206', '207', '208', '226',
86
+ '300', '301', '302', '303', '304', '305', '306', '307', '308',
87
+ '400', '401', '402', '403', '404', '405', '406', '407', '408', '409', '410', '411', '412', '413', '414', '415', '416', '417', '418', '421', '422', '423', '424', '425', '426', '428', '429', '431', '451',
88
+ '500', '501', '502', '503', '504', '505', '506', '507', '508', '510', '511'] */
89
+ },
90
+ yAxis: {
91
+ type: 'value'
92
+ },
93
+ tooltip: {
94
+ trigger: 'axis'
95
+ },
96
+ series: [
97
+ {
98
+ data: SERIES_DATA.map(row => row['Count']),
99
+ type: 'bar',
100
+ color: '#D30001',
101
+ label: {
102
+ show: true,
103
+ position: 'top'
104
+ },
105
+ }
106
+ ]
107
+ }",
96
108
  },
97
109
  {
98
110
  title: "Rails Performance",
99
111
  header: %w[Controller Hits Min Avg Max],
100
112
  column_alignment: %i[left right right right right],
101
113
  rows: data[:performance],
102
- vega_spec: {
103
- "layer": [
104
- {
105
- "mark": { "type": "point",
106
- "name": "data_points"
107
- }
108
- },
109
- {
110
- "mark": { "name": "label",
111
- "type": "text",
112
- "align": "left",
113
- "baseline": "middle",
114
- "dx": 5,
115
- "yOffset": 0
116
- },
117
- "encoding": { "text": {"field": "Controller"},
118
- "fontSize": {"value": 8}
119
- },
120
- },
121
- ],
122
- "encoding": { "x": { "field": "Avg",
123
- "type": "quantitative"
124
- },
125
- "y": { "field": "Hits",
126
- "type": "quantitative"
127
- }
128
- },
129
- }
114
+ col: "small-12 cell",
115
+ echarts_height: "600px",
116
+ echarts_spec: "{
117
+ xAxis: {
118
+ },
119
+ yAxis: {
120
+ },
121
+ tooltip: {
122
+ trigger: 'axis'
123
+ },
124
+ series: [
125
+ {
126
+ data: SERIES_DATA.map(row => [row['Avg'], row['Hits']]),
127
+ type: 'scatter',
128
+ color: '#D30001',
129
+ label: {
130
+ show: true,
131
+ position: 'right',
132
+ formatter: function (params) {
133
+ var row = SERIES_DATA[params.dataIndex]
134
+ return row['Controller'] +
135
+ ': (' + row['Avg'] + ', ' + row['Hits'] + ')';
136
+ }
137
+ },
138
+ }
139
+ ],
140
+ dataZoom: [
141
+ {
142
+ type: 'slider', // slider zooming tool on the x-axis
143
+ xAxisIndex: 0
144
+ },
145
+ {
146
+ type: 'slider', // slider zooming tool on the y-axis
147
+ yAxisIndex: 0
148
+ },
149
+ /*
150
+ {
151
+ type: 'inside', // zooming and panning by dragging and scrolling inside the chart
152
+ xAxisIndex: 0
153
+ },
154
+ {
155
+ type: 'inside', // zooming and panning by dragging and scrolling inside the chart
156
+ yAxisIndex: 0
157
+ }
158
+ */
159
+ ]
160
+ }",
161
+ },
162
+ {
163
+ title: "Controller and Methods by Device",
164
+ header: %w[Controller Method Format iOS Android Mac Windows Linux Other Total],
165
+ column_alignment: %i[left left left right right right right right right right],
166
+ rows: data[:controller_and_methods_by_device],
167
+ col: "small-12 cell",
168
+ echarts_height: "600px",
169
+ echarts_spec: "{
170
+ dataZoom: [{
171
+ type: 'slider', // Use slider type for horizontal zoom
172
+ // ... other dataZoom options
173
+ }],
174
+ series: {
175
+ type: 'treemap',
176
+ roam: 'move',
177
+ label: {
178
+ show: true,
179
+ formatter: function (params) {
180
+ var parentName = params.treePathInfo.length > 1 ? params.treePathInfo[params.treePathInfo.length - 2].name : '';
181
+ var nodeName = params.name;
182
+ var value = params.value;
183
+ return parentName + '\\n' + nodeName + ':\\n' + value;
184
+ }
185
+ },
186
+ data: #{data[:controller_and_methods_treemap].to_json}
187
+ }
188
+ }"
130
189
  },
131
190
  {
132
191
  title: "Fatal Events",
133
192
  header: %w[Date IP URL Description Log ID],
134
193
  column_alignment: %i[left left left left left],
135
194
  rows: data[:fatal],
136
- col: "small-12 cell"
195
+ col: "small-12 cell",
196
+ echarts_extra: "var fatal_plot=#{data[:fatal_plot].to_json}",
197
+ echarts_spec: "{
198
+ toolbox: {
199
+ feature: {
200
+ saveAsImage: {},
201
+ }
202
+ },
203
+ tooltip: {
204
+ trigger: 'axis'
205
+ },
206
+ xAxis: {
207
+ type: 'category',
208
+ data: fatal_plot.filter(row => row[0] != '').map(row => row[0]),
209
+ showGrid: true,
210
+ axisLabel: {
211
+ rotate: 45 // Rotate the labels by 90 degrees
212
+ }
213
+ },
214
+ yAxis: {
215
+ type: 'value',
216
+ name: 'Errors',
217
+ showGrid: true,
218
+ },
219
+ series: [
220
+ {
221
+ data: fatal_plot.filter(row => row[0] != '').map(row => row[1]),
222
+ type: 'bar',
223
+ color: '#D30001',
224
+ label: {
225
+ show: true,
226
+ position: 'top'
227
+ },
228
+ },
229
+ ]
230
+ };"
137
231
  },
138
232
  {
139
233
  title: "Internal Server Errors",
@@ -144,10 +238,106 @@ module LogSense
144
238
  },
145
239
  {
146
240
  title: "Errors",
147
- header: %w[Log ID Context Description Count],
148
- column_alignment: %i[left left left left],
241
+ header: %w[Log ID Description Count],
242
+ column_alignment: %i[left left left],
149
243
  rows: data[:error],
150
- col: "small-12 cell"
244
+ col: "large-6 cell"
245
+ },
246
+ {
247
+ title: "Possible Attacks",
248
+ header: %w[Log ID Description Count],
249
+ column_alignment: %i[left left left],
250
+ rows: data[:possible_attacks],
251
+ col: "large-6 cell"
252
+ },
253
+ {
254
+ title: "Browsers",
255
+ header: %w[Browser Visits],
256
+ column_alignment: %i[left right],
257
+ rows: data[:browsers],
258
+ echarts_spec: "{
259
+ toolbox: {
260
+ feature: {
261
+ saveAsImage: {},
262
+ }
263
+ },
264
+ tooltip: {
265
+ trigger: 'axis'
266
+ },
267
+ xAxis: {
268
+ type: 'category',
269
+ data: SERIES_DATA.sort(order_by_name).map(row => row['Browser']),
270
+ showGrid: true,
271
+ axisLabel: {
272
+ rotate: 45 // Rotate the labels by 90 degrees
273
+ }
274
+ },
275
+ yAxis: {
276
+ type: 'value',
277
+ name: 'Browser Visits',
278
+ showGrid: true,
279
+ },
280
+ series: [
281
+ {
282
+ name: 'Hits',
283
+ data: SERIES_DATA.sort(order_by_name).map(row => row['Visits']),
284
+ type: 'bar',
285
+ color: '#D30001',
286
+ label: {
287
+ show: true,
288
+ position: 'top'
289
+ },
290
+ },
291
+ ]
292
+ }
293
+ function order_by_name(a, b) {
294
+ return a['Browser'] < b['Browser'] ? -1 : 1
295
+ }
296
+ ",
297
+ },
298
+ {
299
+ title: "Platforms",
300
+ header: %w[Platform Visits],
301
+ column_alignment: %i[left right],
302
+ rows: data[:platforms],
303
+ echarts_spec: "{
304
+ toolbox: {
305
+ feature: {
306
+ saveAsImage: {},
307
+ }
308
+ },
309
+ tooltip: {
310
+ trigger: 'axis'
311
+ },
312
+ xAxis: {
313
+ type: 'category',
314
+ data: SERIES_DATA.sort(order_by_platform).map(row => row['Platform']),
315
+ showGrid: true,
316
+ axisLabel: {
317
+ rotate: 45 // Rotate the labels by 90 degrees
318
+ }
319
+ },
320
+ yAxis: {
321
+ type: 'value',
322
+ name: 'Platform Visits',
323
+ showGrid: true,
324
+ },
325
+ series: [
326
+ {
327
+ name: 'Visits',
328
+ data: SERIES_DATA.sort(order_by_platform).map(row => row['Visits']),
329
+ type: 'bar',
330
+ color: '#D30001',
331
+ label: {
332
+ show: true,
333
+ position: 'top'
334
+ },
335
+ },
336
+ ]
337
+ }
338
+ function order_by_platform(a, b) {
339
+ return a['Platform'] < b['Platform'] ? -1 : 1
340
+ }",
151
341
  },
152
342
  {
153
343
  title: "IPs",
@@ -159,7 +349,40 @@ module LogSense
159
349
  title: "Countries",
160
350
  header: ["Country", "Hits", "IPs", "IP List"],
161
351
  column_alignment: %i[left right left],
162
- rows: countries_table(data[:countries])
352
+ rows: countries_table(data[:countries]),
353
+ echarts_height: "600px",
354
+ echarts_spec: "{
355
+ tooltip: {
356
+ trigger: 'axis',
357
+ axisPointer: {
358
+ type: 'shadow'
359
+ }
360
+ },
361
+ xAxis: {
362
+ type: 'value',
363
+ boundaryGap: [0, 0.01]
364
+ },
365
+ yAxis: {
366
+ type: 'category',
367
+ data: SERIES_DATA.sort(order_by_hits).map(row => row['Country'] ),
368
+ },
369
+ series: [
370
+ {
371
+ type: 'bar',
372
+ data: SERIES_DATA.sort(order_by_hits).map(row => row['Hits'] ),
373
+ color: '#D30001',
374
+ label: {
375
+ show: true,
376
+ position: 'right'
377
+ },
378
+ },
379
+ ]
380
+ };
381
+
382
+ function order_by_hits(a, b) {
383
+ return Number(a['Hits']) < Number(b['Hits']) ? -1 : 1
384
+ }
385
+ "
163
386
  },
164
387
  ip_per_hour_report_spec(ips_per_hour(data[:ips_per_hour])),
165
388
  session_report_spec(ips_detailed(data[:ips_per_day_detailed]))
data/lib/log_sense/templates/_log_structure.html.erb CHANGED
@@ -1,32 +1,32 @@
1
1
  <ul class="stats-list">
2
2
  <li>
3
- <%= data[:first_day]&.strftime("%b %d, %Y") %>
4
3
  <span class="stats-list-label">From</span>
4
+ <%= data[:first_day]&.strftime("%b %d, %Y") %>
5
5
  </li>
6
6
  <li>
7
- <%= data[:last_day]&.strftime("%b %d, %Y") %>
8
7
  <span class="stats-list-label">To</span>
8
+ <%= data[:last_day]&.strftime("%b %d, %Y") %>
9
9
  </li>
10
10
  <li class="stats-list-positive">
11
- <%= data[:total_days] %>
12
11
  <span class="stats-list-label">Days in Log</span>
12
+ <%= data[:total_days] %>
13
13
  </li>
14
14
  <% if data[:log_size] %>
15
15
  <li class="stats-list-negative">
16
- <%= data[:log_size] %>
17
16
  <span class="stats-list-label">Total Entries</span>
17
+ <%= data[:log_size] %>
18
18
  </li>
19
19
  <% end %>
20
20
  <% if data[:selfpolls_size] %>
21
21
  <li class="stats-list-negative">
22
- <%= data[:selfpolls_size] %>
23
22
  <span class="stats-list-label">Self Polls</span>
23
+ <%= data[:selfpolls_size] %>
24
24
  </li>
25
25
  <% end %>
26
26
  <% if data[:crawlers] %>
27
27
  <li class="stats-list-negative">
28
- <td><%= data[:crawlers_size] %></td>
29
28
  <span class="stats-list-label">Crawlers</span>
29
+ <%= data[:crawlers_size] %>
30
30
  <% end %>
31
31
  </li>
32
32
  </ul>
data/lib/log_sense/templates/_performance.html.erb CHANGED
@@ -1,21 +1,22 @@
1
1
  <ul class="stats-list">
2
2
  <li>
3
- <%= data[:started_at].strftime("%b %d, %Y @ %H:%M:%S") %>
4
3
  <span class="stats-list-label">Analysis Started</span>
4
+ <%= data[:started_at].strftime("%b %d, %Y @ %H:%M:%S") %>
5
5
  </li>
6
6
  <li>
7
- <%= data[:ended_at].strftime("%b %d, %Y @ %H:%M:%S") %>
8
7
  <span class="stats-list-label">Analysis Ended</span>
8
+ <%= data[:ended_at].strftime("%b %d, %Y @ %H:%M:%S") %>
9
9
  </li>
10
10
  <li class="stats-list-negative">
11
- <%= "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] %>
12
11
  <span class="stats-list-label">Duration</span>
12
+ <%= "%02d:%02d" % [data[:duration] / 60, data[:duration] % 60] %>
13
13
  </li>
14
14
  <li class="stats-list-negative">
15
- <%= data[:log_size] %> <span class="stats-list-label">Events</span>
15
+ <span class="stats-list-label">Events</span>
16
+ <%= data[:log_size] %>
16
17
  </li>
17
18
  <li class="stats-list-positive">
18
- <%= "%.2f" % (data[:log_size] / data[:duration]) %>
19
19
  <span class="stats-list-label">Parsed Events/sec</span>
20
+ <%= "%.2f" % (data[:log_size] / data[:duration]) %>
20
21
  </li>
21
22
  </ul>
data/lib/log_sense/templates/_rails.css.erb CHANGED
@@ -1,8 +1,20 @@
1
+ :root {
2
+ --main-accent: #D30001;
3
+ }
4
+
5
+ /*
6
+ @media (prefers-color-scheme: dark) {
7
+ :root {
8
+ --main-accent: #DA2627;
9
+ }
10
+ }
11
+ */
12
+
1
13
  #offCanvas {
2
- background: #d30001 !important;
14
+ background: var(--main-accent) !important;
3
15
  }
4
16
 
5
17
  h1, h2 {
6
- color: #d30001 !important;
18
+ color: var(--main-accent) !important;
7
19
  }
8
20
 
data/lib/log_sense/templates/_stylesheet.css CHANGED
@@ -1,3 +1,19 @@
1
+ :root {
2
+ --color: #222222;
3
+ --background: white;
4
+ --main-accent: #1C1C1C;
5
+ }
6
+
7
+ /*
8
+ @media (prefers-color-scheme: dark) {
9
+ :root {
10
+ --color: white;
11
+ --background: #222222;
12
+ --main-accent: #E3E3E3
13
+ }
14
+ }
15
+ */
16
+
1
17
  :root {
2
18
  --font-family: Inter, Helvetica, Arial, sans-serif;
3
19
 
@@ -7,11 +23,13 @@ body {
7
23
  font-family: var(--font-family);
8
24
  /* font-size: 12px; */
9
25
  font-size: 0.86rem;
26
+ color: var(--color);
27
+ background: var(--background);
10
28
  }
11
29
 
12
30
  #offCanvas {
13
- color: #DEDEDE;
14
- background: #1C1C1C;
31
+ color: var(--background);
32
+ background: var(--color);
15
33
  border-right: none;
16
34
  box-shadow: none;
17
35
  padding: 0.5rem;
@@ -20,7 +38,7 @@ body {
20
38
  }
21
39
 
22
40
  #offCanvas a {
23
- color: #FFFFFF;
41
+ color: var(--main-accent);
24
42
  }
25
43
 
26
44
  nav {
@@ -43,14 +61,14 @@ h1 {
43
61
  font-family: var(--font-family);
44
62
  font-size: 2rem;
45
63
  font-weight: bold;
46
- color: #1C1C1C;
64
+ color: var(--main-accent);
47
65
  }
48
66
 
49
67
  h2 {
50
68
  font-family: var(--font-family);
51
69
  font-size: 1.6rem;
52
70
  font-weight: bold;
53
- color: #1C1C1C;
71
+ color: var(--main-accent);
54
72
  }
55
73
 
56
74
  th, td {
@@ -116,9 +134,9 @@ ul.pagination, li.paginate_button {
116
134
  display: inline-block;
117
135
  margin-right: 10px;
118
136
  padding-right: 10px;
119
- border-right: 1px solid #cacaca;
137
+ border-right: 1px solid #CACACA;
120
138
  text-align: center;
121
- font-size: 1.1em;
139
+ font-size: 1.2em;
122
140
  font-weight: bold;
123
141
  }
124
142
 
@@ -131,7 +149,7 @@ ul.pagination, li.paginate_button {
131
149
  .stats-list > li .stats-list-label {
132
150
  display: block;
133
151
  margin-top: 2px;
134
- font-size: 0.9em;
152
+ font-size: 0.8em;
135
153
  font-weight: normal;
136
154
  }
137
155