log_sense 1.1.1 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.org +6 -19
- data/README.org +23 -20
- data/lib/log_sense/rails_data_cruncher.rb +4 -0
- data/lib/log_sense/templates/rails.html.erb +347 -0
- data/lib/log_sense/version.rb +1 -1
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d20e6ee8a3f63c660de0bb9943255ed4c19b1da9e895bc269ff40310ec8f8796
|
4
|
+
data.tar.gz: 38338745e085de9b8ab82231dab82405f2933584d7d1dca4886b692a4460e2c0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dbc95e9b65119822687af5a20ca6ea25958b5d427b2e7ca128ac0513c20faebe4f3cd00dcabfde7dc216def72526de67bf21b4afa87883101f96144e842ad7e1
|
7
|
+
data.tar.gz: dd2faf369c1f0bcf10ed151215e4521eacec4c1d46d5aa616f34158f33f03e24e515a0677c0cf9905c11359ffebeefb96fac868ed5b717358359527cedb39691
|
data/CHANGELOG.org
CHANGED
@@ -2,26 +2,13 @@
|
|
2
2
|
#+AUTHOR: Adolfo Villafiorita
|
3
3
|
#+STARTUP: showall
|
4
4
|
|
5
|
-
*
|
5
|
+
* Changes in log_sense 1.1.2
|
6
|
+
<2021-12-17 Fri>
|
6
7
|
|
7
|
-
|
8
|
+
- Added Rails Log HTML output
|
8
9
|
|
9
|
-
|
10
|
+
* Changes in log_sense 1.1.1 and earlier
|
11
|
+
<2021-12-17 Fri>
|
10
12
|
|
11
|
-
|
12
|
-
|
13
|
-
** Documentation
|
14
|
-
|
15
|
-
** Code
|
16
|
-
|
17
|
-
|
18
|
-
* Version 1.0.0
|
19
|
-
|
20
|
-
** New Functions and Changes
|
21
|
-
|
22
|
-
** Fixes
|
23
|
-
|
24
|
-
** Documentation
|
25
|
-
|
26
|
-
** Code
|
13
|
+
- In the Git commit messages (not very informative, I am afraid).
|
27
14
|
|
data/README.org
CHANGED
@@ -14,21 +14,15 @@ and [[https://umami.is/][Umami]], focusing on privacy and data-ownership: the da
|
|
14
14
|
generated by LogSense is stored on your computer and owned by
|
15
15
|
you (like it should be).
|
16
16
|
|
17
|
-
LogSense is also inspired by static websites generators
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
installation headaches.
|
17
|
+
LogSense is also inspired by *static websites generators*: statistics
|
18
|
+
are generated from the command line and accessed as static HTML files.
|
19
|
+
By generating static resources, LogSense significantly reduces the
|
20
|
+
attack surface of your webserver and installation headaches.
|
22
21
|
|
23
22
|
We have, for instance, a cron job running on our servers, generating
|
24
23
|
statistics at night. The generated files are then made available on a
|
25
24
|
private area on the web.
|
26
25
|
|
27
|
-
Statistics are generated from Apache log formats in the =combined=
|
28
|
-
format and from Rails logs. Reports are tailored, but not limited, to
|
29
|
-
web servers serving static websites. No need to install Java Script
|
30
|
-
code on your websites, no cookies installed, no user tracking.
|
31
|
-
|
32
26
|
LogSense reports the following data:
|
33
27
|
|
34
28
|
- Visitors, hits, unique visitors, bandwidth used
|
@@ -62,20 +56,29 @@ LogSense generates HTML, txt (Org Mode), and SQLite outputs.
|
|
62
56
|
|
63
57
|
#+RESULTS:
|
64
58
|
#+begin_example
|
65
|
-
Usage:
|
66
|
-
|
59
|
+
Usage: log_sense [options] [logfile]
|
60
|
+
--title=TITLE Title to use in the report
|
61
|
+
-f, --input-format=FORMAT Input format (either rails or apache)
|
62
|
+
-i, --input-file=INPUT_FILE Input file
|
63
|
+
-t, --output-format=FORMAT Output format: html, org, txt, sqlite. See below for available formats
|
64
|
+
-o, --output-file=OUTPUT_FILE Output file
|
67
65
|
-b, --begin=DATE Consider entries after or on DATE
|
68
66
|
-e, --end=DATE Consider entries before or on DATE
|
69
|
-
-
|
70
|
-
-
|
71
|
-
|
72
|
-
-u, --prefix=PREFIX Prefix to add to all plots (used to run multiple analyses in the same dir)
|
73
|
-
-w, --suffix=SUFFIX Suffix to add to all plots (used to run multiple analyses in the same dir)
|
74
|
-
-c, --code-export=WHAT Control :export directive in Org Mode code blocks (code, results, *both*, none)
|
75
|
-
-f, --format=FORMAT Output format: html, org, sqlite. Defaults to org mode
|
67
|
+
-l, --limit=N Number of entries to show (defaults to 30)
|
68
|
+
-c, --crawlers=POLICY Decide what to do with crawlers (applies to Apache Logs)
|
69
|
+
-n, --no-selfpolls Ignore self poll entries (requests from ::1; applies to Apache Logs)
|
76
70
|
-v, --version Prints version information
|
77
71
|
-h, --help Prints this help
|
78
|
-
|
72
|
+
|
73
|
+
This is version 1.1.1
|
74
|
+
|
75
|
+
Output formats
|
76
|
+
apache parsing can produce the following outputs:
|
77
|
+
- sqlite
|
78
|
+
- html
|
79
|
+
rails parsing can produce the following outputs:
|
80
|
+
- sqlite
|
81
|
+
- txt
|
79
82
|
#+end_example
|
80
83
|
|
81
84
|
* Change Log
|
@@ -26,6 +26,10 @@ module LogSense
|
|
26
26
|
@log_size = db.execute "SELECT count(started_at) from Event"
|
27
27
|
@log_size = @log_size[0][0]
|
28
28
|
|
29
|
+
# TODO: I should make the names of events/size/etc uniform betweeen Apache and Rails Logs
|
30
|
+
# SAME AS ABOVE
|
31
|
+
@total_hits = @log_size
|
32
|
+
|
29
33
|
# SAME AS ABOVE (but log_size is wrong in the case of Rails
|
30
34
|
# logs, since an event takes more than one line)
|
31
35
|
@events = db.execute "SELECT count(started_at) from Event"
|
@@ -0,0 +1,347 @@
|
|
1
|
+
<!doctype html>
|
2
|
+
<html class="no-js" lang="en">
|
3
|
+
<head>
|
4
|
+
<title><%= options[:title] || "Log Sense: #{data[:log_file]}" %></title>
|
5
|
+
|
6
|
+
<meta charset="utf-8" />
|
7
|
+
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
8
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
9
|
+
<meta name="author" content="Log Sense">
|
10
|
+
<meta name="description" content="Analysis of <%= data[:log_file] %>">
|
11
|
+
|
12
|
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
13
|
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
14
|
+
<link href="https://fonts.googleapis.com/css2?family=PT+Sans&display=swap" rel="stylesheet">
|
15
|
+
|
16
|
+
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/foundicons/3.0.0/foundation-icons.min.css">
|
17
|
+
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/css/foundation.min.css">
|
18
|
+
<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.css"/>
|
19
|
+
|
20
|
+
|
21
|
+
<script src="https://cdn.jsdelivr.net/npm/vega@5.21.0"></script>
|
22
|
+
<script src="https://cdn.jsdelivr.net/npm/vega-lite@5.2.0"></script>
|
23
|
+
<script src="https://cdn.jsdelivr.net/npm/vega-embed@6.20.2"></script>
|
24
|
+
|
25
|
+
<style>
|
26
|
+
body {
|
27
|
+
font-family: 'PT Sans', sans-serif;
|
28
|
+
font-size: 80%;
|
29
|
+
}
|
30
|
+
|
31
|
+
#offCanvas {
|
32
|
+
color: white;
|
33
|
+
background: #0D0630;
|
34
|
+
border-right: none;
|
35
|
+
box-shadow: none;
|
36
|
+
padding: 0.5rem;
|
37
|
+
}
|
38
|
+
#offCanvas a {
|
39
|
+
color: #E6F9AF;
|
40
|
+
}
|
41
|
+
|
42
|
+
.contents-button {
|
43
|
+
font-size: xx-large;
|
44
|
+
}
|
45
|
+
|
46
|
+
.main-section {
|
47
|
+
margin-left: 45px;
|
48
|
+
}
|
49
|
+
|
50
|
+
h1 {
|
51
|
+
font-size: 1.8rem;
|
52
|
+
}
|
53
|
+
|
54
|
+
h2 {
|
55
|
+
font-size: 1.2rem;
|
56
|
+
}
|
57
|
+
|
58
|
+
th {
|
59
|
+
padding: 0.2rem 1.2rem 0.2rem 0.2rem !important
|
60
|
+
}
|
61
|
+
|
62
|
+
td {
|
63
|
+
padding: 0.2rem 1rem 0.2rem 0.2rem !important;
|
64
|
+
}
|
65
|
+
|
66
|
+
.hits, .visits, .size, .count, .s2xx, .s3xx, .so4xx, .total-hits, .total-visits {
|
67
|
+
text-align: right !important;
|
68
|
+
}
|
69
|
+
|
70
|
+
.card-divider {
|
71
|
+
padding: 0.2rem 0.4rem 0.2rem 0.4rem;
|
72
|
+
background: #0d0630;
|
73
|
+
color: white;
|
74
|
+
}
|
75
|
+
|
76
|
+
input, select {
|
77
|
+
font-size: 0.8rem !important;
|
78
|
+
height: 1.5rem !important;
|
79
|
+
padding: 0.2rem 0.4rem 0.2rem 0.4rem !important;
|
80
|
+
}
|
81
|
+
|
82
|
+
.dataTables_info {
|
83
|
+
font-size: small;
|
84
|
+
color: rgb(202, 202, 202);
|
85
|
+
}
|
86
|
+
|
87
|
+
ul.pagination, li.paginate_button {
|
88
|
+
font-size: small;
|
89
|
+
margin-top: 0px !important;
|
90
|
+
margin-bottom: 0px !important;
|
91
|
+
padding-top: 0px !important;
|
92
|
+
padding-bottom: 0px !important;
|
93
|
+
}
|
94
|
+
|
95
|
+
.stats-list {
|
96
|
+
list-style-type: none;
|
97
|
+
clear: left;
|
98
|
+
margin: 0;
|
99
|
+
padding: 0;
|
100
|
+
text-align: center;
|
101
|
+
margin-bottom: 30px;
|
102
|
+
}
|
103
|
+
|
104
|
+
.stats-list .stats-list-positive {
|
105
|
+
color: #228b22;
|
106
|
+
}
|
107
|
+
|
108
|
+
.stats-list .stats-list-negative {
|
109
|
+
color: #a52a2a;
|
110
|
+
}
|
111
|
+
|
112
|
+
.stats-list > li {
|
113
|
+
display: inline-block;
|
114
|
+
margin-right: 10px;
|
115
|
+
padding-right: 10px;
|
116
|
+
border-right: 1px solid #cacaca;
|
117
|
+
text-align: center;
|
118
|
+
font-size: 1.1em;
|
119
|
+
font-weight: bold;
|
120
|
+
}
|
121
|
+
|
122
|
+
.stats-list > li:last-child {
|
123
|
+
border: none;
|
124
|
+
margin: 0;
|
125
|
+
padding: 0;
|
126
|
+
}
|
127
|
+
|
128
|
+
.stats-list > li .stats-list-label {
|
129
|
+
display: block;
|
130
|
+
margin-top: 2px;
|
131
|
+
font-size: 0.9em;
|
132
|
+
font-weight: normal;
|
133
|
+
}
|
134
|
+
|
135
|
+
#streaks-table .ip {
|
136
|
+
vertical-align: top;
|
137
|
+
}
|
138
|
+
#streaks-table .date {
|
139
|
+
font-weight: bold;
|
140
|
+
}
|
141
|
+
#streaks-table .res-title {
|
142
|
+
font-decoration: underline;
|
143
|
+
}
|
144
|
+
</style>
|
145
|
+
|
146
|
+
</head>
|
147
|
+
|
148
|
+
<body>
|
149
|
+
<div class="off-canvas-wrapper">
|
150
|
+
<div class="off-canvas position-left" id="offCanvas" data-off-canvas>
|
151
|
+
<nav>
|
152
|
+
<h2>Navigation</h2>
|
153
|
+
<ul class="no-bullet">
|
154
|
+
<% [
|
155
|
+
"Summary",
|
156
|
+
"Log Structure",
|
157
|
+
"Daily Distribution",
|
158
|
+
"Time Distribution",
|
159
|
+
"Most Requested Pages",
|
160
|
+
"Most Requested Resources",
|
161
|
+
"404 on HTML Files",
|
162
|
+
"404 on other Resources",
|
163
|
+
"Attacks",
|
164
|
+
"Statuses",
|
165
|
+
"Daily Statuses",
|
166
|
+
"Browsers",
|
167
|
+
"Platforms",
|
168
|
+
"Referers",
|
169
|
+
"IPs",
|
170
|
+
"Geolocation",
|
171
|
+
"Streaks",
|
172
|
+
"Command Invocation",
|
173
|
+
"Performance"
|
174
|
+
].each do |item| %>
|
175
|
+
<li class="nav-item">
|
176
|
+
<a href="#<%= item.downcase.gsub(' ', '-') %>" data-close><%= item %></a>
|
177
|
+
</li>
|
178
|
+
<% end %>
|
179
|
+
</ul>
|
180
|
+
|
181
|
+
<p>
|
182
|
+
Generated by
|
183
|
+
<a href="https://github.com/avillafiorita/log_sense">LogSense</a> <br />
|
184
|
+
on <%= DateTime.now.strftime("%Y-%m-%d %H:%M") %>.<br />
|
185
|
+
<a href='https://db-ip.com'>IP Geolocation by DB-IP</a>
|
186
|
+
</p>
|
187
|
+
</nav>
|
188
|
+
</div>
|
189
|
+
<div class="off-canvas-content grid-container grid-x fluid" data-off-canvas-content>
|
190
|
+
<div data-sticky-container>
|
191
|
+
<div class="sticky" data-sticky data-margin-top="0">
|
192
|
+
<div class="contents-button">
|
193
|
+
<i id="hamburger" class="fi-list" data-toggle="offCanvas"></i>
|
194
|
+
</div>
|
195
|
+
</div>
|
196
|
+
</div>
|
197
|
+
|
198
|
+
<section class="main-section">
|
199
|
+
<h1><%= options[:title] || "Log Sense: #{data[:log_file]}" %></h1>
|
200
|
+
|
201
|
+
<p><b>Input File:</b> <%= (data[:log_file] || "stdin") %></p>
|
202
|
+
|
203
|
+
<div class="grid-x grid-margin-x">
|
204
|
+
<article class="card small-12 large-6 cell">
|
205
|
+
<div class="card-divider">
|
206
|
+
<h2 id="summary">Summary</h2>
|
207
|
+
</div>
|
208
|
+
<div class="card-section">
|
209
|
+
<%= render "summary.html.erb", data: data %>
|
210
|
+
</div>
|
211
|
+
</article>
|
212
|
+
|
213
|
+
<article class="card cell small-12 large-6">
|
214
|
+
<div class="card-divider">
|
215
|
+
<h2 id="log-structure">Log Structure</h2>
|
216
|
+
</div>
|
217
|
+
<div class="card-section">
|
218
|
+
<%= render "log_structure.html.erb", data: data %>
|
219
|
+
</div>
|
220
|
+
</article>
|
221
|
+
</div>
|
222
|
+
|
223
|
+
<% @reports = [
|
224
|
+
{ title: "Daily Distribution",
|
225
|
+
header: ["Day", "DOW", "Hits"],
|
226
|
+
rows: data[:daily_distribution],
|
227
|
+
vega_spec: {
|
228
|
+
"mark": {
|
229
|
+
"type": "line",
|
230
|
+
"point": {
|
231
|
+
"filled": false,
|
232
|
+
"fill": "white"
|
233
|
+
}
|
234
|
+
},
|
235
|
+
"encoding": {
|
236
|
+
"x": {"field": "Day", "type": "temporal"},
|
237
|
+
"y": {"field": "Hits", "type": "quantitative"}
|
238
|
+
}
|
239
|
+
}
|
240
|
+
},
|
241
|
+
{ title: "Time Distribution",
|
242
|
+
header: ["Hour", "Hits"],
|
243
|
+
rows: data[:time_distribution],
|
244
|
+
vega_spec: {
|
245
|
+
"mark": "bar",
|
246
|
+
"encoding": {
|
247
|
+
"x": {"field": "Hour", "type": "nominal"},
|
248
|
+
"y": {"field": "Hits", "type": "quantitative"}
|
249
|
+
}
|
250
|
+
}
|
251
|
+
},
|
252
|
+
{ title: "Statuses",
|
253
|
+
header: ["Status", "Count"],
|
254
|
+
rows: data[:statuses],
|
255
|
+
vega_spec: {
|
256
|
+
"mark": "bar",
|
257
|
+
"encoding": {
|
258
|
+
"x": {"field": "Status", "type": "nominal"},
|
259
|
+
"y": {"field": "Count", "type": "quantitative"}
|
260
|
+
}
|
261
|
+
}
|
262
|
+
},
|
263
|
+
{ title: "Rails Performance",
|
264
|
+
header: ['Controller', 'Hits', 'Min', 'Avg', 'Max'],
|
265
|
+
rows: @data[:performance] },
|
266
|
+
{ title: "Fatal Events",
|
267
|
+
header: ['Date', 'IP', 'URL', 'Log ID'], rows: @data[:fatal]
|
268
|
+
},
|
269
|
+
{ title: "IPs", header: ["IPs", "Hits", "Country"], rows: data[:ips] },
|
270
|
+
]
|
271
|
+
%>
|
272
|
+
<div class="grid-x grid-margin-x">
|
273
|
+
<% @reports.each_with_index do |report, index| %>
|
274
|
+
<article class="card cell <%= report[:col] || "small-12 large-6" %>" >
|
275
|
+
<div class="card-divider">
|
276
|
+
<h2>
|
277
|
+
<%= report[:title] %>
|
278
|
+
</h2>
|
279
|
+
</div>
|
280
|
+
|
281
|
+
<% if report[:vega_spec] %>
|
282
|
+
<div id="<%= "plot-#{index}" %>"></div>
|
283
|
+
<script>
|
284
|
+
plot_spec_<%= index %> = Object.assign(
|
285
|
+
<%= report[:vega_spec].to_json %>,
|
286
|
+
{ "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
|
287
|
+
width: "container",
|
288
|
+
description: "<%= report[:title] %>",
|
289
|
+
data: {
|
290
|
+
values: [
|
291
|
+
<% report[:rows].each do |row| %>
|
292
|
+
{
|
293
|
+
<% report[:header].each_with_index do |h, i| %>
|
294
|
+
"<%= h %>": <%= (row[i].class == Integer or row[i].class == Float) ? row[i] : "\"#{row[i]}\"" %>,
|
295
|
+
<% end %>
|
296
|
+
},
|
297
|
+
<% end %>
|
298
|
+
]
|
299
|
+
},
|
300
|
+
});
|
301
|
+
vegaEmbed('#<%= "plot-#{index}"%>', plot_spec_<%= index %>);
|
302
|
+
</script>
|
303
|
+
<% end %>
|
304
|
+
<div class="card-section">
|
305
|
+
<%= render "output_table.html.erb", report %>
|
306
|
+
</div>
|
307
|
+
</article>
|
308
|
+
<% end %>
|
309
|
+
</div>
|
310
|
+
|
311
|
+
<div class="grid-x grid-margin-x">
|
312
|
+
<div class="cell small-12 large-6">
|
313
|
+
<article>
|
314
|
+
<h2 id="command-invocation">Command Invocation</h2>
|
315
|
+
|
316
|
+
<%= render "command_invocation.html.erb", data: data, options: options %>
|
317
|
+
</article>
|
318
|
+
</div>
|
319
|
+
|
320
|
+
<div class="small-12 large-6 cell">
|
321
|
+
<article>
|
322
|
+
<h2 id="performance"> Performance</h2>
|
323
|
+
|
324
|
+
<%= render "performance.html.erb", data: data %>
|
325
|
+
</article>
|
326
|
+
</div>
|
327
|
+
</div>
|
328
|
+
</section>
|
329
|
+
</div>
|
330
|
+
|
331
|
+
<script type="text/javascript" src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
|
332
|
+
<script type="text/javascript" src="js/vendor/what-input.js"></script>
|
333
|
+
<script type="text/javascript" src="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.js"></script>
|
334
|
+
<script src="https://cdn.jsdelivr.net/npm/vega@5"></script>
|
335
|
+
<script type="text/javascript" src="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/js/foundation.min.js" crossorigin="anonymous"></script>
|
336
|
+
<script>
|
337
|
+
$(document).foundation();
|
338
|
+
|
339
|
+
$(document).ready(function () {
|
340
|
+
$('.data-table').each(function () {
|
341
|
+
$(this).DataTable();
|
342
|
+
});
|
343
|
+
});
|
344
|
+
</script>
|
345
|
+
</div>
|
346
|
+
</body>
|
347
|
+
</html>
|
data/lib/log_sense/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log_sense
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adolfo Villafiorita
|
@@ -159,6 +159,7 @@ files:
|
|
159
159
|
- lib/log_sense/templates/_summary.html.erb
|
160
160
|
- lib/log_sense/templates/_summary.txt.erb
|
161
161
|
- lib/log_sense/templates/apache.html.erb
|
162
|
+
- lib/log_sense/templates/rails.html.erb
|
162
163
|
- lib/log_sense/templates/rails.txt.erb
|
163
164
|
- lib/log_sense/version.rb
|
164
165
|
- log_sense.gemspec
|