log_sense 1.1.1 → 1.1.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cc9dd86438ff397137ef7981c567e8b52e06a74c80e7dbcceba15d9c389f5e58
4
- data.tar.gz: eac5ab610141d59a9f1cd95e68e40cce835c9a8280be52986f0dfccf4c019955
3
+ metadata.gz: d20e6ee8a3f63c660de0bb9943255ed4c19b1da9e895bc269ff40310ec8f8796
4
+ data.tar.gz: 38338745e085de9b8ab82231dab82405f2933584d7d1dca4886b692a4460e2c0
5
5
  SHA512:
6
- metadata.gz: fb94834ee784c25411c36d0490747c05d97cd9fe6d22cbc7b8c6d7734615b3095beddeaf1245466cf7f7e6148a6cbf500cd76434735bba568125919d1be0cef1
7
- data.tar.gz: b5d66648d8fb0815f71386a4fcaf6eebc071b0408dff77d7a488fb2f348773c33765c120a88238acb52e7878c3a9fcf5a9c2bdc9e73346097021ece83b2090a6
6
+ metadata.gz: dbc95e9b65119822687af5a20ca6ea25958b5d427b2e7ca128ac0513c20faebe4f3cd00dcabfde7dc216def72526de67bf21b4afa87883101f96144e842ad7e1
7
+ data.tar.gz: dd2faf369c1f0bcf10ed151215e4521eacec4c1d46d5aa616f34158f33f03e24e515a0677c0cf9905c11359ffebeefb96fac868ed5b717358359527cedb39691
data/CHANGELOG.org CHANGED
@@ -2,26 +2,13 @@
2
2
  #+AUTHOR: Adolfo Villafiorita
3
3
  #+STARTUP: showall
4
4
 
5
- * Unreleased
5
+ * Changes in log_sense 1.1.2
6
+ <2021-12-17 Fri>
6
7
 
7
- This changes are in the repository but not yet released to Rubygems.
8
+ - Added Rails Log HTML output
8
9
 
9
- ** New Functions and Changes
10
+ * Changes in log_sense 1.1.1 and earlier
11
+ <2021-12-17 Fri>
10
12
 
11
- ** Fixes
12
-
13
- ** Documentation
14
-
15
- ** Code
16
-
17
-
18
- * Version 1.0.0
19
-
20
- ** New Functions and Changes
21
-
22
- ** Fixes
23
-
24
- ** Documentation
25
-
26
- ** Code
13
+ - In the Git commit messages (not very informative, I am afraid).
27
14
 
data/README.org CHANGED
@@ -14,21 +14,15 @@ and [[https://umami.is/][Umami]], focusing on privacy and data-ownership: the da
14
14
  generated by LogSense is stored on your computer and owned by
15
15
  you (like it should be).
16
16
 
17
- LogSense is also inspired by static websites generators:
18
- statistics are generated from the command line and accessed as static
19
- HTML files. By generating static resources, LogSense
20
- significantly reduces the attack surface of your webserver and
21
- installation headaches.
17
+ LogSense is also inspired by *static websites generators*: statistics
18
+ are generated from the command line and accessed as static HTML files.
19
+ By generating static resources, LogSense significantly reduces the
20
+ attack surface of your webserver and installation headaches.
22
21
 
23
22
  We have, for instance, a cron job running on our servers, generating
24
23
  statistics at night. The generated files are then made available on a
25
24
  private area on the web.
26
25
 
27
- Statistics are generated from Apache log formats in the =combined=
28
- format and from Rails logs. Reports are tailored, but not limited, to
29
- web servers serving static websites. No need to install Java Script
30
- code on your websites, no cookies installed, no user tracking.
31
-
32
26
  LogSense reports the following data:
33
27
 
34
28
  - Visitors, hits, unique visitors, bandwidth used
@@ -62,20 +56,29 @@ LogSense generates HTML, txt (Org Mode), and SQLite outputs.
62
56
 
63
57
  #+RESULTS:
64
58
  #+begin_example
65
- Usage: apache_log_report [options] [logfile]
66
- -l, --limit=N Number of entries to show (defaults to 30)
59
+ Usage: log_sense [options] [logfile]
60
+ --title=TITLE Title to use in the report
61
+ -f, --input-format=FORMAT Input format (either rails or apache)
62
+ -i, --input-file=INPUT_FILE Input file
63
+ -t, --output-format=FORMAT Output format: html, org, txt, sqlite. See below for available formats
64
+ -o, --output-file=OUTPUT_FILE Output file
67
65
  -b, --begin=DATE Consider entries after or on DATE
68
66
  -e, --end=DATE Consider entries before or on DATE
69
- -i, --ignore-crawlers Ignore crawlers
70
- -p, --ignore-selfpoll Ignore apaches self poll entries (from ::1)
71
- --only-crawlers Perform analysis on crawlers only
72
- -u, --prefix=PREFIX Prefix to add to all plots (used to run multiple analyses in the same dir)
73
- -w, --suffix=SUFFIX Suffix to add to all plots (used to run multiple analyses in the same dir)
74
- -c, --code-export=WHAT Control :export directive in Org Mode code blocks (code, results, *both*, none)
75
- -f, --format=FORMAT Output format: html, org, sqlite. Defaults to org mode
67
+ -l, --limit=N Number of entries to show (defaults to 30)
68
+ -c, --crawlers=POLICY Decide what to do with crawlers (applies to Apache Logs)
69
+ -n, --no-selfpolls Ignore self poll entries (requests from ::1; applies to Apache Logs)
76
70
  -v, --version Prints version information
77
71
  -h, --help Prints this help
78
- This is version 1.1.6
72
+
73
+ This is version 1.1.1
74
+
75
+ Output formats
76
+ apache parsing can produce the following outputs:
77
+ - sqlite
78
+ - html
79
+ rails parsing can produce the following outputs:
80
+ - sqlite
81
+ - txt
79
82
  #+end_example
80
83
 
81
84
  * Change Log
@@ -26,6 +26,10 @@ module LogSense
26
26
  @log_size = db.execute "SELECT count(started_at) from Event"
27
27
  @log_size = @log_size[0][0]
28
28
 
29
+ # TODO: I should make the names of events/size/etc uniform betweeen Apache and Rails Logs
30
+ # SAME AS ABOVE
31
+ @total_hits = @log_size
32
+
29
33
  # SAME AS ABOVE (but log_size is wrong in the case of Rails
30
34
  # logs, since an event takes more than one line)
31
35
  @events = db.execute "SELECT count(started_at) from Event"
@@ -0,0 +1,347 @@
1
+ <!doctype html>
2
+ <html class="no-js" lang="en">
3
+ <head>
4
+ <title><%= options[:title] || "Log Sense: #{data[:log_file]}" %></title>
5
+
6
+ <meta charset="utf-8" />
7
+ <meta http-equiv="x-ua-compatible" content="ie=edge">
8
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
9
+ <meta name="author" content="Log Sense">
10
+ <meta name="description" content="Analysis of <%= data[:log_file] %>">
11
+
12
+ <link rel="preconnect" href="https://fonts.googleapis.com">
13
+ <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
14
+ <link href="https://fonts.googleapis.com/css2?family=PT+Sans&display=swap" rel="stylesheet">
15
+
16
+ <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/foundicons/3.0.0/foundation-icons.min.css">
17
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/css/foundation.min.css">
18
+ <link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.css"/>
19
+
20
+
21
+ <script src="https://cdn.jsdelivr.net/npm/vega@5.21.0"></script>
22
+ <script src="https://cdn.jsdelivr.net/npm/vega-lite@5.2.0"></script>
23
+ <script src="https://cdn.jsdelivr.net/npm/vega-embed@6.20.2"></script>
24
+
25
+ <style>
26
+ body {
27
+ font-family: 'PT Sans', sans-serif;
28
+ font-size: 80%;
29
+ }
30
+
31
+ #offCanvas {
32
+ color: white;
33
+ background: #0D0630;
34
+ border-right: none;
35
+ box-shadow: none;
36
+ padding: 0.5rem;
37
+ }
38
+ #offCanvas a {
39
+ color: #E6F9AF;
40
+ }
41
+
42
+ .contents-button {
43
+ font-size: xx-large;
44
+ }
45
+
46
+ .main-section {
47
+ margin-left: 45px;
48
+ }
49
+
50
+ h1 {
51
+ font-size: 1.8rem;
52
+ }
53
+
54
+ h2 {
55
+ font-size: 1.2rem;
56
+ }
57
+
58
+ th {
59
+ padding: 0.2rem 1.2rem 0.2rem 0.2rem !important
60
+ }
61
+
62
+ td {
63
+ padding: 0.2rem 1rem 0.2rem 0.2rem !important;
64
+ }
65
+
66
+ .hits, .visits, .size, .count, .s2xx, .s3xx, .so4xx, .total-hits, .total-visits {
67
+ text-align: right !important;
68
+ }
69
+
70
+ .card-divider {
71
+ padding: 0.2rem 0.4rem 0.2rem 0.4rem;
72
+ background: #0d0630;
73
+ color: white;
74
+ }
75
+
76
+ input, select {
77
+ font-size: 0.8rem !important;
78
+ height: 1.5rem !important;
79
+ padding: 0.2rem 0.4rem 0.2rem 0.4rem !important;
80
+ }
81
+
82
+ .dataTables_info {
83
+ font-size: small;
84
+ color: rgb(202, 202, 202);
85
+ }
86
+
87
+ ul.pagination, li.paginate_button {
88
+ font-size: small;
89
+ margin-top: 0px !important;
90
+ margin-bottom: 0px !important;
91
+ padding-top: 0px !important;
92
+ padding-bottom: 0px !important;
93
+ }
94
+
95
+ .stats-list {
96
+ list-style-type: none;
97
+ clear: left;
98
+ margin: 0;
99
+ padding: 0;
100
+ text-align: center;
101
+ margin-bottom: 30px;
102
+ }
103
+
104
+ .stats-list .stats-list-positive {
105
+ color: #228b22;
106
+ }
107
+
108
+ .stats-list .stats-list-negative {
109
+ color: #a52a2a;
110
+ }
111
+
112
+ .stats-list > li {
113
+ display: inline-block;
114
+ margin-right: 10px;
115
+ padding-right: 10px;
116
+ border-right: 1px solid #cacaca;
117
+ text-align: center;
118
+ font-size: 1.1em;
119
+ font-weight: bold;
120
+ }
121
+
122
+ .stats-list > li:last-child {
123
+ border: none;
124
+ margin: 0;
125
+ padding: 0;
126
+ }
127
+
128
+ .stats-list > li .stats-list-label {
129
+ display: block;
130
+ margin-top: 2px;
131
+ font-size: 0.9em;
132
+ font-weight: normal;
133
+ }
134
+
135
+ #streaks-table .ip {
136
+ vertical-align: top;
137
+ }
138
+ #streaks-table .date {
139
+ font-weight: bold;
140
+ }
141
+ #streaks-table .res-title {
142
+ font-decoration: underline;
143
+ }
144
+ </style>
145
+
146
+ </head>
147
+
148
+ <body>
149
+ <div class="off-canvas-wrapper">
150
+ <div class="off-canvas position-left" id="offCanvas" data-off-canvas>
151
+ <nav>
152
+ <h2>Navigation</h2>
153
+ <ul class="no-bullet">
154
+ <% [
155
+ "Summary",
156
+ "Log Structure",
157
+ "Daily Distribution",
158
+ "Time Distribution",
159
+ "Most Requested Pages",
160
+ "Most Requested Resources",
161
+ "404 on HTML Files",
162
+ "404 on other Resources",
163
+ "Attacks",
164
+ "Statuses",
165
+ "Daily Statuses",
166
+ "Browsers",
167
+ "Platforms",
168
+ "Referers",
169
+ "IPs",
170
+ "Geolocation",
171
+ "Streaks",
172
+ "Command Invocation",
173
+ "Performance"
174
+ ].each do |item| %>
175
+ <li class="nav-item">
176
+ <a href="#<%= item.downcase.gsub(' ', '-') %>" data-close><%= item %></a>
177
+ </li>
178
+ <% end %>
179
+ </ul>
180
+
181
+ <p>
182
+ Generated by
183
+ <a href="https://github.com/avillafiorita/log_sense">LogSense</a> <br />
184
+ on <%= DateTime.now.strftime("%Y-%m-%d %H:%M") %>.<br />
185
+ <a href='https://db-ip.com'>IP Geolocation by DB-IP</a>
186
+ </p>
187
+ </nav>
188
+ </div>
189
+ <div class="off-canvas-content grid-container grid-x fluid" data-off-canvas-content>
190
+ <div data-sticky-container>
191
+ <div class="sticky" data-sticky data-margin-top="0">
192
+ <div class="contents-button">
193
+ <i id="hamburger" class="fi-list" data-toggle="offCanvas"></i>
194
+ </div>
195
+ </div>
196
+ </div>
197
+
198
+ <section class="main-section">
199
+ <h1><%= options[:title] || "Log Sense: #{data[:log_file]}" %></h1>
200
+
201
+ <p><b>Input File:</b> <%= (data[:log_file] || "stdin") %></p>
202
+
203
+ <div class="grid-x grid-margin-x">
204
+ <article class="card small-12 large-6 cell">
205
+ <div class="card-divider">
206
+ <h2 id="summary">Summary</h2>
207
+ </div>
208
+ <div class="card-section">
209
+ <%= render "summary.html.erb", data: data %>
210
+ </div>
211
+ </article>
212
+
213
+ <article class="card cell small-12 large-6">
214
+ <div class="card-divider">
215
+ <h2 id="log-structure">Log Structure</h2>
216
+ </div>
217
+ <div class="card-section">
218
+ <%= render "log_structure.html.erb", data: data %>
219
+ </div>
220
+ </article>
221
+ </div>
222
+
223
+ <% @reports = [
224
+ { title: "Daily Distribution",
225
+ header: ["Day", "DOW", "Hits"],
226
+ rows: data[:daily_distribution],
227
+ vega_spec: {
228
+ "mark": {
229
+ "type": "line",
230
+ "point": {
231
+ "filled": false,
232
+ "fill": "white"
233
+ }
234
+ },
235
+ "encoding": {
236
+ "x": {"field": "Day", "type": "temporal"},
237
+ "y": {"field": "Hits", "type": "quantitative"}
238
+ }
239
+ }
240
+ },
241
+ { title: "Time Distribution",
242
+ header: ["Hour", "Hits"],
243
+ rows: data[:time_distribution],
244
+ vega_spec: {
245
+ "mark": "bar",
246
+ "encoding": {
247
+ "x": {"field": "Hour", "type": "nominal"},
248
+ "y": {"field": "Hits", "type": "quantitative"}
249
+ }
250
+ }
251
+ },
252
+ { title: "Statuses",
253
+ header: ["Status", "Count"],
254
+ rows: data[:statuses],
255
+ vega_spec: {
256
+ "mark": "bar",
257
+ "encoding": {
258
+ "x": {"field": "Status", "type": "nominal"},
259
+ "y": {"field": "Count", "type": "quantitative"}
260
+ }
261
+ }
262
+ },
263
+ { title: "Rails Performance",
264
+ header: ['Controller', 'Hits', 'Min', 'Avg', 'Max'],
265
+ rows: @data[:performance] },
266
+ { title: "Fatal Events",
267
+ header: ['Date', 'IP', 'URL', 'Log ID'], rows: @data[:fatal]
268
+ },
269
+ { title: "IPs", header: ["IPs", "Hits", "Country"], rows: data[:ips] },
270
+ ]
271
+ %>
272
+ <div class="grid-x grid-margin-x">
273
+ <% @reports.each_with_index do |report, index| %>
274
+ <article class="card cell <%= report[:col] || "small-12 large-6" %>" >
275
+ <div class="card-divider">
276
+ <h2>
277
+ <%= report[:title] %>
278
+ </h2>
279
+ </div>
280
+
281
+ <% if report[:vega_spec] %>
282
+ <div id="<%= "plot-#{index}" %>"></div>
283
+ <script>
284
+ plot_spec_<%= index %> = Object.assign(
285
+ <%= report[:vega_spec].to_json %>,
286
+ { "$schema": "https://vega.github.io/schema/vega-lite/v5.json",
287
+ width: "container",
288
+ description: "<%= report[:title] %>",
289
+ data: {
290
+ values: [
291
+ <% report[:rows].each do |row| %>
292
+ {
293
+ <% report[:header].each_with_index do |h, i| %>
294
+ "<%= h %>": <%= (row[i].class == Integer or row[i].class == Float) ? row[i] : "\"#{row[i]}\"" %>,
295
+ <% end %>
296
+ },
297
+ <% end %>
298
+ ]
299
+ },
300
+ });
301
+ vegaEmbed('#<%= "plot-#{index}"%>', plot_spec_<%= index %>);
302
+ </script>
303
+ <% end %>
304
+ <div class="card-section">
305
+ <%= render "output_table.html.erb", report %>
306
+ </div>
307
+ </article>
308
+ <% end %>
309
+ </div>
310
+
311
+ <div class="grid-x grid-margin-x">
312
+ <div class="cell small-12 large-6">
313
+ <article>
314
+ <h2 id="command-invocation">Command Invocation</h2>
315
+
316
+ <%= render "command_invocation.html.erb", data: data, options: options %>
317
+ </article>
318
+ </div>
319
+
320
+ <div class="small-12 large-6 cell">
321
+ <article>
322
+ <h2 id="performance"> Performance</h2>
323
+
324
+ <%= render "performance.html.erb", data: data %>
325
+ </article>
326
+ </div>
327
+ </div>
328
+ </section>
329
+ </div>
330
+
331
+ <script type="text/javascript" src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
332
+ <script type="text/javascript" src="js/vendor/what-input.js"></script>
333
+ <script type="text/javascript" src="https://cdn.datatables.net/v/zf/dt-1.11.3/datatables.min.js"></script>
334
+ <script src="https://cdn.jsdelivr.net/npm/vega@5"></script>
335
+ <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/foundation-sites@6.7.4/dist/js/foundation.min.js" crossorigin="anonymous"></script>
336
+ <script>
337
+ $(document).foundation();
338
+
339
+ $(document).ready(function () {
340
+ $('.data-table').each(function () {
341
+ $(this).DataTable();
342
+ });
343
+ });
344
+ </script>
345
+ </div>
346
+ </body>
347
+ </html>
@@ -1,3 +1,3 @@
1
1
  module LogSense
2
- VERSION = "1.1.1"
2
+ VERSION = "1.1.2"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: log_sense
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.1
4
+ version: 1.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adolfo Villafiorita
@@ -159,6 +159,7 @@ files:
159
159
  - lib/log_sense/templates/_summary.html.erb
160
160
  - lib/log_sense/templates/_summary.txt.erb
161
161
  - lib/log_sense/templates/apache.html.erb
162
+ - lib/log_sense/templates/rails.html.erb
162
163
  - lib/log_sense/templates/rails.txt.erb
163
164
  - lib/log_sense/version.rb
164
165
  - log_sense.gemspec