log_sense 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/CHANGELOG.org +27 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +37 -0
- data/LICENSE.txt +21 -0
- data/README.org +114 -0
- data/Rakefile +15 -0
- data/alr-styles.css +61 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/log_sense +66 -0
- data/ip_locations/dbip-country-lite.sqlite3 +0 -0
- data/lib/log_sense/apache_data_cruncher.rb +131 -0
- data/lib/log_sense/apache_log_parser.rb +87 -0
- data/lib/log_sense/emitter.rb +49 -0
- data/lib/log_sense/ip_locator.rb +55 -0
- data/lib/log_sense/options_parser.rb +86 -0
- data/lib/log_sense/rails_data_cruncher.rb +117 -0
- data/lib/log_sense/rails_log_parser.rb +176 -0
- data/lib/log_sense/templates/#apache.org.erb# +266 -0
- data/lib/log_sense/templates/.#apache.org.erb +1 -0
- data/lib/log_sense/templates/_output_table.html.erb +25 -0
- data/lib/log_sense/templates/apache.html.erb +328 -0
- data/lib/log_sense/templates/apache.org.erb +266 -0
- data/lib/log_sense/templates/rails.txt.erb +39 -0
- data/lib/log_sense/version.rb +3 -0
- data/lib/log_sense.rb +8 -0
- data/log_sense.gemspec +39 -0
- metadata +189 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
lib/log_sense/templates/adolfo@cardassian.537596:1636627399
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
<%
|
|
2
|
+
def slugify string
|
|
3
|
+
string.downcase.gsub(/ +/, '-')
|
|
4
|
+
end
|
|
5
|
+
%>
|
|
6
|
+
|
|
7
|
+
<table id="<%= slugify(title || "") %>" class="table <%= slugify(title || "") %>">
|
|
8
|
+
<thead>
|
|
9
|
+
<tr>
|
|
10
|
+
<% header.each do |heading| %>
|
|
11
|
+
<th class="<%= slugify(heading) %>"><%= heading %></th>
|
|
12
|
+
<% end %>
|
|
13
|
+
</tr>
|
|
14
|
+
</thead>
|
|
15
|
+
<tbody>
|
|
16
|
+
<% rows.each do |row| %>
|
|
17
|
+
<tr>
|
|
18
|
+
<% row.each_with_index do |cell, i| %>
|
|
19
|
+
<td class="<%= slugify (header[i] || "") %>"><%= cell %></td>
|
|
20
|
+
<% end %>
|
|
21
|
+
</tr>
|
|
22
|
+
<% end %>
|
|
23
|
+
</tbody>
|
|
24
|
+
</table>
|
|
25
|
+
|
|
@@ -0,0 +1,328 @@
|
|
|
1
|
+
<html>
|
|
2
|
+
<head>
|
|
3
|
+
<title>Apache Log Analysis: <%= data[:log_file] || "stdin" %></title>
|
|
4
|
+
<meta name="author" content="apache_log_report">
|
|
5
|
+
|
|
6
|
+
<link rel="stylesheet" href="alr-styles.css"></style>
|
|
7
|
+
|
|
8
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre.min.css">
|
|
9
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre-exp.min.css">
|
|
10
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre-icons.min.css">
|
|
11
|
+
</head>
|
|
12
|
+
|
|
13
|
+
<body>
|
|
14
|
+
<div class="container">
|
|
15
|
+
<nav>
|
|
16
|
+
<ul class="nav">
|
|
17
|
+
<li class="nav-item active">
|
|
18
|
+
<a href="#">Navigation</a>
|
|
19
|
+
<ul class="nav">
|
|
20
|
+
<% [ "Summary",
|
|
21
|
+
"Log Structure",
|
|
22
|
+
"Daily Distribution",
|
|
23
|
+
"Time Distribution",
|
|
24
|
+
"Most Requested Pages",
|
|
25
|
+
"Most Requested Resources",
|
|
26
|
+
"404 on HTML Files",
|
|
27
|
+
"404 on other Resources",
|
|
28
|
+
"Attacks",
|
|
29
|
+
"Statuses",
|
|
30
|
+
"Daily Statuses",
|
|
31
|
+
"Browsers",
|
|
32
|
+
"Platforms",
|
|
33
|
+
"Referers",
|
|
34
|
+
"IPs",
|
|
35
|
+
"Geolocation",
|
|
36
|
+
"Streaks",
|
|
37
|
+
"Command Invocation",
|
|
38
|
+
"Performance"
|
|
39
|
+
].each do |item| %>
|
|
40
|
+
<li class="nav-item">
|
|
41
|
+
<a href="#<%= item.downcase.gsub(' ', '-') %>"><%= item %></a>
|
|
42
|
+
</li>
|
|
43
|
+
<% end %>
|
|
44
|
+
</ul>
|
|
45
|
+
</li>
|
|
46
|
+
</ul>
|
|
47
|
+
<p>
|
|
48
|
+
Generated by<br />
|
|
49
|
+
<a href="https://github.com/avillafiorita/log_sense">LogSense</a> <br />
|
|
50
|
+
on <%= DateTime.now.strftime("%Y-%m-%d %H:%M") %>.<br />
|
|
51
|
+
<a href='https://db-ip.com'>IP Geolocation by DB-IP</a>
|
|
52
|
+
</p>
|
|
53
|
+
</nav>
|
|
54
|
+
|
|
55
|
+
<section>
|
|
56
|
+
<h1>Apache Log Analysis: <%= data[:log_file] || "stdin" %></h1>
|
|
57
|
+
|
|
58
|
+
<div class="columns">
|
|
59
|
+
<article class="col-6 column">
|
|
60
|
+
<h2 id="summary">Summary</h2>
|
|
61
|
+
|
|
62
|
+
<table class="table summary">
|
|
63
|
+
<tr>
|
|
64
|
+
<th>Input file</th>
|
|
65
|
+
<td><b><%= (data[:log_file] || "stdin") %></b></td>
|
|
66
|
+
</tr>
|
|
67
|
+
<tr>
|
|
68
|
+
<th class="period">Period Analyzed</th>
|
|
69
|
+
<td class="period">
|
|
70
|
+
<%= data[:first_day_in_analysis] %>
|
|
71
|
+
--
|
|
72
|
+
<%= data[:last_day_in_analysis] %>
|
|
73
|
+
</td>
|
|
74
|
+
</tr>
|
|
75
|
+
<tr>
|
|
76
|
+
<th class="days">Days </th>
|
|
77
|
+
<td class="days"><%= data[:total_days_in_analysis] %></td>
|
|
78
|
+
</tr>
|
|
79
|
+
<tr>
|
|
80
|
+
<th class="hits">Hits</th>
|
|
81
|
+
<td class="hits"><%= data[:total_hits][0][0] %></td>
|
|
82
|
+
</tr>
|
|
83
|
+
<tr>
|
|
84
|
+
<th class="unique-visitors">Unique Visitors</th>
|
|
85
|
+
<td class="unique-visitors"><%= data[:total_unique_visitors][0][0] %></td>
|
|
86
|
+
</tr>
|
|
87
|
+
<tr>
|
|
88
|
+
<th class="tx">Tx</th>
|
|
89
|
+
<td class="tx"><%= data[:total_size][0][0] %></td>
|
|
90
|
+
</tr>
|
|
91
|
+
</table>
|
|
92
|
+
</article>
|
|
93
|
+
<article class="column col-6">
|
|
94
|
+
<h2 id="log-structure">Log Structure</h2>
|
|
95
|
+
|
|
96
|
+
<table class="table log-structure">
|
|
97
|
+
<tbody>
|
|
98
|
+
<tr>
|
|
99
|
+
<th>Input file</th>
|
|
100
|
+
<td><b><%= (data[:log_file] || "stdin") %></b></td>
|
|
101
|
+
</tr>
|
|
102
|
+
<tr>
|
|
103
|
+
<th>Period in Log</th>
|
|
104
|
+
<td><%= data[:first_day] %> -- <%= data[:last_day] %></td>
|
|
105
|
+
</tr>
|
|
106
|
+
<tr>
|
|
107
|
+
<th>Total days</th>
|
|
108
|
+
<td><%= data[:total_days] %></td>
|
|
109
|
+
</tr>
|
|
110
|
+
<tr>
|
|
111
|
+
<th>Log size</th>
|
|
112
|
+
<td><%= data[:log_size][0][0] %></td>
|
|
113
|
+
</tr>
|
|
114
|
+
<tr>
|
|
115
|
+
<th>Self poll entries</th>
|
|
116
|
+
<td><%= data[:selfpolls_size][0][0] %></td>
|
|
117
|
+
</tr>
|
|
118
|
+
<tr>
|
|
119
|
+
<th>Crawlers</th>
|
|
120
|
+
<td><%= data[:crawlers_size][0][0] %></td>
|
|
121
|
+
</tr>
|
|
122
|
+
<tr>
|
|
123
|
+
<th>Entries considered</th>
|
|
124
|
+
<td><%= data[:total_hits][0][0] %></td>
|
|
125
|
+
</tr>
|
|
126
|
+
</tbody>
|
|
127
|
+
</table>
|
|
128
|
+
</article>
|
|
129
|
+
</div>
|
|
130
|
+
|
|
131
|
+
<% @reports = [
|
|
132
|
+
{ title: "Daily Distribution", header: ["Day", "DOW", "Hits", "Visits", "Size"], rows: data[:daily_distribution] },
|
|
133
|
+
{ title: "Time Distribution", header: ["Hour", "Hits", "Visits", "Size"], rows: data[:time_distribution] },
|
|
134
|
+
{ title: "Most Requested Pages", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_pages] },
|
|
135
|
+
{ title: "Most Requested Resources", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_resources] },
|
|
136
|
+
{ title: "404 on HTML Files", header: ["Path", "Hits", "Visitors"], rows: data[:missed_pages] },
|
|
137
|
+
{ title: "404 on other Resources", header: ["Path", "Hits", "Visitors"], rows: data[:missed_resources] },
|
|
138
|
+
{ title: "Attacks", header: ["Path", "Hits", "Visitors"], rows: data[:attacks] },
|
|
139
|
+
{ },
|
|
140
|
+
{ title: "Statuses", header: ["Status", "Count"], rows: data[:statuses] },
|
|
141
|
+
{ title: "Daily Statuses", header: ["Status", "2xx", "3xx", "4xx"], rows: data[:statuses_by_day] },
|
|
142
|
+
{ title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
|
|
143
|
+
{ title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
|
|
144
|
+
{ title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers], col: "col-12" },
|
|
145
|
+
{ title: "IPs", header: ["IPs", "Hits", "Visitors", "Size", "Country"], rows: data[:ips] },
|
|
146
|
+
{ },
|
|
147
|
+
]
|
|
148
|
+
%>
|
|
149
|
+
<div class="columns">
|
|
150
|
+
<% @reports.each do |report| %>
|
|
151
|
+
<div class="column <%= report[:col] || "col-6" %>">
|
|
152
|
+
<article>
|
|
153
|
+
<% if report[:title] != nil %>
|
|
154
|
+
<h2 id="<%= report[:title].downcase.gsub(/ +/, '-') %>">
|
|
155
|
+
<%= report[:title] %>
|
|
156
|
+
</h2>
|
|
157
|
+
<%= render "output_table", report %>
|
|
158
|
+
<% end %>
|
|
159
|
+
</article>
|
|
160
|
+
</div>
|
|
161
|
+
<% end %>
|
|
162
|
+
</div>
|
|
163
|
+
|
|
164
|
+
<article>
|
|
165
|
+
<h2 id="geolocation">Geolocation</h2>
|
|
166
|
+
<table class="table">
|
|
167
|
+
<thead>
|
|
168
|
+
<tr>
|
|
169
|
+
<th>Country Code</th>
|
|
170
|
+
<th>Total Hits</th>
|
|
171
|
+
<th>Total Visitors</th>
|
|
172
|
+
<th>IPs</th>
|
|
173
|
+
</tr>
|
|
174
|
+
</thead>
|
|
175
|
+
<tbody>
|
|
176
|
+
<%# IP, Hits, Visitors Size, Country%>
|
|
177
|
+
<% data[:ips].group_by { |x| x[4] }.each do |k, v| %>
|
|
178
|
+
<tr>
|
|
179
|
+
<td><%= k %></td>
|
|
180
|
+
<td><%= v.map { |x| x[1] }.inject(&:+) %></td>
|
|
181
|
+
<td><%= v.map { |x| x[2] }.inject(&:+) %></td>
|
|
182
|
+
<td><%= v.map { |x| x[0] }.join(", ") %></td>
|
|
183
|
+
</tr>
|
|
184
|
+
<% end %>
|
|
185
|
+
</tbody>
|
|
186
|
+
</table>
|
|
187
|
+
</article>
|
|
188
|
+
|
|
189
|
+
<article>
|
|
190
|
+
<h2 id="streaks">Streaks</h2>
|
|
191
|
+
|
|
192
|
+
<table class="table streaks">
|
|
193
|
+
<thead>
|
|
194
|
+
<tr>
|
|
195
|
+
<th>IP</th>
|
|
196
|
+
<th>
|
|
197
|
+
<div class="columns">
|
|
198
|
+
<div class="col-2 column">
|
|
199
|
+
Day
|
|
200
|
+
</div>
|
|
201
|
+
<div class="col-10 column">
|
|
202
|
+
Resources
|
|
203
|
+
</div>
|
|
204
|
+
</div>
|
|
205
|
+
</th>
|
|
206
|
+
</tr>
|
|
207
|
+
</thead>
|
|
208
|
+
<tbody>
|
|
209
|
+
<% data[:streaks].group_by(&:first).each do |ip, date_urls| %>
|
|
210
|
+
<tr>
|
|
211
|
+
<td class="ip"><%= ip %></td>
|
|
212
|
+
<td class="streaks">
|
|
213
|
+
<div class="columns">
|
|
214
|
+
<% date_urls.group_by { |x| x[1] }.each do |date, urls| %>
|
|
215
|
+
<div class="col-2 column">
|
|
216
|
+
<%= date %>
|
|
217
|
+
</div>
|
|
218
|
+
<div class="col-10 column">
|
|
219
|
+
<span class="res-title">HTML:</span>
|
|
220
|
+
<ul>
|
|
221
|
+
<% urls.map { |x| x[2] }.select { |x| x.match /.*\.html?/ }.each do |url| %>
|
|
222
|
+
<li><%= url %></li>
|
|
223
|
+
<% end %>
|
|
224
|
+
</ul>
|
|
225
|
+
|
|
226
|
+
<span class="res-title">Other Resources:</span>
|
|
227
|
+
<ul>
|
|
228
|
+
<% urls.map { |x| x[2] }.sort.select { |x| not x.match /.*\.html?/ }.each do |url| %>
|
|
229
|
+
<li><%= url %></li>
|
|
230
|
+
<% end %>
|
|
231
|
+
</ul>
|
|
232
|
+
</div>
|
|
233
|
+
<% end %>
|
|
234
|
+
</div>
|
|
235
|
+
</td>
|
|
236
|
+
</tr>
|
|
237
|
+
<% end %>
|
|
238
|
+
</tbody>
|
|
239
|
+
</table>
|
|
240
|
+
</article>
|
|
241
|
+
|
|
242
|
+
<div class="columns">
|
|
243
|
+
<div class="column col-6">
|
|
244
|
+
<article>
|
|
245
|
+
<h2 id="command-invocation">Command Invocation</h2>
|
|
246
|
+
|
|
247
|
+
<table class="table command-invocation">
|
|
248
|
+
<tbody>
|
|
249
|
+
<tr>
|
|
250
|
+
<th>CLI Command</th>
|
|
251
|
+
<td><code><%= data[:command] %></code></td>
|
|
252
|
+
</tr>
|
|
253
|
+
<tr>
|
|
254
|
+
<th>Input file</th>
|
|
255
|
+
<td><code><%= (data[:log_file] || "stdin") %></code></td>
|
|
256
|
+
</tr>
|
|
257
|
+
<tr>
|
|
258
|
+
<th>Ignore crawlers</th>
|
|
259
|
+
<td><code><%= options[:ignore_crawlers] %></code></td></tr>
|
|
260
|
+
<tr>
|
|
261
|
+
<th>Only crawlers</th>
|
|
262
|
+
<td><code><%= options[:only_crawlers] %></code></td>
|
|
263
|
+
</tr>
|
|
264
|
+
<tr>
|
|
265
|
+
<th>No selfpoll</th>
|
|
266
|
+
<td><code><%= options[:no_selfpoll] %></code></td>
|
|
267
|
+
</tr>
|
|
268
|
+
<tr>
|
|
269
|
+
<th>Filter by date</th>
|
|
270
|
+
<td>
|
|
271
|
+
<code><%= (options[:from_date] != nil or options[:to_date] != nil) %></code>
|
|
272
|
+
</td>
|
|
273
|
+
</tr>
|
|
274
|
+
<tr>
|
|
275
|
+
<th>Prefix</th>
|
|
276
|
+
<td><code><%= @prefix %></code></td>
|
|
277
|
+
</tr>
|
|
278
|
+
<tr>
|
|
279
|
+
<th>Suffix</th>
|
|
280
|
+
<td><code><%= @suffix %></code></td>
|
|
281
|
+
</tr>
|
|
282
|
+
</tbody>
|
|
283
|
+
</table>
|
|
284
|
+
</article>
|
|
285
|
+
</div>
|
|
286
|
+
|
|
287
|
+
<div class="column col-6">
|
|
288
|
+
<article>
|
|
289
|
+
<h2 id="performance"> Performance</h2>
|
|
290
|
+
|
|
291
|
+
<table class="table performance">
|
|
292
|
+
<tbody>
|
|
293
|
+
<tr>
|
|
294
|
+
<th>Analysis started at</th>
|
|
295
|
+
<td><%= data[:started_at].to_s %></td>
|
|
296
|
+
</tr>
|
|
297
|
+
<tr>
|
|
298
|
+
<th>Analysis ended at</th>
|
|
299
|
+
<td><%= data[:ended_at].to_s %></td>
|
|
300
|
+
</tr>
|
|
301
|
+
<tr>
|
|
302
|
+
<th>Duration (sec)</th>
|
|
303
|
+
<td><%= "%.1f" % data[:duration] %></td>
|
|
304
|
+
</tr>
|
|
305
|
+
<tr>
|
|
306
|
+
<th>Duration (min)</th>
|
|
307
|
+
<td><%= "%d" % (data[:duration] / 60 ) %></td>
|
|
308
|
+
</tr>
|
|
309
|
+
<tr>
|
|
310
|
+
<th>Log size</th>
|
|
311
|
+
<td><%= data[:log_size][0][0] %></td>
|
|
312
|
+
</tr>
|
|
313
|
+
<tr>
|
|
314
|
+
<th>Lines/sec</th>
|
|
315
|
+
<td><%= "%.2f" % (data[:log_size][0][0] / data[:duration]) %></td></tr>
|
|
316
|
+
</tbody>
|
|
317
|
+
</table>
|
|
318
|
+
</article>
|
|
319
|
+
</div>
|
|
320
|
+
</div>
|
|
321
|
+
</section>
|
|
322
|
+
</div>
|
|
323
|
+
</body>
|
|
324
|
+
</html>
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
#+TITLE: Apache Log Analysis: <%= data[:log_file] %>
|
|
2
|
+
#+DATE: <<%= Date.today %>>
|
|
3
|
+
#+STARTUP: showall
|
|
4
|
+
#+OPTIONS: ^:{}
|
|
5
|
+
#+HTML_HEAD: <link rel="stylesheet" type="text/css" href="ala-style.css" />
|
|
6
|
+
#+OPTIONS: html-style:nil
|
|
7
|
+
|
|
8
|
+
* Summary
|
|
9
|
+
|
|
10
|
+
| Hits | <%= "%10d" % data[:total_hits][0][0] %> |
|
|
11
|
+
| Unique Visitors | <%= "%10d" % data[:total_unique_visitors][0][0] %> |
|
|
12
|
+
| Tx | <%= "%10s" % data[:total_size][0][0] %> |
|
|
13
|
+
| Logged Period | <%= data[:first_day] %> -- <%= data[:last_day] %> |
|
|
14
|
+
| Days | <%= "%10d" % data[:total_days] %> |
|
|
15
|
+
| Period Requested | <%= data[:first_day_requested] %> -- <%= data[:last_day_requested] %> |
|
|
16
|
+
| Period Analyzed | <%= data[:first_day_in_analysis] %> -- <%= data[:last_day_in_analysis] %> |
|
|
17
|
+
| Days in Analysis | <%= data[:total_days_in_analysis] %> |
|
|
18
|
+
|
|
19
|
+
* Daily Distribution
|
|
20
|
+
|
|
21
|
+
<%= self.output_txt_table "daily_distribution", ["Day", "Hits", "Visits", "Size"], data[:daily_distribution] %>
|
|
22
|
+
|
|
23
|
+
#+BEGIN_SRC gnuplot :var data = daily_distribution :results output :exports <%= @export %> :file <%= @prefix %>daily<%= @suffix %>.svg
|
|
24
|
+
reset
|
|
25
|
+
set grid ytics linestyle 0
|
|
26
|
+
set grid xtics linestyle 0
|
|
27
|
+
set terminal svg size 1200,800 fname 'Arial'
|
|
28
|
+
|
|
29
|
+
set xdata time
|
|
30
|
+
set timefmt "%Y-%m-%d"
|
|
31
|
+
set format x "%a, %b %d"
|
|
32
|
+
set xtics rotate by 60 right
|
|
33
|
+
|
|
34
|
+
set title "Hits and Visitors"
|
|
35
|
+
set xlabel "Date"
|
|
36
|
+
set ylabel "Hits"
|
|
37
|
+
set y2label "Visits"
|
|
38
|
+
set y2tics
|
|
39
|
+
|
|
40
|
+
set style fill transparent solid 0.2 noborder
|
|
41
|
+
|
|
42
|
+
plot data using 1:2 with linespoints lw 3 lc rgb "#0000AA" pointtype 5 title "Hits" axes x1y2, \\
|
|
43
|
+
data using 1:2 with filledcurves below x1 linecolor rgb "#0000AA" notitle axes x1y2, \\
|
|
44
|
+
data using 1:3 with linespoints lw 3 lc rgb "#AA0000" pointtype 7 title "Visitors", \\
|
|
45
|
+
data using 1:3 with filledcurves below x1 notitle linecolor rgb "#AA0000", \\
|
|
46
|
+
data using 1:($3+0.1*$3):3 with labels notitle textcolor rgb "#AA0000", \\
|
|
47
|
+
data using 1:($2+0.1*$2):2 with labels notitle textcolor rgb "#0000AA" axes x1y2
|
|
48
|
+
#+END_SRC
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
* Time Distribution
|
|
52
|
+
|
|
53
|
+
<%= self.output_txt_table "time_distribution", ["Hour", "Hits", "Visits", "Size"], data[:time_distribution] %>
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports <%= @export %> :file <%= @prefix %>time<%= @suffix %>.svg
|
|
57
|
+
reset
|
|
58
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
|
59
|
+
|
|
60
|
+
set grid ytics linestyle 0
|
|
61
|
+
|
|
62
|
+
set title "Hits and Visitors"
|
|
63
|
+
set xlabel "Date"
|
|
64
|
+
set ylabel "Hits"
|
|
65
|
+
set y2label "Visitors"
|
|
66
|
+
set y2tics
|
|
67
|
+
|
|
68
|
+
set style fill solid 0.25
|
|
69
|
+
set boxwidth 0.6
|
|
70
|
+
|
|
71
|
+
set style data histograms
|
|
72
|
+
set style histogram clustered gap 1
|
|
73
|
+
|
|
74
|
+
plot data using 2:xtic(1) lc rgb "#0000AA" title "Hits", \\
|
|
75
|
+
data using 3 lc rgb "#AA0000" title "Visitors" axes x1y2, \\
|
|
76
|
+
data using ($0 - 0.2):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#0000AA"), \\
|
|
77
|
+
data using ($0 + 0.2):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#AA0000") axes x1y2
|
|
78
|
+
#+END_SRC
|
|
79
|
+
|
|
80
|
+
#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports <%= @export %> :file <%= @prefix %>time-traffic<%= @suffix %>.svg
|
|
81
|
+
reset
|
|
82
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
|
83
|
+
|
|
84
|
+
set grid ytics linestyle 0
|
|
85
|
+
|
|
86
|
+
set title "Traffic"
|
|
87
|
+
set xlabel "Date"
|
|
88
|
+
set ylabel "Traffic"
|
|
89
|
+
|
|
90
|
+
set style fill solid 0.50
|
|
91
|
+
set boxwidth 0.6
|
|
92
|
+
|
|
93
|
+
set style data histograms
|
|
94
|
+
set style histogram clustered gap 1
|
|
95
|
+
|
|
96
|
+
plot data using 2:xtic(1) lc rgb "#00AA00" title "Traffic", \\
|
|
97
|
+
data using ($0):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00")
|
|
98
|
+
#+END_SRC
|
|
99
|
+
|
|
100
|
+
* Most Requested Pages
|
|
101
|
+
|
|
102
|
+
<%= self.output_txt_table "most_requested_pages", ["Path", "Hits", "Visits", "Size"], data[:most_requested_pages] %>
|
|
103
|
+
|
|
104
|
+
* Most Requested URIs
|
|
105
|
+
|
|
106
|
+
<%= self.output_txt_table "most_requested_resources", ["Path", "Hits", "Visits", "Size"], data[:most_requested_resources] %>
|
|
107
|
+
|
|
108
|
+
* 404s on HTML files
|
|
109
|
+
|
|
110
|
+
<%= self.output_txt_table "pages_404", ["Path", "Hits", "Visitors"], data[:missed_pages] %>
|
|
111
|
+
|
|
112
|
+
* 404s on other resources
|
|
113
|
+
|
|
114
|
+
<%= self.output_txt_table "resources_404", ["Path", "Hits", "Visitors"], data[:missed_resources] %>
|
|
115
|
+
|
|
116
|
+
* Possible Attacks
|
|
117
|
+
|
|
118
|
+
<%= self.output_txt_table "attacks", ["Path", "Hits", "Visitors"], data[:attacks] %>
|
|
119
|
+
|
|
120
|
+
* Statuses
|
|
121
|
+
|
|
122
|
+
<%= self.output_txt_table "statuses", ["Status", "Count"], data[:statuses] %>
|
|
123
|
+
|
|
124
|
+
#+BEGIN_SRC gnuplot :var data = statuses :results output :exports <%= @export %> :file <%= @prefix %>statuses<%= @suffix %>.svg
|
|
125
|
+
reset
|
|
126
|
+
set grid ytics linestyle 0
|
|
127
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
|
128
|
+
|
|
129
|
+
set style fill solid 0.25
|
|
130
|
+
set boxwidth 0.6
|
|
131
|
+
|
|
132
|
+
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
|
133
|
+
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
|
134
|
+
#+END_SRC
|
|
135
|
+
|
|
136
|
+
* Daily Statuses
|
|
137
|
+
|
|
138
|
+
<%= self.output_txt_table "daily_statuses", ["Status", "2xx", "3xx", "4xx"], data[:statuses_by_day] %>
|
|
139
|
+
|
|
140
|
+
#+BEGIN_SRC gnuplot :var data = daily_statuses :results output :exports <%= @export %> :file <%= @prefix %>daily-statuses<%= @suffix %>.svg
|
|
141
|
+
reset
|
|
142
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
|
143
|
+
|
|
144
|
+
set grid ytics linestyle 0
|
|
145
|
+
|
|
146
|
+
set title "Daily Statuses"
|
|
147
|
+
set xlabel "Date"
|
|
148
|
+
set ylabel "Number of Hits"
|
|
149
|
+
set xtics rotate by 60 right
|
|
150
|
+
|
|
151
|
+
set style fill solid 0.25
|
|
152
|
+
set boxwidth 0.6
|
|
153
|
+
|
|
154
|
+
set style data histograms
|
|
155
|
+
set style histogram clustered gap 1
|
|
156
|
+
|
|
157
|
+
plot data using 2:xtic(1) lc rgb "#00AA00" title "2xx", \\
|
|
158
|
+
data using 3 lc rgb "#0000CC" title "3xx", \\
|
|
159
|
+
data using 4 lc rgb "#AA0000" title "4xx", \\
|
|
160
|
+
data using ($0 - 1. / 4):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00"), \\
|
|
161
|
+
data using ($0):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0000CC"), \\
|
|
162
|
+
data using ($0 + 1. / 4):($4 + 0.1*$4):4 with labels title "" textcolor rgb("#AA0000")
|
|
163
|
+
#+END_SRC
|
|
164
|
+
|
|
165
|
+
* Browsers
|
|
166
|
+
|
|
167
|
+
<%= self.output_txt_table "browsers", ["Browser", "Hits", "Visitors", "Size"], data[:browsers] %>
|
|
168
|
+
|
|
169
|
+
#+BEGIN_SRC gnuplot :var data = browsers :results output :exports <%= @export %> :file <%= @prefix %>browser<%= @suffix %>.svg
|
|
170
|
+
reset
|
|
171
|
+
set grid ytics linestyle 0
|
|
172
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
|
173
|
+
|
|
174
|
+
set style fill solid 0.25
|
|
175
|
+
set boxwidth 0.6
|
|
176
|
+
|
|
177
|
+
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
|
178
|
+
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
|
179
|
+
#+END_SRC
|
|
180
|
+
|
|
181
|
+
* Platforms
|
|
182
|
+
|
|
183
|
+
<%= self.output_txt_table "platforms", ["Platform", "Hits", "Visitors", "Size"], data[:platforms] %>
|
|
184
|
+
|
|
185
|
+
#+BEGIN_SRC gnuplot :var data = platforms :results output :exports <%= @export %> :file <%= @prefix %>platforms<%= @suffix %>.svg
|
|
186
|
+
reset
|
|
187
|
+
set grid ytics linestyle 0
|
|
188
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
|
189
|
+
|
|
190
|
+
set style fill solid 0.25
|
|
191
|
+
set boxwidth 0.6
|
|
192
|
+
|
|
193
|
+
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
|
194
|
+
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
|
195
|
+
#+END_SRC
|
|
196
|
+
|
|
197
|
+
* IPs
|
|
198
|
+
|
|
199
|
+
<%= self.output_txt_table "ips", ["IPs", "Hits", "Visitors", "Size"], data[:ips] %>
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
* Referers
|
|
203
|
+
|
|
204
|
+
<%= self.output_txt_table "referers", ["Referers", "Hits", "Visitors", "Size"], data[:referers] %>
|
|
205
|
+
|
|
206
|
+
#+BEGIN_SRC gnuplot :var data = referers :results output :exports <%= @export %> :file <%= @prefix %>referers<%= @suffix %>.svg
|
|
207
|
+
reset
|
|
208
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
|
209
|
+
|
|
210
|
+
set grid ytics linestyle 0
|
|
211
|
+
set grid xtics linestyle 0
|
|
212
|
+
|
|
213
|
+
set title "Referers"
|
|
214
|
+
set xlabel "Date"
|
|
215
|
+
set xtics rotate by 60 right
|
|
216
|
+
set ylabel "Hits and Visits"
|
|
217
|
+
|
|
218
|
+
set style fill solid 0.45
|
|
219
|
+
set boxwidth 0.7
|
|
220
|
+
|
|
221
|
+
set style data histograms
|
|
222
|
+
set style histogram clustered gap 1
|
|
223
|
+
|
|
224
|
+
plot data using 2:xtic(1) lc rgb "#AA00AA" title "Hits", \\
|
|
225
|
+
data using 3 lc rgb "#0AAAA0" title "Visits", \\
|
|
226
|
+
data using ($0 - 1. / 3):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#AA00AA"), \\
|
|
227
|
+
data using ($0 + 1. / 3):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0AAAA0")
|
|
228
|
+
#+END_SRC
|
|
229
|
+
|
|
230
|
+
* Command Invocation and Performance
|
|
231
|
+
|
|
232
|
+
** Command Invocation
|
|
233
|
+
|
|
234
|
+
#+BEGIN_EXAMPLE shell
|
|
235
|
+
<%= data[:command] %>
|
|
236
|
+
#+END_EXAMPLE
|
|
237
|
+
|
|
238
|
+
| Input file | <%= "%-50s" % (data[:log_file] || "stdin") %> |
|
|
239
|
+
| Ignore crawlers | <%= "%-50s" % options[:ignore_crawlers] %> |
|
|
240
|
+
| Only crawlers | <%= "%-50s" % options[:only_crawlers] %> |
|
|
241
|
+
| No selfpoll | <%= "%-50s" % options[:no_selfpoll] %> |
|
|
242
|
+
| Filter by date | <%= "%-50s" % (options[:from_date] != nil or options[:to_date] != nil) %> |
|
|
243
|
+
| Prefix | <%= "%-50s" % @prefix %> |
|
|
244
|
+
| Suffix | <%= "%-50s" % @suffix %> |
|
|
245
|
+
|
|
246
|
+
** Log Structure
|
|
247
|
+
|
|
248
|
+
| Log size | <%= "%10d" % data[:log_size][0][0] %> |
|
|
249
|
+
| Self poll entries | <%= "%10d" % data[:selfpolls_size][0][0] %> |
|
|
250
|
+
| Crawlers | <%= "%10d" % data[:crawlers_size][0][0] %> |
|
|
251
|
+
| Entries considered | <%= "%10d" % data[:total_hits][0][0] %> |
|
|
252
|
+
|
|
253
|
+
** Performance
|
|
254
|
+
|
|
255
|
+
| Analysis started at | <%= data[:started_at].to_s %> |
|
|
256
|
+
| Analysis ended at | <%= data[:ended_at].to_s %> |
|
|
257
|
+
| Duration (sec) | <%= "%5.3d" % data[:duration] %> |
|
|
258
|
+
| Duration (min) | <%= "%5.3d" % (data[:duration] / 60 ) %> |
|
|
259
|
+
| Log size | <%= "%9d" % data[:log_size][0][0] %> |
|
|
260
|
+
| Lines/sec | <%= "%6.2f" % (data[:log_size][0][0] / data[:duration]) %> |
|
|
261
|
+
|
|
262
|
+
* Local Variables :noexport:
|
|
263
|
+
# Local Variables:
|
|
264
|
+
# org-confirm-babel-evaluate: nil
|
|
265
|
+
# org-display-inline-images: t
|
|
266
|
+
# end:
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
Rails Log Analysis
|
|
2
|
+
|
|
3
|
+
<%= data[:first_day] %>
|
|
4
|
+
<%= data[:last_day] %>
|
|
5
|
+
<%= data[:total_days] %>
|
|
6
|
+
<%= data[:events] %>
|
|
7
|
+
|
|
8
|
+
<%=
|
|
9
|
+
table = Terminal::Table.new headings: ['Date', 'Day', 'Events'], rows: @data[:daily_distribution]
|
|
10
|
+
table.align_column(3, :right)
|
|
11
|
+
table
|
|
12
|
+
%>
|
|
13
|
+
|
|
14
|
+
<%=
|
|
15
|
+
table = Terminal::Table.new headings: ['Hour', 'Events'], rows: @data[:time_distribution]
|
|
16
|
+
table.align_column(2, :right)
|
|
17
|
+
table
|
|
18
|
+
%>
|
|
19
|
+
|
|
20
|
+
<%=
|
|
21
|
+
table = Terminal::Table.new headings: ['Status', 'Events'], rows: @data[:statuses]
|
|
22
|
+
table.align_column(2, :right)
|
|
23
|
+
table
|
|
24
|
+
%>
|
|
25
|
+
|
|
26
|
+
<%=
|
|
27
|
+
table = Terminal::Table.new headings: ['IP', 'Events'], rows: @data[:ips]
|
|
28
|
+
table.align_column(2, :right)
|
|
29
|
+
table
|
|
30
|
+
%>
|
|
31
|
+
|
|
32
|
+
<%= table = Terminal::Table.new headings: ['Controller', 'Hits', 'Min', 'Avg', 'Max'], rows: @data[:performance]
|
|
33
|
+
table.align_column(2, :right)
|
|
34
|
+
table.align_column(3, :right)
|
|
35
|
+
table.align_column(4, :right)
|
|
36
|
+
table
|
|
37
|
+
%>
|
|
38
|
+
|
|
39
|
+
|
data/lib/log_sense.rb
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
require 'log_sense/version'
|
|
2
|
+
require 'log_sense/options_parser'
|
|
3
|
+
require 'log_sense/apache_log_parser'
|
|
4
|
+
require 'log_sense/apache_data_cruncher'
|
|
5
|
+
require 'log_sense/rails_log_parser'
|
|
6
|
+
require 'log_sense/rails_data_cruncher'
|
|
7
|
+
require 'log_sense/ip_locator'
|
|
8
|
+
require 'log_sense/emitter'
|