apache_log_report 1.1.0 → 1.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.org +2 -4
- data/apache_log_report.gemspec +2 -2
- data/exe/apache_log_report +24 -17
- data/lib/apache_log_report/data_cruncher.rb +2 -0
- data/lib/apache_log_report/emitter.rb +0 -1
- data/lib/apache_log_report/options_parser.rb +1 -1
- data/lib/apache_log_report/templates/_output_table.html.erb +1 -1
- data/lib/apache_log_report/templates/template.html.erb +219 -130
- data/lib/apache_log_report/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8b90448ec269bd64d60c2323888fb77ea8ac7307a00198434119a6f0da605585
|
|
4
|
+
data.tar.gz: 24569c78964b795478c50e5de01c4f42fb2be8f02c30911e4e81229a96917a55
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 604697690824080fbbed523f26c1727f969fcf791803ebd15d6838384c5c77edbcde25ad74fa47c2371c0690019745948b6d78f9f1a51e6742e51e35c5c5ba76
|
|
7
|
+
data.tar.gz: b7d5a6940308a6a098cb61b9118cd17760592e299e091193b07a003ae87e6f7058ee1aab81aed2cadd65dcc07c6b64652ede126274be5be552dbbf0ca3ba11b1
|
data/Gemfile.lock
CHANGED
data/README.org
CHANGED
|
@@ -14,10 +14,8 @@ See the [[file:CHANGELOG.org][CHANGELOG]] file.
|
|
|
14
14
|
|
|
15
15
|
* Todo
|
|
16
16
|
|
|
17
|
-
** TODO
|
|
18
|
-
** TODO
|
|
19
|
-
** TODO Move performance stats var to class (to isolate vars)
|
|
20
|
-
** TODO Check total number of days (which is not working, now)
|
|
17
|
+
** TODO Graphs in HTML output
|
|
18
|
+
** TODO Countries
|
|
21
19
|
|
|
22
20
|
* Compatibility
|
|
23
21
|
|
data/apache_log_report.gemspec
CHANGED
|
@@ -6,8 +6,8 @@ Gem::Specification.new do |spec|
|
|
|
6
6
|
spec.authors = ["Adolfo Villafiorita"]
|
|
7
7
|
spec.email = ["adolfo.villafiorita@ict4g.net"]
|
|
8
8
|
|
|
9
|
-
spec.summary = %q{Generate
|
|
10
|
-
spec.description = %q{Generate
|
|
9
|
+
spec.summary = %q{Generate analytics from an Apache log file.}
|
|
10
|
+
spec.description = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file.}
|
|
11
11
|
spec.homepage = "https://www.ict4g.net/gitea/adolfo/apache_log_report"
|
|
12
12
|
spec.license = "MIT"
|
|
13
13
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
|
data/exe/apache_log_report
CHANGED
|
@@ -23,21 +23,28 @@ end
|
|
|
23
23
|
@started_at = Time.now
|
|
24
24
|
|
|
25
25
|
@db = ApacheLogReport::LogParser.parse @log_file
|
|
26
|
-
@data = ApacheLogReport::DataCruncher.crunch @db, @options
|
|
27
26
|
|
|
28
|
-
@
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
27
|
+
if @options[:output_format] == "sqlite"
|
|
28
|
+
ddb = SQLite3::Database.new('db.sqlite3')
|
|
29
|
+
b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
|
|
30
|
+
b.step(-1) #=> DONE
|
|
31
|
+
b.finish
|
|
32
|
+
else
|
|
33
|
+
@data = ApacheLogReport::DataCruncher.crunch @db, @options
|
|
34
|
+
|
|
35
|
+
@ended_at = Time.now
|
|
36
|
+
@duration = @ended_at - @started_at
|
|
37
|
+
|
|
38
|
+
@data = @data.merge({
|
|
39
|
+
command: @command,
|
|
40
|
+
log_file: @log_file,
|
|
41
|
+
started_at: @started_at,
|
|
42
|
+
ended_at: @ended_at,
|
|
43
|
+
duration: @duration
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
#
|
|
47
|
+
# Emit Output
|
|
48
|
+
#
|
|
49
|
+
puts ApacheLogReport::Emitter.emit @data, @options
|
|
50
|
+
end
|
|
@@ -71,6 +71,8 @@ module ApacheLogReport
|
|
|
71
71
|
@ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
|
|
72
72
|
@referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
|
|
73
73
|
|
|
74
|
+
@streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
|
|
75
|
+
|
|
74
76
|
data = {}
|
|
75
77
|
self.instance_variables.each do |variable|
|
|
76
78
|
var_as_symbol = variable.to_s[1..-1].to_sym
|
|
@@ -50,7 +50,7 @@ module ApacheLogReport
|
|
|
50
50
|
args[:code_export] = n
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
opts.on("-fFORMAT", "--format=FORMAT", String, "Output format: html, org. Defaults to org mode") do |n|
|
|
53
|
+
opts.on("-fFORMAT", "--format=FORMAT", String, "Output format: html, org, sqlite. Defaults to org mode") do |n|
|
|
54
54
|
args[:output_format] = n
|
|
55
55
|
end
|
|
56
56
|
|
|
@@ -4,34 +4,106 @@
|
|
|
4
4
|
<meta name="author" content="apache_log_report">
|
|
5
5
|
|
|
6
6
|
<link rel="stylesheet" href="alr-styles.css"></style>
|
|
7
|
+
|
|
8
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre.min.css">
|
|
9
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre-exp.min.css">
|
|
10
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre-icons.min.css">
|
|
7
11
|
</head>
|
|
8
12
|
|
|
9
|
-
<body>
|
|
10
|
-
<
|
|
11
|
-
|
|
13
|
+
<body class="container">
|
|
14
|
+
<h1>Apache Log Analysis: <%= data[:log_file] || "stdin" %></h1>
|
|
15
|
+
|
|
16
|
+
<nav>
|
|
17
|
+
<ul class="nav">
|
|
18
|
+
<li class="nav-item active">
|
|
19
|
+
<a href="#">Navigation</a>
|
|
20
|
+
<ul class="nav">
|
|
21
|
+
<% [ "Summary",
|
|
22
|
+
"Log Structure",
|
|
23
|
+
"Daily Distribution",
|
|
24
|
+
"Time Distribution",
|
|
25
|
+
"Most Requested Pages",
|
|
26
|
+
"Most Requested Resources",
|
|
27
|
+
"404 on HTML Files",
|
|
28
|
+
"404 on other Resources",
|
|
29
|
+
"Attacks",
|
|
30
|
+
"Statuses",
|
|
31
|
+
"Daily Statuses",
|
|
32
|
+
"Browsers",
|
|
33
|
+
"Platforms",
|
|
34
|
+
"Referers",
|
|
35
|
+
"IPs"
|
|
36
|
+
].each do |item| %>
|
|
37
|
+
<li class="nav-item">
|
|
38
|
+
<a href="#<%= item.downcase.gsub(' ', '-') %>"><%= item %></a>
|
|
39
|
+
</li>
|
|
40
|
+
<% end %>
|
|
41
|
+
</ul>
|
|
42
|
+
</li>
|
|
43
|
+
</ul>
|
|
44
|
+
</nav>
|
|
12
45
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
<
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
46
|
+
<section>
|
|
47
|
+
<div class="columns">
|
|
48
|
+
<article class="col-6 column">
|
|
49
|
+
<h2 id="summary">Summary</h2>
|
|
50
|
+
|
|
51
|
+
<table class="table summary">
|
|
52
|
+
<tr>
|
|
53
|
+
<th class="hits">Hits</th>
|
|
54
|
+
<td class="hits"><%= data[:total_hits][0][0] %></td>
|
|
55
|
+
</tr>
|
|
56
|
+
<tr>
|
|
57
|
+
<th class="unique-visitors">Unique Visitors</th>
|
|
58
|
+
<td class="unique-visitors"><%= data[:total_unique_visitors][0][0] %></td>
|
|
59
|
+
</tr>
|
|
60
|
+
<tr>
|
|
61
|
+
<th class="tx">Tx</th>
|
|
62
|
+
<td class="tx"><%= data[:total_size][0][0] %></td>
|
|
63
|
+
</tr>
|
|
64
|
+
<tr>
|
|
65
|
+
<th class="period">Period</th>
|
|
66
|
+
<td class="period">
|
|
67
|
+
<%= data[:first_day][0][0] %>
|
|
68
|
+
--
|
|
69
|
+
<%= data[:last_day][0][0] %>
|
|
70
|
+
</td>
|
|
71
|
+
</tr>
|
|
72
|
+
<tr>
|
|
73
|
+
<th class="days">Days </th>
|
|
74
|
+
<td class="days"><%= data[:total_days] %></td>
|
|
75
|
+
</tr>
|
|
76
|
+
</table>
|
|
77
|
+
</article>
|
|
78
|
+
<article class="column col-6">
|
|
79
|
+
<h2 id="log-structure">Log Structure</h2>
|
|
80
|
+
|
|
81
|
+
<table class="table log-structure">
|
|
82
|
+
<tbody>
|
|
83
|
+
<tr>
|
|
84
|
+
<th>Input file</th>
|
|
85
|
+
<td><b><%= (data[:log_file] || "stdin") %></b></td>
|
|
86
|
+
</tr>
|
|
87
|
+
<tr>
|
|
88
|
+
<th>Log size</th>
|
|
89
|
+
<td><%= data[:log_size][0][0] %></td>
|
|
90
|
+
</tr>
|
|
91
|
+
<tr>
|
|
92
|
+
<th>Self poll entries</th>
|
|
93
|
+
<td><%= data[:selfpolls_size][0][0] %></td>
|
|
94
|
+
</tr>
|
|
95
|
+
<tr>
|
|
96
|
+
<th>Crawlers</th>
|
|
97
|
+
<td><%= data[:crawlers_size][0][0] %></td>
|
|
98
|
+
</tr>
|
|
99
|
+
<tr>
|
|
100
|
+
<th>Entries considered</th>
|
|
101
|
+
<td><%= data[:total_hits][0][0] %></td>
|
|
102
|
+
</tr>
|
|
103
|
+
</tbody>
|
|
104
|
+
</table>
|
|
105
|
+
</article>
|
|
106
|
+
</div>
|
|
35
107
|
|
|
36
108
|
<% @reports = [
|
|
37
109
|
{ title: "Daily Distribution", header: ["Day", "Hits", "Visits", "Size"], rows: data[:daily_distribution] },
|
|
@@ -41,120 +113,137 @@
|
|
|
41
113
|
{ title: "404 on HTML Files", header: ["Path", "Hits", "Visitors"], rows: data[:missed_pages] },
|
|
42
114
|
{ title: "404 on other Resources", header: ["Path", "Hits", "Visitors"], rows: data[:missed_resources] },
|
|
43
115
|
{ title: "Attacks", header: ["Path", "Hits", "Visitors"], rows: data[:attacks] },
|
|
116
|
+
{ },
|
|
44
117
|
{ title: "Statuses", header: ["Status", "Count"], rows: data[:statuses] },
|
|
45
118
|
{ title: "Daily Statuses", header: ["Status", "2xx", "3xx", "4xx"], rows: data[:statuses_by_day] },
|
|
46
119
|
{ title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
|
|
47
120
|
{ title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
|
|
121
|
+
{ title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers], col: "col-12" },
|
|
48
122
|
{ title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
|
|
49
|
-
{
|
|
123
|
+
{ },
|
|
50
124
|
]
|
|
51
125
|
%>
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
<
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
126
|
+
<div class="columns">
|
|
127
|
+
<% @reports.each do |report| %>
|
|
128
|
+
<div class="column <%= report[:col] || "col-6" %>">
|
|
129
|
+
<article>
|
|
130
|
+
<% if report[:title] != nil %>
|
|
131
|
+
<h2 id="<%= report[:title].downcase.gsub(/ +/, '-') %>">
|
|
132
|
+
<%= report[:title] %>
|
|
133
|
+
</h2>
|
|
134
|
+
<%= render "output_table", report %>
|
|
135
|
+
<% end %>
|
|
136
|
+
</article>
|
|
137
|
+
</div>
|
|
138
|
+
<% end %>
|
|
139
|
+
</div>
|
|
58
140
|
|
|
59
141
|
<article>
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
<
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
<
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
<
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
142
|
+
<h2 id="streaks">Streaks</h2>
|
|
143
|
+
|
|
144
|
+
<table class="table streaks">
|
|
145
|
+
<thead>
|
|
146
|
+
<tr>
|
|
147
|
+
<th>IP</th>
|
|
148
|
+
<th>Day and URL</th>
|
|
149
|
+
</tr>
|
|
150
|
+
</thead>
|
|
151
|
+
<tbody>
|
|
152
|
+
<% data[:streaks].group_by(&:first).each do |ip, date_urls| %>
|
|
153
|
+
<tr>
|
|
154
|
+
<td class="ip"><%= ip %></td>
|
|
155
|
+
<td class="streaks">
|
|
156
|
+
<% date_urls.group_by(&:first).each do |date, urls| %>
|
|
157
|
+
<% urls.each do |url| %>
|
|
158
|
+
<b><%= url[1] %>:</b> <%= url[2] %> <br />
|
|
159
|
+
<% end %>
|
|
160
|
+
<% end %>
|
|
161
|
+
</td>
|
|
162
|
+
</tr>
|
|
163
|
+
<% end %>
|
|
164
|
+
</tbody>
|
|
165
|
+
</table>
|
|
166
|
+
</article>
|
|
167
|
+
|
|
168
|
+
<div class="columns">
|
|
169
|
+
<div class="column col-6">
|
|
170
|
+
<article>
|
|
171
|
+
<h2 id="command-invocation">Command Invocation</h2>
|
|
172
|
+
|
|
173
|
+
<table class="table command-invocation">
|
|
174
|
+
<tbody>
|
|
175
|
+
<tr>
|
|
176
|
+
<th>CLI Command</th>
|
|
177
|
+
<td><pre><%= data[:command] %></pre></td>
|
|
178
|
+
</tr>
|
|
179
|
+
<tr>
|
|
180
|
+
<th>Input file</th>
|
|
181
|
+
<td><code><%= (data[:log_file] || "stdin") %></code></td>
|
|
182
|
+
</tr>
|
|
183
|
+
<tr>
|
|
184
|
+
<th>Ignore crawlers</th>
|
|
185
|
+
<td><code><%= options[:ignore_crawlers] %></code></td></tr>
|
|
186
|
+
<tr>
|
|
187
|
+
<th>Only crawlers</th>
|
|
188
|
+
<td><code><%= options[:only_crawlers] %></code></td>
|
|
189
|
+
</tr>
|
|
190
|
+
<tr>
|
|
191
|
+
<th>No selfpoll</th>
|
|
192
|
+
<td><code><%= options[:no_selfpoll] %></code></td>
|
|
193
|
+
</tr>
|
|
194
|
+
<tr>
|
|
195
|
+
<th>Filter by date</th>
|
|
196
|
+
<td>
|
|
197
|
+
<code><%= (options[:from_date] != nil or options[:to_date] != nil) %></code>
|
|
198
|
+
</td>
|
|
199
|
+
</tr>
|
|
200
|
+
<tr>
|
|
201
|
+
<th>Prefix</th>
|
|
202
|
+
<td><code><%= @prefix %></code></td>
|
|
203
|
+
</tr>
|
|
204
|
+
<tr>
|
|
205
|
+
<th>Suffix</th>
|
|
206
|
+
<td><code><%= @suffix %></code></td>
|
|
207
|
+
</tr>
|
|
208
|
+
</tbody>
|
|
209
|
+
</table>
|
|
210
|
+
</article>
|
|
211
|
+
</div>
|
|
212
|
+
|
|
213
|
+
<div class="column col-6">
|
|
214
|
+
<article>
|
|
215
|
+
<h2 id="performance"> Performance</h2>
|
|
216
|
+
|
|
217
|
+
<table class="table performance">
|
|
218
|
+
<tbody>
|
|
219
|
+
<tr>
|
|
220
|
+
<th>Analysis started at</th>
|
|
221
|
+
<td><%= data[:started_at].to_s %></td>
|
|
222
|
+
</tr>
|
|
223
|
+
<tr>
|
|
224
|
+
<th>Analysis ended at</th>
|
|
225
|
+
<td><%= data[:ended_at].to_s %></td>
|
|
226
|
+
</tr>
|
|
227
|
+
<tr>
|
|
228
|
+
<th>Duration (sec)</th>
|
|
229
|
+
<td><%= "%.1f" % data[:duration] %></td>
|
|
230
|
+
</tr>
|
|
231
|
+
<tr>
|
|
232
|
+
<th>Duration (min)</th>
|
|
233
|
+
<td><%= "%d" % (data[:duration] / 60 ) %></td>
|
|
234
|
+
</tr>
|
|
235
|
+
<tr>
|
|
236
|
+
<th>Log size</th>
|
|
237
|
+
<td><%= data[:log_size][0][0] %></td>
|
|
238
|
+
</tr>
|
|
239
|
+
<tr>
|
|
240
|
+
<th>Lines/sec</th>
|
|
241
|
+
<td><%= "%.2f" % (data[:log_size][0][0] / data[:duration]) %></td></tr>
|
|
242
|
+
</tbody>
|
|
243
|
+
</table>
|
|
244
|
+
</article>
|
|
245
|
+
</div>
|
|
246
|
+
</div>
|
|
158
247
|
</section>
|
|
159
248
|
</body>
|
|
160
249
|
</html>
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: apache_log_report
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.1.
|
|
4
|
+
version: 1.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Adolfo Villafiorita
|
|
@@ -66,7 +66,8 @@ dependencies:
|
|
|
66
66
|
- - ">="
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
68
|
version: '0'
|
|
69
|
-
description: Generate
|
|
69
|
+
description: Generate requests reports in HTML, OrgMode, and SQLite format from an
|
|
70
|
+
Apache log file.
|
|
70
71
|
email:
|
|
71
72
|
- adolfo.villafiorita@ict4g.net
|
|
72
73
|
executables:
|
|
@@ -121,5 +122,5 @@ requirements: []
|
|
|
121
122
|
rubygems_version: 3.2.22
|
|
122
123
|
signing_key:
|
|
123
124
|
specification_version: 4
|
|
124
|
-
summary: Generate
|
|
125
|
+
summary: Generate analytics from an Apache log file.
|
|
125
126
|
test_files: []
|