apache_log_report 1.1.0 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.org +2 -4
- data/apache_log_report.gemspec +2 -2
- data/exe/apache_log_report +24 -17
- data/lib/apache_log_report/data_cruncher.rb +2 -0
- data/lib/apache_log_report/emitter.rb +0 -1
- data/lib/apache_log_report/options_parser.rb +1 -1
- data/lib/apache_log_report/templates/_output_table.html.erb +1 -1
- data/lib/apache_log_report/templates/template.html.erb +219 -130
- data/lib/apache_log_report/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8b90448ec269bd64d60c2323888fb77ea8ac7307a00198434119a6f0da605585
|
|
4
|
+
data.tar.gz: 24569c78964b795478c50e5de01c4f42fb2be8f02c30911e4e81229a96917a55
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 604697690824080fbbed523f26c1727f969fcf791803ebd15d6838384c5c77edbcde25ad74fa47c2371c0690019745948b6d78f9f1a51e6742e51e35c5c5ba76
|
|
7
|
+
data.tar.gz: b7d5a6940308a6a098cb61b9118cd17760592e299e091193b07a003ae87e6f7058ee1aab81aed2cadd65dcc07c6b64652ede126274be5be552dbbf0ca3ba11b1
|
data/Gemfile.lock
CHANGED
data/README.org
CHANGED
|
@@ -14,10 +14,8 @@ See the [[file:CHANGELOG.org][CHANGELOG]] file.
|
|
|
14
14
|
|
|
15
15
|
* Todo
|
|
16
16
|
|
|
17
|
-
** TODO
|
|
18
|
-
** TODO
|
|
19
|
-
** TODO Move performance stats var to class (to isolate vars)
|
|
20
|
-
** TODO Check total number of days (which is not working, now)
|
|
17
|
+
** TODO Graphs in HTML output
|
|
18
|
+
** TODO Countries
|
|
21
19
|
|
|
22
20
|
* Compatibility
|
|
23
21
|
|
data/apache_log_report.gemspec
CHANGED
|
@@ -6,8 +6,8 @@ Gem::Specification.new do |spec|
|
|
|
6
6
|
spec.authors = ["Adolfo Villafiorita"]
|
|
7
7
|
spec.email = ["adolfo.villafiorita@ict4g.net"]
|
|
8
8
|
|
|
9
|
-
spec.summary = %q{Generate
|
|
10
|
-
spec.description = %q{Generate
|
|
9
|
+
spec.summary = %q{Generate analytics from an Apache log file.}
|
|
10
|
+
spec.description = %q{Generate requests reports in HTML, OrgMode, and SQLite format from an Apache log file.}
|
|
11
11
|
spec.homepage = "https://www.ict4g.net/gitea/adolfo/apache_log_report"
|
|
12
12
|
spec.license = "MIT"
|
|
13
13
|
spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
|
data/exe/apache_log_report
CHANGED
|
@@ -23,21 +23,28 @@ end
|
|
|
23
23
|
@started_at = Time.now
|
|
24
24
|
|
|
25
25
|
@db = ApacheLogReport::LogParser.parse @log_file
|
|
26
|
-
@data = ApacheLogReport::DataCruncher.crunch @db, @options
|
|
27
26
|
|
|
28
|
-
@
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
27
|
+
if @options[:output_format] == "sqlite"
|
|
28
|
+
ddb = SQLite3::Database.new('db.sqlite3')
|
|
29
|
+
b = SQLite3::Backup.new(ddb, 'main', @db, 'main')
|
|
30
|
+
b.step(-1) #=> DONE
|
|
31
|
+
b.finish
|
|
32
|
+
else
|
|
33
|
+
@data = ApacheLogReport::DataCruncher.crunch @db, @options
|
|
34
|
+
|
|
35
|
+
@ended_at = Time.now
|
|
36
|
+
@duration = @ended_at - @started_at
|
|
37
|
+
|
|
38
|
+
@data = @data.merge({
|
|
39
|
+
command: @command,
|
|
40
|
+
log_file: @log_file,
|
|
41
|
+
started_at: @started_at,
|
|
42
|
+
ended_at: @ended_at,
|
|
43
|
+
duration: @duration
|
|
44
|
+
})
|
|
45
|
+
|
|
46
|
+
#
|
|
47
|
+
# Emit Output
|
|
48
|
+
#
|
|
49
|
+
puts ApacheLogReport::Emitter.emit @data, @options
|
|
50
|
+
end
|
|
@@ -71,6 +71,8 @@ module ApacheLogReport
|
|
|
71
71
|
@ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by ip order by count(ip) desc limit #{options[:limit]}"
|
|
72
72
|
@referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{filter} group by referer order by count(referer) desc limit #{options[:limit]}"
|
|
73
73
|
|
|
74
|
+
@streaks = db.execute "SELECT ip, substr(datetime, 1, 10), path from LogLine order by ip, datetime"
|
|
75
|
+
|
|
74
76
|
data = {}
|
|
75
77
|
self.instance_variables.each do |variable|
|
|
76
78
|
var_as_symbol = variable.to_s[1..-1].to_sym
|
|
@@ -50,7 +50,7 @@ module ApacheLogReport
|
|
|
50
50
|
args[:code_export] = n
|
|
51
51
|
end
|
|
52
52
|
|
|
53
|
-
opts.on("-fFORMAT", "--format=FORMAT", String, "Output format: html, org. Defaults to org mode") do |n|
|
|
53
|
+
opts.on("-fFORMAT", "--format=FORMAT", String, "Output format: html, org, sqlite. Defaults to org mode") do |n|
|
|
54
54
|
args[:output_format] = n
|
|
55
55
|
end
|
|
56
56
|
|
|
@@ -4,34 +4,106 @@
|
|
|
4
4
|
<meta name="author" content="apache_log_report">
|
|
5
5
|
|
|
6
6
|
<link rel="stylesheet" href="alr-styles.css"></style>
|
|
7
|
+
|
|
8
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre.min.css">
|
|
9
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre-exp.min.css">
|
|
10
|
+
<link rel="stylesheet" href="https://unpkg.com/spectre.css/dist/spectre-icons.min.css">
|
|
7
11
|
</head>
|
|
8
12
|
|
|
9
|
-
<body>
|
|
10
|
-
<
|
|
11
|
-
|
|
13
|
+
<body class="container">
|
|
14
|
+
<h1>Apache Log Analysis: <%= data[:log_file] || "stdin" %></h1>
|
|
15
|
+
|
|
16
|
+
<nav>
|
|
17
|
+
<ul class="nav">
|
|
18
|
+
<li class="nav-item active">
|
|
19
|
+
<a href="#">Navigation</a>
|
|
20
|
+
<ul class="nav">
|
|
21
|
+
<% [ "Summary",
|
|
22
|
+
"Log Structure",
|
|
23
|
+
"Daily Distribution",
|
|
24
|
+
"Time Distribution",
|
|
25
|
+
"Most Requested Pages",
|
|
26
|
+
"Most Requested Resources",
|
|
27
|
+
"404 on HTML Files",
|
|
28
|
+
"404 on other Resources",
|
|
29
|
+
"Attacks",
|
|
30
|
+
"Statuses",
|
|
31
|
+
"Daily Statuses",
|
|
32
|
+
"Browsers",
|
|
33
|
+
"Platforms",
|
|
34
|
+
"Referers",
|
|
35
|
+
"IPs"
|
|
36
|
+
].each do |item| %>
|
|
37
|
+
<li class="nav-item">
|
|
38
|
+
<a href="#<%= item.downcase.gsub(' ', '-') %>"><%= item %></a>
|
|
39
|
+
</li>
|
|
40
|
+
<% end %>
|
|
41
|
+
</ul>
|
|
42
|
+
</li>
|
|
43
|
+
</ul>
|
|
44
|
+
</nav>
|
|
12
45
|
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
<
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
46
|
+
<section>
|
|
47
|
+
<div class="columns">
|
|
48
|
+
<article class="col-6 column">
|
|
49
|
+
<h2 id="summary">Summary</h2>
|
|
50
|
+
|
|
51
|
+
<table class="table summary">
|
|
52
|
+
<tr>
|
|
53
|
+
<th class="hits">Hits</th>
|
|
54
|
+
<td class="hits"><%= data[:total_hits][0][0] %></td>
|
|
55
|
+
</tr>
|
|
56
|
+
<tr>
|
|
57
|
+
<th class="unique-visitors">Unique Visitors</th>
|
|
58
|
+
<td class="unique-visitors"><%= data[:total_unique_visitors][0][0] %></td>
|
|
59
|
+
</tr>
|
|
60
|
+
<tr>
|
|
61
|
+
<th class="tx">Tx</th>
|
|
62
|
+
<td class="tx"><%= data[:total_size][0][0] %></td>
|
|
63
|
+
</tr>
|
|
64
|
+
<tr>
|
|
65
|
+
<th class="period">Period</th>
|
|
66
|
+
<td class="period">
|
|
67
|
+
<%= data[:first_day][0][0] %>
|
|
68
|
+
--
|
|
69
|
+
<%= data[:last_day][0][0] %>
|
|
70
|
+
</td>
|
|
71
|
+
</tr>
|
|
72
|
+
<tr>
|
|
73
|
+
<th class="days">Days </th>
|
|
74
|
+
<td class="days"><%= data[:total_days] %></td>
|
|
75
|
+
</tr>
|
|
76
|
+
</table>
|
|
77
|
+
</article>
|
|
78
|
+
<article class="column col-6">
|
|
79
|
+
<h2 id="log-structure">Log Structure</h2>
|
|
80
|
+
|
|
81
|
+
<table class="table log-structure">
|
|
82
|
+
<tbody>
|
|
83
|
+
<tr>
|
|
84
|
+
<th>Input file</th>
|
|
85
|
+
<td><b><%= (data[:log_file] || "stdin") %></b></td>
|
|
86
|
+
</tr>
|
|
87
|
+
<tr>
|
|
88
|
+
<th>Log size</th>
|
|
89
|
+
<td><%= data[:log_size][0][0] %></td>
|
|
90
|
+
</tr>
|
|
91
|
+
<tr>
|
|
92
|
+
<th>Self poll entries</th>
|
|
93
|
+
<td><%= data[:selfpolls_size][0][0] %></td>
|
|
94
|
+
</tr>
|
|
95
|
+
<tr>
|
|
96
|
+
<th>Crawlers</th>
|
|
97
|
+
<td><%= data[:crawlers_size][0][0] %></td>
|
|
98
|
+
</tr>
|
|
99
|
+
<tr>
|
|
100
|
+
<th>Entries considered</th>
|
|
101
|
+
<td><%= data[:total_hits][0][0] %></td>
|
|
102
|
+
</tr>
|
|
103
|
+
</tbody>
|
|
104
|
+
</table>
|
|
105
|
+
</article>
|
|
106
|
+
</div>
|
|
35
107
|
|
|
36
108
|
<% @reports = [
|
|
37
109
|
{ title: "Daily Distribution", header: ["Day", "Hits", "Visits", "Size"], rows: data[:daily_distribution] },
|
|
@@ -41,120 +113,137 @@
|
|
|
41
113
|
{ title: "404 on HTML Files", header: ["Path", "Hits", "Visitors"], rows: data[:missed_pages] },
|
|
42
114
|
{ title: "404 on other Resources", header: ["Path", "Hits", "Visitors"], rows: data[:missed_resources] },
|
|
43
115
|
{ title: "Attacks", header: ["Path", "Hits", "Visitors"], rows: data[:attacks] },
|
|
116
|
+
{ },
|
|
44
117
|
{ title: "Statuses", header: ["Status", "Count"], rows: data[:statuses] },
|
|
45
118
|
{ title: "Daily Statuses", header: ["Status", "2xx", "3xx", "4xx"], rows: data[:statuses_by_day] },
|
|
46
119
|
{ title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
|
|
47
120
|
{ title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
|
|
121
|
+
{ title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers], col: "col-12" },
|
|
48
122
|
{ title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
|
|
49
|
-
{
|
|
123
|
+
{ },
|
|
50
124
|
]
|
|
51
125
|
%>
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
<
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
126
|
+
<div class="columns">
|
|
127
|
+
<% @reports.each do |report| %>
|
|
128
|
+
<div class="column <%= report[:col] || "col-6" %>">
|
|
129
|
+
<article>
|
|
130
|
+
<% if report[:title] != nil %>
|
|
131
|
+
<h2 id="<%= report[:title].downcase.gsub(/ +/, '-') %>">
|
|
132
|
+
<%= report[:title] %>
|
|
133
|
+
</h2>
|
|
134
|
+
<%= render "output_table", report %>
|
|
135
|
+
<% end %>
|
|
136
|
+
</article>
|
|
137
|
+
</div>
|
|
138
|
+
<% end %>
|
|
139
|
+
</div>
|
|
58
140
|
|
|
59
141
|
<article>
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
<
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
<
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
<
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
142
|
+
<h2 id="streaks">Streaks</h2>
|
|
143
|
+
|
|
144
|
+
<table class="table streaks">
|
|
145
|
+
<thead>
|
|
146
|
+
<tr>
|
|
147
|
+
<th>IP</th>
|
|
148
|
+
<th>Day and URL</th>
|
|
149
|
+
</tr>
|
|
150
|
+
</thead>
|
|
151
|
+
<tbody>
|
|
152
|
+
<% data[:streaks].group_by(&:first).each do |ip, date_urls| %>
|
|
153
|
+
<tr>
|
|
154
|
+
<td class="ip"><%= ip %></td>
|
|
155
|
+
<td class="streaks">
|
|
156
|
+
<% date_urls.group_by(&:first).each do |date, urls| %>
|
|
157
|
+
<% urls.each do |url| %>
|
|
158
|
+
<b><%= url[1] %>:</b> <%= url[2] %> <br />
|
|
159
|
+
<% end %>
|
|
160
|
+
<% end %>
|
|
161
|
+
</td>
|
|
162
|
+
</tr>
|
|
163
|
+
<% end %>
|
|
164
|
+
</tbody>
|
|
165
|
+
</table>
|
|
166
|
+
</article>
|
|
167
|
+
|
|
168
|
+
<div class="columns">
|
|
169
|
+
<div class="column col-6">
|
|
170
|
+
<article>
|
|
171
|
+
<h2 id="command-invocation">Command Invocation</h2>
|
|
172
|
+
|
|
173
|
+
<table class="table command-invocation">
|
|
174
|
+
<tbody>
|
|
175
|
+
<tr>
|
|
176
|
+
<th>CLI Command</th>
|
|
177
|
+
<td><pre><%= data[:command] %></pre></td>
|
|
178
|
+
</tr>
|
|
179
|
+
<tr>
|
|
180
|
+
<th>Input file</th>
|
|
181
|
+
<td><code><%= (data[:log_file] || "stdin") %></code></td>
|
|
182
|
+
</tr>
|
|
183
|
+
<tr>
|
|
184
|
+
<th>Ignore crawlers</th>
|
|
185
|
+
<td><code><%= options[:ignore_crawlers] %></code></td></tr>
|
|
186
|
+
<tr>
|
|
187
|
+
<th>Only crawlers</th>
|
|
188
|
+
<td><code><%= options[:only_crawlers] %></code></td>
|
|
189
|
+
</tr>
|
|
190
|
+
<tr>
|
|
191
|
+
<th>No selfpoll</th>
|
|
192
|
+
<td><code><%= options[:no_selfpoll] %></code></td>
|
|
193
|
+
</tr>
|
|
194
|
+
<tr>
|
|
195
|
+
<th>Filter by date</th>
|
|
196
|
+
<td>
|
|
197
|
+
<code><%= (options[:from_date] != nil or options[:to_date] != nil) %></code>
|
|
198
|
+
</td>
|
|
199
|
+
</tr>
|
|
200
|
+
<tr>
|
|
201
|
+
<th>Prefix</th>
|
|
202
|
+
<td><code><%= @prefix %></code></td>
|
|
203
|
+
</tr>
|
|
204
|
+
<tr>
|
|
205
|
+
<th>Suffix</th>
|
|
206
|
+
<td><code><%= @suffix %></code></td>
|
|
207
|
+
</tr>
|
|
208
|
+
</tbody>
|
|
209
|
+
</table>
|
|
210
|
+
</article>
|
|
211
|
+
</div>
|
|
212
|
+
|
|
213
|
+
<div class="column col-6">
|
|
214
|
+
<article>
|
|
215
|
+
<h2 id="performance"> Performance</h2>
|
|
216
|
+
|
|
217
|
+
<table class="table performance">
|
|
218
|
+
<tbody>
|
|
219
|
+
<tr>
|
|
220
|
+
<th>Analysis started at</th>
|
|
221
|
+
<td><%= data[:started_at].to_s %></td>
|
|
222
|
+
</tr>
|
|
223
|
+
<tr>
|
|
224
|
+
<th>Analysis ended at</th>
|
|
225
|
+
<td><%= data[:ended_at].to_s %></td>
|
|
226
|
+
</tr>
|
|
227
|
+
<tr>
|
|
228
|
+
<th>Duration (sec)</th>
|
|
229
|
+
<td><%= "%.1f" % data[:duration] %></td>
|
|
230
|
+
</tr>
|
|
231
|
+
<tr>
|
|
232
|
+
<th>Duration (min)</th>
|
|
233
|
+
<td><%= "%d" % (data[:duration] / 60 ) %></td>
|
|
234
|
+
</tr>
|
|
235
|
+
<tr>
|
|
236
|
+
<th>Log size</th>
|
|
237
|
+
<td><%= data[:log_size][0][0] %></td>
|
|
238
|
+
</tr>
|
|
239
|
+
<tr>
|
|
240
|
+
<th>Lines/sec</th>
|
|
241
|
+
<td><%= "%.2f" % (data[:log_size][0][0] / data[:duration]) %></td></tr>
|
|
242
|
+
</tbody>
|
|
243
|
+
</table>
|
|
244
|
+
</article>
|
|
245
|
+
</div>
|
|
246
|
+
</div>
|
|
158
247
|
</section>
|
|
159
248
|
</body>
|
|
160
249
|
</html>
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: apache_log_report
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.1.
|
|
4
|
+
version: 1.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Adolfo Villafiorita
|
|
@@ -66,7 +66,8 @@ dependencies:
|
|
|
66
66
|
- - ">="
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
68
|
version: '0'
|
|
69
|
-
description: Generate
|
|
69
|
+
description: Generate requests reports in HTML, OrgMode, and SQLite format from an
|
|
70
|
+
Apache log file.
|
|
70
71
|
email:
|
|
71
72
|
- adolfo.villafiorita@ict4g.net
|
|
72
73
|
executables:
|
|
@@ -121,5 +122,5 @@ requirements: []
|
|
|
121
122
|
rubygems_version: 3.2.22
|
|
122
123
|
signing_key:
|
|
123
124
|
specification_version: 4
|
|
124
|
-
summary: Generate
|
|
125
|
+
summary: Generate analytics from an Apache log file.
|
|
125
126
|
test_files: []
|