apache_log_report 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +6 -6
- data/exe/apache_log_report +16 -4
- data/lib/apache_log_report/apache_log_report.rb +245 -0
- data/lib/apache_log_report/data_cruncher.rb +84 -0
- data/lib/apache_log_report/emitter.rb +50 -0
- data/lib/apache_log_report/log_parser.rb +87 -0
- data/lib/apache_log_report/options_parser.rb +86 -0
- data/lib/apache_log_report/templates/_output_table.html.erb +25 -0
- data/lib/apache_log_report/templates/template.html.erb +164 -0
- data/lib/apache_log_report/templates/template.org.erb +262 -0
- data/lib/apache_log_report/version.rb +1 -1
- data/lib/apache_log_report.rb +5 -532
- metadata +11 -3
@@ -0,0 +1,164 @@
|
|
1
|
+
<html>
|
2
|
+
<head>
|
3
|
+
<title>Apache Log Analysis: <%= data[:log_file] || "stdin" %></title>
|
4
|
+
<meta name="author" content="apache_log_report">
|
5
|
+
|
6
|
+
<link rel="stylesheet" href="alr-styles.css"></style>
|
7
|
+
</head>
|
8
|
+
|
9
|
+
<body>
|
10
|
+
<section>
|
11
|
+
<h1>Apache Log Analysis: <%= data[:log_file] || "stdin" %></h1>
|
12
|
+
|
13
|
+
<article>
|
14
|
+
<h2>Summary</h2>
|
15
|
+
|
16
|
+
<table class="pure-table summary">
|
17
|
+
<tr>
|
18
|
+
<th class="hits">Hits</th>
|
19
|
+
<td class="hits"><%= data[:total_hits][0][0] %></td>
|
20
|
+
</tr>
|
21
|
+
<tr>
|
22
|
+
<th class="unique-visitors">Unique Visitors</th>
|
23
|
+
<td class="unique-visitors"><%= data[:total_unique_visitors][0][0] %></td>
|
24
|
+
</tr>
|
25
|
+
<tr>
|
26
|
+
<th class="tx">Tx</th>
|
27
|
+
<td class="tx"><%= data[:total_size][0][0] %></td>
|
28
|
+
</tr>
|
29
|
+
<tr>
|
30
|
+
<th class="days">Days </th>
|
31
|
+
<td class="days"><%= data[:total_days][0][0] %></td>
|
32
|
+
</tr>
|
33
|
+
</table>
|
34
|
+
</article>
|
35
|
+
|
36
|
+
<% @reports = [
|
37
|
+
{ title: "Daily Distribution", header: ["Day", "Hits", "Visits", "Size"], rows: data[:daily_distribution] },
|
38
|
+
{ title: "Time Distribution", header: ["Hour", "Hits", "Visits", "Size"], rows: data[:time_distribution] },
|
39
|
+
{ title: "Most Requested Pages", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_pages] },
|
40
|
+
{ title: "Most Requested Resources", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_resources] },
|
41
|
+
{ title: "404 on HTML Files", header: ["Path", "Hits", "Visitors"], rows: data[:missed_pages] },
|
42
|
+
{ title: "404 on other Resources", header: ["Path", "Hits", "Visitors"], rows: data[:missed_resources] },
|
43
|
+
{ title: "Attacks", header: ["Path", "Hits", "Visitors"], rows: data[:attacks] },
|
44
|
+
{ title: "Statuses", header: ["Status", "Count"], rows: data[:statuses] },
|
45
|
+
{ title: "Daily Statuses", header: ["Status", "2xx", "3xx", "4xx"], rows: data[:statuses_by_day] },
|
46
|
+
{ title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
|
47
|
+
{ title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
|
48
|
+
{ title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
|
49
|
+
{ title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers] },
|
50
|
+
]
|
51
|
+
%>
|
52
|
+
<% @reports.each do |report| %>
|
53
|
+
<article>
|
54
|
+
<h2><%= report[:title] %></h2>
|
55
|
+
<%= render "output_table", report %>
|
56
|
+
</article>
|
57
|
+
<% end %>
|
58
|
+
|
59
|
+
<article>
|
60
|
+
<h2>Command Invocation and Performance</h2>
|
61
|
+
|
62
|
+
<h3>Command Invocation</h3>
|
63
|
+
|
64
|
+
<code>
|
65
|
+
</code>
|
66
|
+
|
67
|
+
<table class="pure-table command-invocation">
|
68
|
+
<tbody>
|
69
|
+
<tr>
|
70
|
+
<th>CLI Command</th>
|
71
|
+
<td><pre><%= data[:command] %></pre></td>
|
72
|
+
</tr>
|
73
|
+
<tr>
|
74
|
+
<th>Input file</th>
|
75
|
+
<td><code><%= (data[:log_file] || "stdin") %></code></td>
|
76
|
+
</tr>
|
77
|
+
<tr>
|
78
|
+
<th>Ignore crawlers</th>
|
79
|
+
<td><code><%= options[:ignore_crawlers] %></code></td></tr>
|
80
|
+
<tr>
|
81
|
+
<th>Only crawlers</th>
|
82
|
+
<td><code><%= options[:only_crawlers] %></code></td>
|
83
|
+
</tr>
|
84
|
+
<tr>
|
85
|
+
<th>No selfpoll</th>
|
86
|
+
<td><code><%= options[:no_selfpoll] %></code></td>
|
87
|
+
</tr>
|
88
|
+
<tr>
|
89
|
+
<th>Filter by date</th>
|
90
|
+
<td>
|
91
|
+
<code><%= (options[:from_date] != nil or options[:to_date] != nil) %></code>
|
92
|
+
</td>
|
93
|
+
</tr>
|
94
|
+
<tr>
|
95
|
+
<th>Prefix</th>
|
96
|
+
<td><code><%= @prefix %></code></td>
|
97
|
+
</tr>
|
98
|
+
<tr>
|
99
|
+
<th>Suffix</th>
|
100
|
+
<td><code><%= @suffix %></code></td>
|
101
|
+
</tr>
|
102
|
+
</tbody>
|
103
|
+
</table>
|
104
|
+
|
105
|
+
<h3> Log Structure</h3>
|
106
|
+
|
107
|
+
<table class="pure-table log-structure">
|
108
|
+
<tbody>
|
109
|
+
<tr>
|
110
|
+
<th>Log size</th>
|
111
|
+
<td><%= data[:log_size][0][0] %></td>
|
112
|
+
</tr>
|
113
|
+
<tr>
|
114
|
+
<th>Self poll entries</th>
|
115
|
+
<td><%= data[:selfpolls_size][0][0] %></td>
|
116
|
+
</tr>
|
117
|
+
<tr>
|
118
|
+
<th>Crawlers</th>
|
119
|
+
<td><%= data[:crawlers_size][0][0] %></td>
|
120
|
+
</tr>
|
121
|
+
<tr>
|
122
|
+
<th>Entries considered</th>
|
123
|
+
<td><%= data[:total_hits][0][0] %></td>
|
124
|
+
</tr>
|
125
|
+
</tbody>
|
126
|
+
</table>
|
127
|
+
|
128
|
+
<h3> Performance</h3>
|
129
|
+
|
130
|
+
<table class="pure-table performance">
|
131
|
+
<tbody>
|
132
|
+
<tr>
|
133
|
+
<th>Analysis started at</th>
|
134
|
+
<td><%= data[:started_at].to_s %></td>
|
135
|
+
</tr>
|
136
|
+
<tr>
|
137
|
+
<th>Analysis ended at</th>
|
138
|
+
<td><%= data[:ended_at].to_s %></td>
|
139
|
+
</tr>
|
140
|
+
<tr>
|
141
|
+
<th>Duration (sec)</th>
|
142
|
+
<td><%= "%.1f" % data[:duration] %></td>
|
143
|
+
</tr>
|
144
|
+
<tr>
|
145
|
+
<th>Duration (min)</th>
|
146
|
+
<td><%= "%d" % (data[:duration] / 60 ) %></td>
|
147
|
+
</tr>
|
148
|
+
<tr>
|
149
|
+
<th>Log size</th>
|
150
|
+
<td><%= data[:log_size][0][0] %></td>
|
151
|
+
</tr>
|
152
|
+
<tr>
|
153
|
+
<th>Lines/sec</th>
|
154
|
+
<td><%= "%.2f" % (data[:log_size][0][0] / data[:duration]) %></td></tr>
|
155
|
+
</tbody>
|
156
|
+
</table>
|
157
|
+
</article>
|
158
|
+
</section>
|
159
|
+
</body>
|
160
|
+
</html>
|
161
|
+
|
162
|
+
|
163
|
+
|
164
|
+
|
@@ -0,0 +1,262 @@
|
|
1
|
+
#+TITLE: Apache Log Analysis: <%= data[:log_file] %>
|
2
|
+
#+DATE: <<%= Date.today %>>
|
3
|
+
#+STARTUP: showall
|
4
|
+
#+OPTIONS: ^:{}
|
5
|
+
#+HTML_HEAD: <link rel="stylesheet" type="text/css" href="ala-style.css" />
|
6
|
+
#+OPTIONS: html-style:nil
|
7
|
+
|
8
|
+
* Summary
|
9
|
+
|
10
|
+
| Hits | <%= "%10d" % data[:total_hits][0][0] %> |
|
11
|
+
| Unique Visitors | <%= "%10d" % data[:total_unique_visitors][0][0] %> |
|
12
|
+
| Tx | <%= "%10s" % data[:total_size][0][0] %> |
|
13
|
+
| Days | <%= "%10d" % data[:total_days][0][0] %> |
|
14
|
+
|
15
|
+
* Daily Distribution
|
16
|
+
|
17
|
+
<%= self.output_table "daily_distribution", ["Day", "Hits", "Visits", "Size"], data[:daily_distribution] %>
|
18
|
+
|
19
|
+
#+BEGIN_SRC gnuplot :var data = daily_distribution :results output :exports <%= @export %> :file <%= @prefix %>daily<%= @suffix %>.svg
|
20
|
+
reset
|
21
|
+
set grid ytics linestyle 0
|
22
|
+
set grid xtics linestyle 0
|
23
|
+
set terminal svg size 1200,800 fname 'Arial'
|
24
|
+
|
25
|
+
set xdata time
|
26
|
+
set timefmt "%Y-%m-%d"
|
27
|
+
set format x "%a, %b %d"
|
28
|
+
set xtics rotate by 60 right
|
29
|
+
|
30
|
+
set title "Hits and Visitors"
|
31
|
+
set xlabel "Date"
|
32
|
+
set ylabel "Hits"
|
33
|
+
set y2label "Visits"
|
34
|
+
set y2tics
|
35
|
+
|
36
|
+
set style fill transparent solid 0.2 noborder
|
37
|
+
|
38
|
+
plot data using 1:2 with linespoints lw 3 lc rgb "#0000AA" pointtype 5 title "Hits" axes x1y2, \\
|
39
|
+
data using 1:2 with filledcurves below x1 linecolor rgb "#0000AA" notitle axes x1y2, \\
|
40
|
+
data using 1:3 with linespoints lw 3 lc rgb "#AA0000" pointtype 7 title "Visitors", \\
|
41
|
+
data using 1:3 with filledcurves below x1 notitle linecolor rgb "#AA0000", \\
|
42
|
+
data using 1:($3+0.1*$3):3 with labels notitle textcolor rgb "#AA0000", \\
|
43
|
+
data using 1:($2+0.1*$2):2 with labels notitle textcolor rgb "#0000AA" axes x1y2
|
44
|
+
#+END_SRC
|
45
|
+
|
46
|
+
|
47
|
+
* Time Distribution
|
48
|
+
|
49
|
+
<%= self.output_table "time_distribution", ["Hour", "Hits", "Visits", "Size"], data[:time_distribution] %>
|
50
|
+
|
51
|
+
|
52
|
+
#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports <%= @export %> :file <%= @prefix %>time<%= @suffix %>.svg
|
53
|
+
reset
|
54
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
55
|
+
|
56
|
+
set grid ytics linestyle 0
|
57
|
+
|
58
|
+
set title "Hits and Visitors"
|
59
|
+
set xlabel "Date"
|
60
|
+
set ylabel "Hits"
|
61
|
+
set y2label "Visitors"
|
62
|
+
set y2tics
|
63
|
+
|
64
|
+
set style fill solid 0.25
|
65
|
+
set boxwidth 0.6
|
66
|
+
|
67
|
+
set style data histograms
|
68
|
+
set style histogram clustered gap 1
|
69
|
+
|
70
|
+
plot data using 2:xtic(1) lc rgb "#0000AA" title "Hits", \\
|
71
|
+
data using 3 lc rgb "#AA0000" title "Visitors" axes x1y2, \\
|
72
|
+
data using ($0 - 0.2):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#0000AA"), \\
|
73
|
+
data using ($0 + 0.2):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#AA0000") axes x1y2
|
74
|
+
#+END_SRC
|
75
|
+
|
76
|
+
#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports <%= @export %> :file <%= @prefix %>time-traffic<%= @suffix %>.svg
|
77
|
+
reset
|
78
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
79
|
+
|
80
|
+
set grid ytics linestyle 0
|
81
|
+
|
82
|
+
set title "Traffic"
|
83
|
+
set xlabel "Date"
|
84
|
+
set ylabel "Traffic"
|
85
|
+
|
86
|
+
set style fill solid 0.50
|
87
|
+
set boxwidth 0.6
|
88
|
+
|
89
|
+
set style data histograms
|
90
|
+
set style histogram clustered gap 1
|
91
|
+
|
92
|
+
plot data using 2:xtic(1) lc rgb "#00AA00" title "Traffic", \\
|
93
|
+
data using ($0):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00")
|
94
|
+
#+END_SRC
|
95
|
+
|
96
|
+
* Most Requested Pages
|
97
|
+
|
98
|
+
<%= self.output_table "most_requested_pages", ["Path", "Hits", "Visits", "Size"], data[:most_requested_pages] %>
|
99
|
+
|
100
|
+
* Most Requested URIs
|
101
|
+
|
102
|
+
<%= self.output_table "most_requested_resources", ["Path", "Hits", "Visits", "Size"], data[:most_requested_resources] %>
|
103
|
+
|
104
|
+
* 404s on HTML files
|
105
|
+
|
106
|
+
<%= self.output_table "pages_404", ["Path", "Hits", "Visitors"], data[:missed_pages] %>
|
107
|
+
|
108
|
+
* 404s on other resources
|
109
|
+
|
110
|
+
<%= self.output_table "resources_404", ["Path", "Hits", "Visitors"], data[:missed_resources] %>
|
111
|
+
|
112
|
+
* Possible Attacks
|
113
|
+
|
114
|
+
<%= self.output_table "attacks", ["Path", "Hits", "Visitors"], data[:attacks] %>
|
115
|
+
|
116
|
+
* Statuses
|
117
|
+
|
118
|
+
<%= self.output_table "statuses", ["Status", "Count"], data[:statuses] %>
|
119
|
+
|
120
|
+
#+BEGIN_SRC gnuplot :var data = statuses :results output :exports <%= @export %> :file <%= @prefix %>statuses<%= @suffix %>.svg
|
121
|
+
reset
|
122
|
+
set grid ytics linestyle 0
|
123
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
124
|
+
|
125
|
+
set style fill solid 0.25
|
126
|
+
set boxwidth 0.6
|
127
|
+
|
128
|
+
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
129
|
+
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
130
|
+
#+END_SRC
|
131
|
+
|
132
|
+
* Daily Statuses
|
133
|
+
|
134
|
+
<%= self.output_table "daily_statuses", ["Status", "2xx", "3xx", "4xx"], data[:statuses_by_day] %>
|
135
|
+
|
136
|
+
#+BEGIN_SRC gnuplot :var data = daily_statuses :results output :exports <%= @export %> :file <%= @prefix %>daily-statuses<%= @suffix %>.svg
|
137
|
+
reset
|
138
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
139
|
+
|
140
|
+
set grid ytics linestyle 0
|
141
|
+
|
142
|
+
set title "Daily Statuses"
|
143
|
+
set xlabel "Date"
|
144
|
+
set ylabel "Number of Hits"
|
145
|
+
set xtics rotate by 60 right
|
146
|
+
|
147
|
+
set style fill solid 0.25
|
148
|
+
set boxwidth 0.6
|
149
|
+
|
150
|
+
set style data histograms
|
151
|
+
set style histogram clustered gap 1
|
152
|
+
|
153
|
+
plot data using 2:xtic(1) lc rgb "#00AA00" title "2xx", \\
|
154
|
+
data using 3 lc rgb "#0000CC" title "3xx", \\
|
155
|
+
data using 4 lc rgb "#AA0000" title "4xx", \\
|
156
|
+
data using ($0 - 1. / 4):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00"), \\
|
157
|
+
data using ($0):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0000CC"), \\
|
158
|
+
data using ($0 + 1. / 4):($4 + 0.1*$4):4 with labels title "" textcolor rgb("#AA0000")
|
159
|
+
#+END_SRC
|
160
|
+
|
161
|
+
* Browsers
|
162
|
+
|
163
|
+
<%= self.output_table "browsers", ["Browser", "Hits", "Visitors", "Size"], data[:browsers] %>
|
164
|
+
|
165
|
+
#+BEGIN_SRC gnuplot :var data = browsers :results output :exports <%= @export %> :file <%= @prefix %>browser<%= @suffix %>.svg
|
166
|
+
reset
|
167
|
+
set grid ytics linestyle 0
|
168
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
169
|
+
|
170
|
+
set style fill solid 0.25
|
171
|
+
set boxwidth 0.6
|
172
|
+
|
173
|
+
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
174
|
+
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
175
|
+
#+END_SRC
|
176
|
+
|
177
|
+
* Platforms
|
178
|
+
|
179
|
+
<%= self.output_table "platforms", ["Platform", "Hits", "Visitors", "Size"], data[:platforms] %>
|
180
|
+
|
181
|
+
#+BEGIN_SRC gnuplot :var data = platforms :results output :exports <%= @export %> :file <%= @prefix %>platforms<%= @suffix %>.svg
|
182
|
+
reset
|
183
|
+
set grid ytics linestyle 0
|
184
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
185
|
+
|
186
|
+
set style fill solid 0.25
|
187
|
+
set boxwidth 0.6
|
188
|
+
|
189
|
+
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
190
|
+
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
191
|
+
#+END_SRC
|
192
|
+
|
193
|
+
* IPs
|
194
|
+
|
195
|
+
<%= self.output_table "ips", ["IPs", "Hits", "Visitors", "Size"], data[:ips] %>
|
196
|
+
|
197
|
+
|
198
|
+
* Referers
|
199
|
+
|
200
|
+
<%= self.output_table "referers", ["Referers", "Hits", "Visitors", "Size"], data[:referers] %>
|
201
|
+
|
202
|
+
#+BEGIN_SRC gnuplot :var data = referers :results output :exports <%= @export %> :file <%= @prefix %>referers<%= @suffix %>.svg
|
203
|
+
reset
|
204
|
+
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
205
|
+
|
206
|
+
set grid ytics linestyle 0
|
207
|
+
set grid xtics linestyle 0
|
208
|
+
|
209
|
+
set title "Referers"
|
210
|
+
set xlabel "Date"
|
211
|
+
set xtics rotate by 60 right
|
212
|
+
set ylabel "Hits and Visits"
|
213
|
+
|
214
|
+
set style fill solid 0.45
|
215
|
+
set boxwidth 0.7
|
216
|
+
|
217
|
+
set style data histograms
|
218
|
+
set style histogram clustered gap 1
|
219
|
+
|
220
|
+
plot data using 2:xtic(1) lc rgb "#AA00AA" title "Hits", \\
|
221
|
+
data using 3 lc rgb "#0AAAA0" title "Visits", \\
|
222
|
+
data using ($0 - 1. / 3):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#AA00AA"), \\
|
223
|
+
data using ($0 + 1. / 3):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0AAAA0")
|
224
|
+
#+END_SRC
|
225
|
+
|
226
|
+
* Command Invocation and Performance
|
227
|
+
|
228
|
+
** Command Invocation
|
229
|
+
|
230
|
+
#+BEGIN_EXAMPLE shell
|
231
|
+
<%= data[:command] %>
|
232
|
+
#+END_EXAMPLE
|
233
|
+
|
234
|
+
| Input file | <%= "%-50s" % (data[:log_file] || "stdin") %> |
|
235
|
+
| Ignore crawlers | <%= "%-50s" % options[:ignore_crawlers] %> |
|
236
|
+
| Only crawlers | <%= "%-50s" % options[:only_crawlers] %> |
|
237
|
+
| No selfpoll | <%= "%-50s" % options[:no_selfpoll] %> |
|
238
|
+
| Filter by date | <%= "%-50s" % (options[:from_date] != nil or options[:to_date] != nil) %> |
|
239
|
+
| Prefix | <%= "%-50s" % @prefix %> |
|
240
|
+
| Suffix | <%= "%-50s" % @suffix %> |
|
241
|
+
|
242
|
+
** Log Structure
|
243
|
+
|
244
|
+
| Log size | <%= "%10d" % data[:log_size][0][0] %> |
|
245
|
+
| Self poll entries | <%= "%10d" % data[:selfpolls_size][0][0] %> |
|
246
|
+
| Crawlers | <%= "%10d" % data[:crawlers_size][0][0] %> |
|
247
|
+
| Entries considered | <%= "%10d" % data[:total_hits][0][0] %> |
|
248
|
+
|
249
|
+
** Performance
|
250
|
+
|
251
|
+
| Analysis started at | <%= data[:started_at].to_s %> |
|
252
|
+
| Analysis ended at | <%= data[:ended_at].to_s %> |
|
253
|
+
| Duration (sec) | <%= "%5.3d" % data[:duration] %> |
|
254
|
+
| Duration (min) | <%= "%5.3d" % (data[:duration] / 60 ) %> |
|
255
|
+
| Log size | <%= "%9d" % data[:log_size][0][0] %> |
|
256
|
+
| Lines/sec | <%= "%6.2f" % (data[:log_size][0][0] / data[:duration]) %> |
|
257
|
+
|
258
|
+
* Local Variables :noexport:
|
259
|
+
# Local Variables:
|
260
|
+
# org-confirm-babel-evaluate: nil
|
261
|
+
# org-display-inline-images: t
|
262
|
+
# end:
|