apache_log_report 1.0.0 → 1.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (14) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile.lock +6 -6
  3. data/exe/apache_log_report +16 -4
  4. data/lib/apache_log_report/apache_log_report.rb +245 -0
  5. data/lib/apache_log_report/data_cruncher.rb +84 -0
  6. data/lib/apache_log_report/emitter.rb +50 -0
  7. data/lib/apache_log_report/log_parser.rb +87 -0
  8. data/lib/apache_log_report/options_parser.rb +86 -0
  9. data/lib/apache_log_report/templates/_output_table.html.erb +25 -0
  10. data/lib/apache_log_report/templates/template.html.erb +164 -0
  11. data/lib/apache_log_report/templates/template.org.erb +262 -0
  12. data/lib/apache_log_report/version.rb +1 -1
  13. data/lib/apache_log_report.rb +5 -532
  14. metadata +11 -3
data/lib/apache_log_report/templates/template.html.erb ADDED
@@ -0,0 +1,164 @@
1
+ <html>
2
+ <head>
3
+ <title>Apache Log Analysis: <%= data[:log_file] || "stdin" %></title>
4
+ <meta name="author" content="apache_log_report">
5
+
6
+ <link rel="stylesheet" href="alr-styles.css"></style>
7
+ </head>
8
+
9
+ <body>
10
+ <section>
11
+ <h1>Apache Log Analysis: <%= data[:log_file] || "stdin" %></h1>
12
+
13
+ <article>
14
+ <h2>Summary</h2>
15
+
16
+ <table class="pure-table summary">
17
+ <tr>
18
+ <th class="hits">Hits</th>
19
+ <td class="hits"><%= data[:total_hits][0][0] %></td>
20
+ </tr>
21
+ <tr>
22
+ <th class="unique-visitors">Unique Visitors</th>
23
+ <td class="unique-visitors"><%= data[:total_unique_visitors][0][0] %></td>
24
+ </tr>
25
+ <tr>
26
+ <th class="tx">Tx</th>
27
+ <td class="tx"><%= data[:total_size][0][0] %></td>
28
+ </tr>
29
+ <tr>
30
+ <th class="days">Days </th>
31
+ <td class="days"><%= data[:total_days][0][0] %></td>
32
+ </tr>
33
+ </table>
34
+ </article>
35
+
36
+ <% @reports = [
37
+ { title: "Daily Distribution", header: ["Day", "Hits", "Visits", "Size"], rows: data[:daily_distribution] },
38
+ { title: "Time Distribution", header: ["Hour", "Hits", "Visits", "Size"], rows: data[:time_distribution] },
39
+ { title: "Most Requested Pages", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_pages] },
40
+ { title: "Most Requested Resources", header: ["Path", "Hits", "Visits", "Size"], rows: data[:most_requested_resources] },
41
+ { title: "404 on HTML Files", header: ["Path", "Hits", "Visitors"], rows: data[:missed_pages] },
42
+ { title: "404 on other Resources", header: ["Path", "Hits", "Visitors"], rows: data[:missed_resources] },
43
+ { title: "Attacks", header: ["Path", "Hits", "Visitors"], rows: data[:attacks] },
44
+ { title: "Statuses", header: ["Status", "Count"], rows: data[:statuses] },
45
+ { title: "Daily Statuses", header: ["Status", "2xx", "3xx", "4xx"], rows: data[:statuses_by_day] },
46
+ { title: "Browsers", header: ["Browser", "Hits", "Visitors", "Size"], rows: data[:browsers] },
47
+ { title: "Platforms", header: ["Platform", "Hits", "Visitors", "Size"], rows: data[:platforms] },
48
+ { title: "IPs", header: ["IPs", "Hits", "Visitors", "Size"], rows: data[:ips] },
49
+ { title: "Referers", header: ["Referers", "Hits", "Visitors", "Size"], rows: data[:referers] },
50
+ ]
51
+ %>
52
+ <% @reports.each do |report| %>
53
+ <article>
54
+ <h2><%= report[:title] %></h2>
55
+ <%= render "output_table", report %>
56
+ </article>
57
+ <% end %>
58
+
59
+ <article>
60
+ <h2>Command Invocation and Performance</h2>
61
+
62
+ <h3>Command Invocation</h3>
63
+
64
+ <code>
65
+ </code>
66
+
67
+ <table class="pure-table command-invocation">
68
+ <tbody>
69
+ <tr>
70
+ <th>CLI Command</th>
71
+ <td><pre><%= data[:command] %></pre></td>
72
+ </tr>
73
+ <tr>
74
+ <th>Input file</th>
75
+ <td><code><%= (data[:log_file] || "stdin") %></code></td>
76
+ </tr>
77
+ <tr>
78
+ <th>Ignore crawlers</th>
79
+ <td><code><%= options[:ignore_crawlers] %></code></td></tr>
80
+ <tr>
81
+ <th>Only crawlers</th>
82
+ <td><code><%= options[:only_crawlers] %></code></td>
83
+ </tr>
84
+ <tr>
85
+ <th>No selfpoll</th>
86
+ <td><code><%= options[:no_selfpoll] %></code></td>
87
+ </tr>
88
+ <tr>
89
+ <th>Filter by date</th>
90
+ <td>
91
+ <code><%= (options[:from_date] != nil or options[:to_date] != nil) %></code>
92
+ </td>
93
+ </tr>
94
+ <tr>
95
+ <th>Prefix</th>
96
+ <td><code><%= @prefix %></code></td>
97
+ </tr>
98
+ <tr>
99
+ <th>Suffix</th>
100
+ <td><code><%= @suffix %></code></td>
101
+ </tr>
102
+ </tbody>
103
+ </table>
104
+
105
+ <h3> Log Structure</h3>
106
+
107
+ <table class="pure-table log-structure">
108
+ <tbody>
109
+ <tr>
110
+ <th>Log size</th>
111
+ <td><%= data[:log_size][0][0] %></td>
112
+ </tr>
113
+ <tr>
114
+ <th>Self poll entries</th>
115
+ <td><%= data[:selfpolls_size][0][0] %></td>
116
+ </tr>
117
+ <tr>
118
+ <th>Crawlers</th>
119
+ <td><%= data[:crawlers_size][0][0] %></td>
120
+ </tr>
121
+ <tr>
122
+ <th>Entries considered</th>
123
+ <td><%= data[:total_hits][0][0] %></td>
124
+ </tr>
125
+ </tbody>
126
+ </table>
127
+
128
+ <h3> Performance</h3>
129
+
130
+ <table class="pure-table performance">
131
+ <tbody>
132
+ <tr>
133
+ <th>Analysis started at</th>
134
+ <td><%= data[:started_at].to_s %></td>
135
+ </tr>
136
+ <tr>
137
+ <th>Analysis ended at</th>
138
+ <td><%= data[:ended_at].to_s %></td>
139
+ </tr>
140
+ <tr>
141
+ <th>Duration (sec)</th>
142
+ <td><%= "%.1f" % data[:duration] %></td>
143
+ </tr>
144
+ <tr>
145
+ <th>Duration (min)</th>
146
+ <td><%= "%d" % (data[:duration] / 60 ) %></td>
147
+ </tr>
148
+ <tr>
149
+ <th>Log size</th>
150
+ <td><%= data[:log_size][0][0] %></td>
151
+ </tr>
152
+ <tr>
153
+ <th>Lines/sec</th>
154
+ <td><%= "%.2f" % (data[:log_size][0][0] / data[:duration]) %></td></tr>
155
+ </tbody>
156
+ </table>
157
+ </article>
158
+ </section>
159
+ </body>
160
+ </html>
161
+
162
+
163
+
164
+
data/lib/apache_log_report/templates/template.org.erb ADDED
@@ -0,0 +1,262 @@
1
+ #+TITLE: Apache Log Analysis: <%= data[:log_file] %>
2
+ #+DATE: <<%= Date.today %>>
3
+ #+STARTUP: showall
4
+ #+OPTIONS: ^:{}
5
+ #+HTML_HEAD: <link rel="stylesheet" type="text/css" href="ala-style.css" />
6
+ #+OPTIONS: html-style:nil
7
+
8
+ * Summary
9
+
10
+ | Hits | <%= "%10d" % data[:total_hits][0][0] %> |
11
+ | Unique Visitors | <%= "%10d" % data[:total_unique_visitors][0][0] %> |
12
+ | Tx | <%= "%10s" % data[:total_size][0][0] %> |
13
+ | Days | <%= "%10d" % data[:total_days][0][0] %> |
14
+
15
+ * Daily Distribution
16
+
17
+ <%= self.output_table "daily_distribution", ["Day", "Hits", "Visits", "Size"], data[:daily_distribution] %>
18
+
19
+ #+BEGIN_SRC gnuplot :var data = daily_distribution :results output :exports <%= @export %> :file <%= @prefix %>daily<%= @suffix %>.svg
20
+ reset
21
+ set grid ytics linestyle 0
22
+ set grid xtics linestyle 0
23
+ set terminal svg size 1200,800 fname 'Arial'
24
+
25
+ set xdata time
26
+ set timefmt "%Y-%m-%d"
27
+ set format x "%a, %b %d"
28
+ set xtics rotate by 60 right
29
+
30
+ set title "Hits and Visitors"
31
+ set xlabel "Date"
32
+ set ylabel "Hits"
33
+ set y2label "Visits"
34
+ set y2tics
35
+
36
+ set style fill transparent solid 0.2 noborder
37
+
38
+ plot data using 1:2 with linespoints lw 3 lc rgb "#0000AA" pointtype 5 title "Hits" axes x1y2, \\
39
+ data using 1:2 with filledcurves below x1 linecolor rgb "#0000AA" notitle axes x1y2, \\
40
+ data using 1:3 with linespoints lw 3 lc rgb "#AA0000" pointtype 7 title "Visitors", \\
41
+ data using 1:3 with filledcurves below x1 notitle linecolor rgb "#AA0000", \\
42
+ data using 1:($3+0.1*$3):3 with labels notitle textcolor rgb "#AA0000", \\
43
+ data using 1:($2+0.1*$2):2 with labels notitle textcolor rgb "#0000AA" axes x1y2
44
+ #+END_SRC
45
+
46
+
47
+ * Time Distribution
48
+
49
+ <%= self.output_table "time_distribution", ["Hour", "Hits", "Visits", "Size"], data[:time_distribution] %>
50
+
51
+
52
+ #+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports <%= @export %> :file <%= @prefix %>time<%= @suffix %>.svg
53
+ reset
54
+ set terminal svg size 1200,800 fname 'Arial' fsize 10
55
+
56
+ set grid ytics linestyle 0
57
+
58
+ set title "Hits and Visitors"
59
+ set xlabel "Date"
60
+ set ylabel "Hits"
61
+ set y2label "Visitors"
62
+ set y2tics
63
+
64
+ set style fill solid 0.25
65
+ set boxwidth 0.6
66
+
67
+ set style data histograms
68
+ set style histogram clustered gap 1
69
+
70
+ plot data using 2:xtic(1) lc rgb "#0000AA" title "Hits", \\
71
+ data using 3 lc rgb "#AA0000" title "Visitors" axes x1y2, \\
72
+ data using ($0 - 0.2):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#0000AA"), \\
73
+ data using ($0 + 0.2):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#AA0000") axes x1y2
74
+ #+END_SRC
75
+
76
+ #+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports <%= @export %> :file <%= @prefix %>time-traffic<%= @suffix %>.svg
77
+ reset
78
+ set terminal svg size 1200,800 fname 'Arial' fsize 10
79
+
80
+ set grid ytics linestyle 0
81
+
82
+ set title "Traffic"
83
+ set xlabel "Date"
84
+ set ylabel "Traffic"
85
+
86
+ set style fill solid 0.50
87
+ set boxwidth 0.6
88
+
89
+ set style data histograms
90
+ set style histogram clustered gap 1
91
+
92
+ plot data using 2:xtic(1) lc rgb "#00AA00" title "Traffic", \\
93
+ data using ($0):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00")
94
+ #+END_SRC
95
+
96
+ * Most Requested Pages
97
+
98
+ <%= self.output_table "most_requested_pages", ["Path", "Hits", "Visits", "Size"], data[:most_requested_pages] %>
99
+
100
+ * Most Requested URIs
101
+
102
+ <%= self.output_table "most_requested_resources", ["Path", "Hits", "Visits", "Size"], data[:most_requested_resources] %>
103
+
104
+ * 404s on HTML files
105
+
106
+ <%= self.output_table "pages_404", ["Path", "Hits", "Visitors"], data[:missed_pages] %>
107
+
108
+ * 404s on other resources
109
+
110
+ <%= self.output_table "resources_404", ["Path", "Hits", "Visitors"], data[:missed_resources] %>
111
+
112
+ * Possible Attacks
113
+
114
+ <%= self.output_table "attacks", ["Path", "Hits", "Visitors"], data[:attacks] %>
115
+
116
+ * Statuses
117
+
118
+ <%= self.output_table "statuses", ["Status", "Count"], data[:statuses] %>
119
+
120
+ #+BEGIN_SRC gnuplot :var data = statuses :results output :exports <%= @export %> :file <%= @prefix %>statuses<%= @suffix %>.svg
121
+ reset
122
+ set grid ytics linestyle 0
123
+ set terminal svg size 1200,800 fname 'Arial' fsize 10
124
+
125
+ set style fill solid 0.25
126
+ set boxwidth 0.6
127
+
128
+ plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
129
+ data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
130
+ #+END_SRC
131
+
132
+ * Daily Statuses
133
+
134
+ <%= self.output_table "daily_statuses", ["Status", "2xx", "3xx", "4xx"], data[:statuses_by_day] %>
135
+
136
+ #+BEGIN_SRC gnuplot :var data = daily_statuses :results output :exports <%= @export %> :file <%= @prefix %>daily-statuses<%= @suffix %>.svg
137
+ reset
138
+ set terminal svg size 1200,800 fname 'Arial' fsize 10
139
+
140
+ set grid ytics linestyle 0
141
+
142
+ set title "Daily Statuses"
143
+ set xlabel "Date"
144
+ set ylabel "Number of Hits"
145
+ set xtics rotate by 60 right
146
+
147
+ set style fill solid 0.25
148
+ set boxwidth 0.6
149
+
150
+ set style data histograms
151
+ set style histogram clustered gap 1
152
+
153
+ plot data using 2:xtic(1) lc rgb "#00AA00" title "2xx", \\
154
+ data using 3 lc rgb "#0000CC" title "3xx", \\
155
+ data using 4 lc rgb "#AA0000" title "4xx", \\
156
+ data using ($0 - 1. / 4):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00"), \\
157
+ data using ($0):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0000CC"), \\
158
+ data using ($0 + 1. / 4):($4 + 0.1*$4):4 with labels title "" textcolor rgb("#AA0000")
159
+ #+END_SRC
160
+
161
+ * Browsers
162
+
163
+ <%= self.output_table "browsers", ["Browser", "Hits", "Visitors", "Size"], data[:browsers] %>
164
+
165
+ #+BEGIN_SRC gnuplot :var data = browsers :results output :exports <%= @export %> :file <%= @prefix %>browser<%= @suffix %>.svg
166
+ reset
167
+ set grid ytics linestyle 0
168
+ set terminal svg size 1200,800 fname 'Arial' fsize 10
169
+
170
+ set style fill solid 0.25
171
+ set boxwidth 0.6
172
+
173
+ plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
174
+ data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
175
+ #+END_SRC
176
+
177
+ * Platforms
178
+
179
+ <%= self.output_table "platforms", ["Platform", "Hits", "Visitors", "Size"], data[:platforms] %>
180
+
181
+ #+BEGIN_SRC gnuplot :var data = platforms :results output :exports <%= @export %> :file <%= @prefix %>platforms<%= @suffix %>.svg
182
+ reset
183
+ set grid ytics linestyle 0
184
+ set terminal svg size 1200,800 fname 'Arial' fsize 10
185
+
186
+ set style fill solid 0.25
187
+ set boxwidth 0.6
188
+
189
+ plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
190
+ data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
191
+ #+END_SRC
192
+
193
+ * IPs
194
+
195
+ <%= self.output_table "ips", ["IPs", "Hits", "Visitors", "Size"], data[:ips] %>
196
+
197
+
198
+ * Referers
199
+
200
+ <%= self.output_table "referers", ["Referers", "Hits", "Visitors", "Size"], data[:referers] %>
201
+
202
+ #+BEGIN_SRC gnuplot :var data = referers :results output :exports <%= @export %> :file <%= @prefix %>referers<%= @suffix %>.svg
203
+ reset
204
+ set terminal svg size 1200,800 fname 'Arial' fsize 10
205
+
206
+ set grid ytics linestyle 0
207
+ set grid xtics linestyle 0
208
+
209
+ set title "Referers"
210
+ set xlabel "Date"
211
+ set xtics rotate by 60 right
212
+ set ylabel "Hits and Visits"
213
+
214
+ set style fill solid 0.45
215
+ set boxwidth 0.7
216
+
217
+ set style data histograms
218
+ set style histogram clustered gap 1
219
+
220
+ plot data using 2:xtic(1) lc rgb "#AA00AA" title "Hits", \\
221
+ data using 3 lc rgb "#0AAAA0" title "Visits", \\
222
+ data using ($0 - 1. / 3):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#AA00AA"), \\
223
+ data using ($0 + 1. / 3):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0AAAA0")
224
+ #+END_SRC
225
+
226
+ * Command Invocation and Performance
227
+
228
+ ** Command Invocation
229
+
230
+ #+BEGIN_EXAMPLE shell
231
+ <%= data[:command] %>
232
+ #+END_EXAMPLE
233
+
234
+ | Input file | <%= "%-50s" % (data[:log_file] || "stdin") %> |
235
+ | Ignore crawlers | <%= "%-50s" % options[:ignore_crawlers] %> |
236
+ | Only crawlers | <%= "%-50s" % options[:only_crawlers] %> |
237
+ | No selfpoll | <%= "%-50s" % options[:no_selfpoll] %> |
238
+ | Filter by date | <%= "%-50s" % (options[:from_date] != nil or options[:to_date] != nil) %> |
239
+ | Prefix | <%= "%-50s" % @prefix %> |
240
+ | Suffix | <%= "%-50s" % @suffix %> |
241
+
242
+ ** Log Structure
243
+
244
+ | Log size | <%= "%10d" % data[:log_size][0][0] %> |
245
+ | Self poll entries | <%= "%10d" % data[:selfpolls_size][0][0] %> |
246
+ | Crawlers | <%= "%10d" % data[:crawlers_size][0][0] %> |
247
+ | Entries considered | <%= "%10d" % data[:total_hits][0][0] %> |
248
+
249
+ ** Performance
250
+
251
+ | Analysis started at | <%= data[:started_at].to_s %> |
252
+ | Analysis ended at | <%= data[:ended_at].to_s %> |
253
+ | Duration (sec) | <%= "%5.3d" % data[:duration] %> |
254
+ | Duration (min) | <%= "%5.3d" % (data[:duration] / 60 ) %> |
255
+ | Log size | <%= "%9d" % data[:log_size][0][0] %> |
256
+ | Lines/sec | <%= "%6.2f" % (data[:log_size][0][0] / data[:duration]) %> |
257
+
258
+ * Local Variables :noexport:
259
+ # Local Variables:
260
+ # org-confirm-babel-evaluate: nil
261
+ # org-display-inline-images: t
262
+ # end:
data/lib/apache_log_report/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module ApacheLogReport
2
- VERSION = "1.0.0"
2
+ VERSION = "1.1.0"
3
3
  end