apache_log_report 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +6 -6
- data/exe/apache_log_report +16 -4
- data/lib/apache_log_report/apache_log_report.rb +245 -0
- data/lib/apache_log_report/data_cruncher.rb +84 -0
- data/lib/apache_log_report/emitter.rb +50 -0
- data/lib/apache_log_report/log_parser.rb +87 -0
- data/lib/apache_log_report/options_parser.rb +86 -0
- data/lib/apache_log_report/templates/_output_table.html.erb +25 -0
- data/lib/apache_log_report/templates/template.html.erb +164 -0
- data/lib/apache_log_report/templates/template.org.erb +262 -0
- data/lib/apache_log_report/version.rb +1 -1
- data/lib/apache_log_report.rb +5 -532
- metadata +11 -3
data/lib/apache_log_report.rb
CHANGED
@@ -1,532 +1,5 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
require 'optparse'
|
7
|
-
require 'optparse/date'
|
8
|
-
require 'apache_log_report/version'
|
9
|
-
|
10
|
-
def self.options_parse options
|
11
|
-
limit = 30
|
12
|
-
args = {}
|
13
|
-
|
14
|
-
opt_parser = OptionParser.new do |opts|
|
15
|
-
opts.banner = "Usage: apache_log_report [options] [logfile]"
|
16
|
-
|
17
|
-
opts.on("-lN", "--limit=N", Integer, "Number of entries to show (defaults to #{limit})") do |n|
|
18
|
-
args[:limit] = n
|
19
|
-
end
|
20
|
-
|
21
|
-
opts.on("-bDATE", "--begin=DATE", DateTime, "Consider entries after or on DATE") do |n|
|
22
|
-
args[:from_date] = n
|
23
|
-
end
|
24
|
-
|
25
|
-
opts.on("-eDATE", "--end=DATE", DateTime, "Consider entries before or on DATE") do |n|
|
26
|
-
args[:to_date] = n
|
27
|
-
end
|
28
|
-
|
29
|
-
opts.on("-i", "--ignore-crawlers", "Ignore crawlers") do
|
30
|
-
args[:ignore_crawlers] = true
|
31
|
-
end
|
32
|
-
|
33
|
-
opts.on("-p", "--ignore-selfpoll", "Ignore apaches self poll entries (from ::1)") do
|
34
|
-
args[:no_selfpoll] = true
|
35
|
-
end
|
36
|
-
|
37
|
-
opts.on("-c", "--only-crawlers", "Perform analysis on crawlers only") do
|
38
|
-
args[:only_crawlers] = true
|
39
|
-
end
|
40
|
-
|
41
|
-
opts.on("-uPREFIX", "--prefix=PREFIX", String, "Prefix to add to all plots (used to run multiple analyses in the same dir)") do |n|
|
42
|
-
args[:prefix] = n
|
43
|
-
end
|
44
|
-
|
45
|
-
opts.on("-wSUFFIX", "--suffix=SUFFIX", String, "Suffix to add to all plots (used to run multiple analyses in the same dir)") do |n|
|
46
|
-
args[:suffix] = n
|
47
|
-
end
|
48
|
-
|
49
|
-
opts.on("-cWHAT", "--code-export=WHAT", String, "Control :export directive in code blocks (code, results, *both*, none)") do |n|
|
50
|
-
args[:code_export] = n
|
51
|
-
end
|
52
|
-
|
53
|
-
opts.on("-v", "--version", "Prints version information") do
|
54
|
-
puts "apache_log_report version #{ApacheLogReport::VERSION}"
|
55
|
-
puts "Copyright (C) 2020 Adolfo Villafiorita"
|
56
|
-
puts "Distributed under the terms of the MIT license"
|
57
|
-
puts ""
|
58
|
-
puts "Written by Adolfo Villafiorita"
|
59
|
-
exit
|
60
|
-
end
|
61
|
-
|
62
|
-
opts.on("-h", "--help", "Prints this help") do
|
63
|
-
puts opts
|
64
|
-
puts "This is version #{ApacheLogReport::VERSION}"
|
65
|
-
exit
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
opt_parser.parse!(options)
|
70
|
-
|
71
|
-
args[:limit] ||= limit
|
72
|
-
args[:ignore_crawlers] ||= false
|
73
|
-
args[:no_selfpoll] ||= false
|
74
|
-
args[:only_crawlers] ||= false
|
75
|
-
args[:prefix] ||= ""
|
76
|
-
args[:suffix] ||= ""
|
77
|
-
args[:code_export] ||= "both"
|
78
|
-
|
79
|
-
return args
|
80
|
-
end
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
#
|
85
|
-
# parse an Apache log file and return a SQLite3 DB
|
86
|
-
#
|
87
|
-
require 'apache_log/parser'
|
88
|
-
require 'sqlite3'
|
89
|
-
require 'browser'
|
90
|
-
|
91
|
-
def self.parse filename, options = {}
|
92
|
-
content = filename ? File.readlines(filename) : ARGF.readlines
|
93
|
-
|
94
|
-
db = SQLite3::Database.new ":memory:"
|
95
|
-
db.execute "CREATE TABLE IF NOT EXISTS LogLine(
|
96
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
97
|
-
datetime TEXT,
|
98
|
-
ip TEXT,
|
99
|
-
user TEXT,
|
100
|
-
unique_visitor TEXT,
|
101
|
-
method TEXT,
|
102
|
-
path TEXT,
|
103
|
-
extension TEXT,
|
104
|
-
status TEXT,
|
105
|
-
size INTEGER,
|
106
|
-
referer TEXT,
|
107
|
-
user_agent TEXT,
|
108
|
-
bot INTEGER,
|
109
|
-
browser TEXT,
|
110
|
-
browser_version TEXT,
|
111
|
-
platform TEXT,
|
112
|
-
platform_version TEXT)"
|
113
|
-
|
114
|
-
ins = db.prepare('insert into LogLine (
|
115
|
-
datetime,
|
116
|
-
ip,
|
117
|
-
user,
|
118
|
-
unique_visitor,
|
119
|
-
method,
|
120
|
-
path,
|
121
|
-
extension,
|
122
|
-
status,
|
123
|
-
size,
|
124
|
-
referer,
|
125
|
-
user_agent,
|
126
|
-
bot,
|
127
|
-
browser,
|
128
|
-
browser_version,
|
129
|
-
platform,
|
130
|
-
platform_version)
|
131
|
-
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)')
|
132
|
-
|
133
|
-
parser = ApacheLog::Parser.new(options[:format] || 'combined')
|
134
|
-
|
135
|
-
content.each do |line|
|
136
|
-
begin
|
137
|
-
hash = parser.parse line
|
138
|
-
|
139
|
-
ua = Browser.new(hash[:user_agent], accept_language: "en-us")
|
140
|
-
ins.execute(
|
141
|
-
hash[:datetime].iso8601,
|
142
|
-
hash[:remote_host],
|
143
|
-
hash[:user],
|
144
|
-
hash[:datetime].strftime("%Y-%m-%d") + " " + hash[:remote_host] + " " + hash[:user_agent],
|
145
|
-
hash[:request][:method],
|
146
|
-
hash[:request][:path],
|
147
|
-
(hash[:request][:path] ? File.extname(hash[:request][:path]) : ""),
|
148
|
-
hash[:status],
|
149
|
-
hash[:size].to_i,
|
150
|
-
hash[:referer],
|
151
|
-
hash[:user_agent],
|
152
|
-
ua.bot? ? 1 : 0,
|
153
|
-
(ua.name || ""),
|
154
|
-
(ua.version || ""),
|
155
|
-
(ua.platform.name || ""),
|
156
|
-
(ua.platform.version || "")
|
157
|
-
)
|
158
|
-
rescue
|
159
|
-
STDERR.puts "Apache Log parser error: could not parse #{line}"
|
160
|
-
end
|
161
|
-
end
|
162
|
-
|
163
|
-
db
|
164
|
-
end
|
165
|
-
|
166
|
-
#
|
167
|
-
# take a sqlite3 databae and analyze data
|
168
|
-
#
|
169
|
-
def self.analyze_data db, options = {}
|
170
|
-
|
171
|
-
@first_day = db.execute "SELECT datetime from LogLine order by datetime limit 1"
|
172
|
-
@last_day = db.execute "SELECT datetime from LogLine order by datetime desc limit 1"
|
173
|
-
@log_size = db.execute "SELECT count(datetime) from LogLine"
|
174
|
-
@crawlers_size = db.execute "SELECT count(datetime) from LogLine where bot == 1"
|
175
|
-
@selfpolls_size = db.execute "SELECT count(datetime) from LogLine where ip == '::1'"
|
176
|
-
|
177
|
-
#
|
178
|
-
# generate the where clause corresponding to the command line options to filter data
|
179
|
-
#
|
180
|
-
@filter = [
|
181
|
-
(options[:from_date] ? "date(datetime) >= '#{options[:from_date]}'" : nil),
|
182
|
-
(options[:to_date] ? "date(datetime) <= '#{options[:to_date]}'" : nil),
|
183
|
-
(options[:only_crawlers] ? "bot == 1" : nil),
|
184
|
-
(options[:ignore_crawlers] ? "bot == 0" : nil),
|
185
|
-
(options[:no_selfpolls] ? "ip != '::1'" : nil),
|
186
|
-
"true"
|
187
|
-
].compact.join " and "
|
188
|
-
|
189
|
-
# in alternative to sum(size)
|
190
|
-
human_readable_size = <<-EOS
|
191
|
-
CASE
|
192
|
-
WHEN sum(size) < 1024 THEN sum(size) || ' B'
|
193
|
-
WHEN sum(size) >= 1024 AND sum(size) < (1024 * 1024) THEN ROUND((CAST(sum(size) AS REAL) / 1024),2) || ' KB'
|
194
|
-
WHEN sum(size) >= (1024 * 1024) AND sum(size) < (1024 * 1024 * 1024) THEN ROUND((CAST(sum(size) AS REAL) / (1024 * 1024)),2) || ' MB'
|
195
|
-
WHEN sum(size) >= (1024 * 1024 * 1024) AND sum(size) < (1024 * 1024 * 1024 *1024) THEN ROUND((CAST(sum(size) AS REAL) / (1024 * 1024 * 1024)),2) || ' GB'
|
196
|
-
WHEN sum(size) >= (1024 * 1024 * 1024 * 1024) THEN ROUND((CAST(sum(size) AS REAL) / (1024 * 1024 * 1024 * 1024)),2) || ' TB'
|
197
|
-
END AS size
|
198
|
-
EOS
|
199
|
-
|
200
|
-
@total_hits = db.execute "SELECT count(datetime) from LogLine where #{@filter}"
|
201
|
-
@total_unique_visitors = db.execute "SELECT count(distinct(unique_visitor)) from LogLine where #{@filter}"
|
202
|
-
@total_size = db.execute "SELECT #{human_readable_size} from LogLine where #{@filter}"
|
203
|
-
@total_days = (Date.parse(@last_day[0][0]) - Date.parse(@first_day[0][0])).to_i
|
204
|
-
|
205
|
-
@daily_distribution = db.execute "SELECT date(datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by date(datetime)"
|
206
|
-
|
207
|
-
@time_distribution = db.execute "SELECT strftime('%H', datetime), count(datetime), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by strftime('%H', datetime)"
|
208
|
-
|
209
|
-
@most_requested_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where extension == '.html' and #{@filter} group by path order by count(path) desc limit #{options[:limit]}"
|
210
|
-
|
211
|
-
@most_requested_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by path order by count(path) desc limit #{options[:limit]}"
|
212
|
-
|
213
|
-
@missed_pages = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and extension == '.html' and #{@filter} group by path order by count(path) desc limit #{options[:limit]}"
|
214
|
-
|
215
|
-
@missed_resources = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{@filter} group by path order by count(path) desc limit #{options[:limit]}"
|
216
|
-
|
217
|
-
@reasonable_requests_exts = [ ".html", ".css", ".js", ".jpg", ".svg", ".png", ".woff", ".xml", ".ttf", ".ico", ".pdf", ".htm", ".txt", ".org" ].map { |x|
|
218
|
-
"extension != '#{x}'"
|
219
|
-
}.join " and "
|
220
|
-
|
221
|
-
@attacks = db.execute "SELECT path, count(path), count(distinct(unique_visitor)) from LogLine where status == '404' and #{@filter} and (#{@reasonable_requests_exts}) group by path order by count(path) desc limit #{options[:limit]}"
|
222
|
-
|
223
|
-
@statuses = db.execute "SELECT status, count(status) from LogLine where #{@filter} group by status order by status"
|
224
|
-
|
225
|
-
@by_day_4xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '4' and #{@filter} group by date(datetime)"
|
226
|
-
@by_day_3xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '3' and #{@filter} group by date(datetime)"
|
227
|
-
@by_day_2xx = db.execute "SELECT date(datetime), count(datetime) from LogLine where substr(status, 1,1) == '2' and #{@filter} group by date(datetime)"
|
228
|
-
|
229
|
-
@statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
|
230
|
-
[x[0], x[1].map { |y| y[1] }].flatten
|
231
|
-
}
|
232
|
-
|
233
|
-
@browsers = db.execute "SELECT browser, count(browser), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by browser order by count(browser) desc"
|
234
|
-
|
235
|
-
@platforms = db.execute "SELECT platform, count(platform), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by platform order by count(platform) desc"
|
236
|
-
|
237
|
-
@ips = db.execute "SELECT ip, count(ip), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by ip order by count(ip) desc limit #{options[:limit]}"
|
238
|
-
|
239
|
-
@referers = db.execute "SELECT referer, count(referer), count(distinct(unique_visitor)), #{human_readable_size} from LogLine where #{@filter} group by referer order by count(referer) desc limit #{options[:limit]}"
|
240
|
-
end
|
241
|
-
|
242
|
-
|
243
|
-
#
|
244
|
-
# Emit Data
|
245
|
-
#
|
246
|
-
|
247
|
-
require 'terminal-table'
|
248
|
-
|
249
|
-
def self.output_table name, headings, rows
|
250
|
-
name = "#+NAME: #{name}"
|
251
|
-
table = Terminal::Table.new headings: headings, rows: rows, style: { border_x: "-", border_i: "|" }
|
252
|
-
|
253
|
-
#(2..headings.size).each do |i|
|
254
|
-
# table.align_column(i, :right)
|
255
|
-
#end
|
256
|
-
|
257
|
-
name + "\n" + table.to_s
|
258
|
-
end
|
259
|
-
|
260
|
-
def self.emit options = {}, command, log_file, started_at, ended_at, duration
|
261
|
-
@prefix = options[:prefix]
|
262
|
-
@suffix = options[:suffix]
|
263
|
-
@export = options[:code_export]
|
264
|
-
|
265
|
-
<<EOS
|
266
|
-
#+TITLE: Apache Log Analysis: #{log_file}
|
267
|
-
#+DATE: <#{Date.today}>
|
268
|
-
#+STARTUP: showall
|
269
|
-
#+OPTIONS: ^:{}
|
270
|
-
#+HTML_HEAD: <link rel="stylesheet" type="text/css" href="ala-style.css" />
|
271
|
-
#+OPTIONS: html-style:nil
|
272
|
-
|
273
|
-
* Summary
|
274
|
-
|
275
|
-
| Hits | #{"%10d" % @total_hits[0][0]} |
|
276
|
-
| Unique Visitors | #{"%10d" % @total_unique_visitors[0][0] } |
|
277
|
-
| Tx | #{"%10s" % @total_size[0][0]} |
|
278
|
-
| Days | #{"%10d" % @total_days[0][0] } |
|
279
|
-
|
280
|
-
* Daily Distribution
|
281
|
-
|
282
|
-
#{ output_table "daily_distribution", ["Day", "Hits", "Visits", "Size"], @daily_distribution }
|
283
|
-
|
284
|
-
#+BEGIN_SRC gnuplot :var data = daily_distribution :results output :exports #{@export} :file #{@prefix}daily#{@suffix}.svg
|
285
|
-
reset
|
286
|
-
set grid ytics linestyle 0
|
287
|
-
set grid xtics linestyle 0
|
288
|
-
set terminal svg size 1200,800 fname 'Arial'
|
289
|
-
|
290
|
-
set xdata time
|
291
|
-
set timefmt "%Y-%m-%d"
|
292
|
-
set format x "%a, %b %d"
|
293
|
-
set xtics rotate by 60 right
|
294
|
-
|
295
|
-
set title "Hits and Visitors"
|
296
|
-
set xlabel "Date"
|
297
|
-
set ylabel "Hits"
|
298
|
-
set y2label "Visits"
|
299
|
-
set y2tics
|
300
|
-
|
301
|
-
set style fill transparent solid 0.2 noborder
|
302
|
-
|
303
|
-
plot data using 1:2 with linespoints lw 3 lc rgb "#0000AA" pointtype 5 title "Hits" axes x1y2, \\
|
304
|
-
data using 1:2 with filledcurves below x1 linecolor rgb "#0000AA" notitle axes x1y2, \\
|
305
|
-
data using 1:3 with linespoints lw 3 lc rgb "#AA0000" pointtype 7 title "Visitors", \\
|
306
|
-
data using 1:3 with filledcurves below x1 notitle linecolor rgb "#AA0000", \\
|
307
|
-
data using 1:($3+0.1*$3):3 with labels notitle textcolor rgb "#AA0000", \\
|
308
|
-
data using 1:($2+0.1*$2):2 with labels notitle textcolor rgb "#0000AA" axes x1y2
|
309
|
-
#+END_SRC
|
310
|
-
|
311
|
-
|
312
|
-
* Time Distribution
|
313
|
-
|
314
|
-
#{ output_table "time_distribution", ["Hour", "Hits", "Visits", "Size"], @time_distribution }
|
315
|
-
|
316
|
-
|
317
|
-
#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports #{@export} :file #{@prefix}time#{@suffix}.svg
|
318
|
-
reset
|
319
|
-
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
320
|
-
|
321
|
-
set grid ytics linestyle 0
|
322
|
-
|
323
|
-
set title "Hits and Visitors"
|
324
|
-
set xlabel "Date"
|
325
|
-
set ylabel "Hits"
|
326
|
-
set y2label "Visitors"
|
327
|
-
set y2tics
|
328
|
-
|
329
|
-
set style fill solid 0.25
|
330
|
-
set boxwidth 0.6
|
331
|
-
|
332
|
-
set style data histograms
|
333
|
-
set style histogram clustered gap 1
|
334
|
-
|
335
|
-
plot data using 2:xtic(1) lc rgb "#0000AA" title "Hits", \\
|
336
|
-
data using 3 lc rgb "#AA0000" title "Visitors" axes x1y2, \\
|
337
|
-
data using ($0 - 0.2):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#0000AA"), \\
|
338
|
-
data using ($0 + 0.2):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#AA0000") axes x1y2
|
339
|
-
#+END_SRC
|
340
|
-
|
341
|
-
#+BEGIN_SRC gnuplot :var data = time_distribution :results output :exports #{@export} :file #{@prefix}time-traffic#{@suffix}.svg
|
342
|
-
reset
|
343
|
-
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
344
|
-
|
345
|
-
set grid ytics linestyle 0
|
346
|
-
|
347
|
-
set title "Traffic"
|
348
|
-
set xlabel "Date"
|
349
|
-
set ylabel "Traffic"
|
350
|
-
|
351
|
-
set style fill solid 0.50
|
352
|
-
set boxwidth 0.6
|
353
|
-
|
354
|
-
set style data histograms
|
355
|
-
set style histogram clustered gap 1
|
356
|
-
|
357
|
-
plot data using 2:xtic(1) lc rgb "#00AA00" title "Traffic", \\
|
358
|
-
data using ($0):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00")
|
359
|
-
#+END_SRC
|
360
|
-
|
361
|
-
* Most Requested Pages
|
362
|
-
|
363
|
-
#{ output_table "most_requested_pages", ["Path", "Hits", "Visits", "Size"], @most_requested_pages }
|
364
|
-
|
365
|
-
* Most Requested URIs
|
366
|
-
|
367
|
-
#{ output_table "most_requested_resources", ["Path", "Hits", "Visits", "Size"], @most_requested_resources }
|
368
|
-
|
369
|
-
* 404s on HTML files
|
370
|
-
|
371
|
-
#{ output_table "pages_404", ["Path", "Hits", "Visitors"], @missed_pages }
|
372
|
-
|
373
|
-
* 404s on other resources
|
374
|
-
|
375
|
-
#{ output_table "resources_404", ["Path", "Hits", "Visitors"], @missed_resources }
|
376
|
-
|
377
|
-
* Possible Attacks
|
378
|
-
|
379
|
-
#{ output_table "attacks", ["Path", "Hits", "Visitors"], @attacks }
|
380
|
-
|
381
|
-
* Statuses
|
382
|
-
|
383
|
-
#{ output_table "statuses", ["Status", "Count"], @statuses }
|
384
|
-
|
385
|
-
#+BEGIN_SRC gnuplot :var data = statuses :results output :exports #{@export} :file #{@prefix}statuses#{@suffix}.svg
|
386
|
-
reset
|
387
|
-
set grid ytics linestyle 0
|
388
|
-
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
389
|
-
|
390
|
-
set style fill solid 0.25
|
391
|
-
set boxwidth 0.6
|
392
|
-
|
393
|
-
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
394
|
-
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
395
|
-
#+END_SRC
|
396
|
-
|
397
|
-
* Daily Statuses
|
398
|
-
|
399
|
-
#{ output_table "daily_statuses", ["Status", "2xx", "3xx", "4xx"], @statuses_by_day }
|
400
|
-
|
401
|
-
#+BEGIN_SRC gnuplot :var data = daily_statuses :results output :exports #{@export} :file #{@prefix}daily-statuses#{@suffix}.svg
|
402
|
-
reset
|
403
|
-
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
404
|
-
|
405
|
-
set grid ytics linestyle 0
|
406
|
-
|
407
|
-
set title "Daily Statuses"
|
408
|
-
set xlabel "Date"
|
409
|
-
set ylabel "Number of Hits"
|
410
|
-
set xtics rotate by 60 right
|
411
|
-
|
412
|
-
set style fill solid 0.25
|
413
|
-
set boxwidth 0.6
|
414
|
-
|
415
|
-
set style data histograms
|
416
|
-
set style histogram clustered gap 1
|
417
|
-
|
418
|
-
plot data using 2:xtic(1) lc rgb "#00AA00" title "2xx", \\
|
419
|
-
data using 3 lc rgb "#0000CC" title "3xx", \\
|
420
|
-
data using 4 lc rgb "#AA0000" title "4xx", \\
|
421
|
-
data using ($0 - 1. / 4):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#00AA00"), \\
|
422
|
-
data using ($0):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0000CC"), \\
|
423
|
-
data using ($0 + 1. / 4):($4 + 0.1*$4):4 with labels title "" textcolor rgb("#AA0000")
|
424
|
-
#+END_SRC
|
425
|
-
|
426
|
-
* Browsers
|
427
|
-
|
428
|
-
#{ output_table "browsers", ["Browser", "Hits", "Visitors", "Size"], @browsers }
|
429
|
-
|
430
|
-
#+BEGIN_SRC gnuplot :var data = browsers :results output :exports #{@export} :file #{@prefix}browser#{@suffix}.svg
|
431
|
-
reset
|
432
|
-
set grid ytics linestyle 0
|
433
|
-
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
434
|
-
|
435
|
-
set style fill solid 0.25
|
436
|
-
set boxwidth 0.6
|
437
|
-
|
438
|
-
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
439
|
-
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
440
|
-
#+END_SRC
|
441
|
-
|
442
|
-
* Platforms
|
443
|
-
|
444
|
-
#{ output_table "platforms", ["Platform", "Hits", "Visitors", "Size"], @platforms }
|
445
|
-
|
446
|
-
#+BEGIN_SRC gnuplot :var data = platforms :results output :exports #{@export} :file #{@prefix}platforms#{@suffix}.svg
|
447
|
-
reset
|
448
|
-
set grid ytics linestyle 0
|
449
|
-
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
450
|
-
|
451
|
-
set style fill solid 0.25
|
452
|
-
set boxwidth 0.6
|
453
|
-
|
454
|
-
plot data using 2:xtic(1) with boxes lc rgb "#0000AA" title "Hits", \\
|
455
|
-
data using ($0):($2+0.1*$2):2 with labels textcolor rgb "#0000AA"
|
456
|
-
#+END_SRC
|
457
|
-
|
458
|
-
* IPs
|
459
|
-
|
460
|
-
#{ output_table "ips", ["IPs", "Hits", "Visitors", "Size"], @ips }
|
461
|
-
|
462
|
-
|
463
|
-
* Referers
|
464
|
-
|
465
|
-
#{ output_table "referers", ["Referers", "Hits", "Visitors", "Size"], @referers }
|
466
|
-
|
467
|
-
#+BEGIN_SRC gnuplot :var data = referers :results output :exports #{@export} :file #{@prefix}referers#{@suffix}.svg
|
468
|
-
reset
|
469
|
-
set terminal svg size 1200,800 fname 'Arial' fsize 10
|
470
|
-
|
471
|
-
set grid ytics linestyle 0
|
472
|
-
set grid xtics linestyle 0
|
473
|
-
|
474
|
-
set title "Referers"
|
475
|
-
set xlabel "Date"
|
476
|
-
set xtics rotate by 60 right
|
477
|
-
set ylabel "Hits and Visits"
|
478
|
-
|
479
|
-
set style fill solid 0.45
|
480
|
-
set boxwidth 0.7
|
481
|
-
|
482
|
-
set style data histograms
|
483
|
-
set style histogram clustered gap 1
|
484
|
-
|
485
|
-
plot data using 2:xtic(1) lc rgb "#AA00AA" title "Hits", \\
|
486
|
-
data using 3 lc rgb "#0AAAA0" title "Visits", \\
|
487
|
-
data using ($0 - 1. / 3):($2 + 0.1*$2):2 with labels title "" textcolor rgb("#AA00AA"), \\
|
488
|
-
data using ($0 + 1. / 3):($3 + 0.1*$3):3 with labels title "" textcolor rgb("#0AAAA0")
|
489
|
-
#+END_SRC
|
490
|
-
|
491
|
-
* Command Invocation and Performance
|
492
|
-
|
493
|
-
** Command Invocation
|
494
|
-
|
495
|
-
#+BEGIN_EXAMPLE shell
|
496
|
-
#{command}
|
497
|
-
#+END_EXAMPLE
|
498
|
-
|
499
|
-
| Input file | #{"%-50s" % (log_file || "stdin")} |
|
500
|
-
| Ignore crawlers | #{"%-50s" % options[:ignore_crawlers]} |
|
501
|
-
| Only crawlers | #{"%-50s" % options[:only_crawlers]} |
|
502
|
-
| No selfpoll | #{"%-50s" % options[:no_selfpoll]} |
|
503
|
-
| Filter by date | #{"%-50s" % (options[:from_date] != nil or options[:to_date] != nil)} |
|
504
|
-
| Prefix | #{"%-50s" % @prefix} |
|
505
|
-
| Suffix | #{"%-50s" % @suffix} |
|
506
|
-
|
507
|
-
** Log Structure
|
508
|
-
|
509
|
-
| Log size | #{"%10d" % @log_size[0][0]} |
|
510
|
-
| Self poll entries | #{"%10d" % @selfpolls_size[0][0]} |
|
511
|
-
| Crawlers | #{"%10d" % @crawlers_size[0][0]} |
|
512
|
-
| Entries considered | #{"%10d" % @total_hits[0][0]} |
|
513
|
-
|
514
|
-
** Performance
|
515
|
-
|
516
|
-
| Analysis started at | #{started_at.to_s} |
|
517
|
-
| Analysis ended at | #{ended_at.to_s} |
|
518
|
-
| Duration (sec) | #{"%5.3d" % duration } |
|
519
|
-
| Duration (min) | #{"%5.3d" % (duration / 60 )} |
|
520
|
-
| Log size | #{"%9d" % @log_size[0][0]} |
|
521
|
-
| Lines/sec | #{"%6.2f" % (@log_size[0][0] / duration)} |
|
522
|
-
|
523
|
-
* Local Variables :noexport:
|
524
|
-
# Local Variables:
|
525
|
-
# org-confirm-babel-evaluate: nil
|
526
|
-
# org-display-inline-images: t
|
527
|
-
# end:
|
528
|
-
EOS
|
529
|
-
end
|
530
|
-
end
|
531
|
-
|
532
|
-
|
1
|
+
require 'apache_log_report/version'
|
2
|
+
require 'apache_log_report/options_parser'
|
3
|
+
require 'apache_log_report/log_parser'
|
4
|
+
require 'apache_log_report/data_cruncher'
|
5
|
+
require 'apache_log_report/emitter'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: apache_log_report
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adolfo Villafiorita
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-10-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: apache_log-parser
|
@@ -86,6 +86,14 @@ files:
|
|
86
86
|
- bin/setup
|
87
87
|
- exe/apache_log_report
|
88
88
|
- lib/apache_log_report.rb
|
89
|
+
- lib/apache_log_report/apache_log_report.rb
|
90
|
+
- lib/apache_log_report/data_cruncher.rb
|
91
|
+
- lib/apache_log_report/emitter.rb
|
92
|
+
- lib/apache_log_report/log_parser.rb
|
93
|
+
- lib/apache_log_report/options_parser.rb
|
94
|
+
- lib/apache_log_report/templates/_output_table.html.erb
|
95
|
+
- lib/apache_log_report/templates/template.html.erb
|
96
|
+
- lib/apache_log_report/templates/template.org.erb
|
89
97
|
- lib/apache_log_report/version.rb
|
90
98
|
homepage: https://www.ict4g.net/gitea/adolfo/apache_log_report
|
91
99
|
licenses:
|
@@ -110,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
110
118
|
- !ruby/object:Gem::Version
|
111
119
|
version: '0'
|
112
120
|
requirements: []
|
113
|
-
rubygems_version: 3.
|
121
|
+
rubygems_version: 3.2.22
|
114
122
|
signing_key:
|
115
123
|
specification_version: 4
|
116
124
|
summary: Generate a request report in OrgMode format from an Apache log file.
|