log_sense 1.5.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.org +27 -0
- data/Gemfile.lock +6 -4
- data/README.org +108 -34
- data/Rakefile +6 -6
- data/exe/log_sense +110 -39
- data/ip_locations/dbip-country-lite.sqlite3 +0 -0
- data/lib/log_sense/aggregator.rb +191 -0
- data/lib/log_sense/apache_aggregator.rb +122 -0
- data/lib/log_sense/apache_log_line_parser.rb +23 -21
- data/lib/log_sense/apache_log_parser.rb +15 -12
- data/lib/log_sense/apache_report_shaper.rb +309 -0
- data/lib/log_sense/emitter.rb +55 -553
- data/lib/log_sense/ip_locator.rb +24 -12
- data/lib/log_sense/options_checker.rb +24 -0
- data/lib/log_sense/options_parser.rb +81 -51
- data/lib/log_sense/rails_aggregator.rb +69 -0
- data/lib/log_sense/rails_log_parser.rb +82 -68
- data/lib/log_sense/rails_report_shaper.rb +183 -0
- data/lib/log_sense/report_shaper.rb +105 -0
- data/lib/log_sense/templates/_cdn_links.html.erb +11 -0
- data/lib/log_sense/templates/_command_invocation.html.erb +4 -0
- data/lib/log_sense/templates/_log_structure.html.erb +7 -1
- data/lib/log_sense/templates/_output_table.html.erb +6 -2
- data/lib/log_sense/templates/_rails.css.erb +7 -0
- data/lib/log_sense/templates/_summary.html.erb +9 -7
- data/lib/log_sense/templates/_summary.txt.erb +2 -2
- data/lib/log_sense/templates/{rails.html.erb → report_html.erb} +19 -37
- data/lib/log_sense/templates/{apache.txt.erb → report_txt.erb} +1 -1
- data/lib/log_sense/version.rb +1 -1
- data/lib/log_sense.rb +19 -9
- data/log_sense.gemspec +1 -1
- data/{apache-screenshot.png → screenshots/apache-screenshot.png} +0 -0
- data/screenshots/rails-screenshot.png +0 -0
- metadata +17 -11
- data/lib/log_sense/apache_data_cruncher.rb +0 -147
- data/lib/log_sense/rails_data_cruncher.rb +0 -141
- data/lib/log_sense/templates/apache.html.erb +0 -115
- data/lib/log_sense/templates/rails.txt.erb +0 -22
data/lib/log_sense/ip_locator.rb
CHANGED
@@ -1,37 +1,47 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
1
|
+
require "csv"
|
2
|
+
require "sqlite3"
|
3
|
+
require "ipaddr"
|
4
|
+
require "iso_country_codes"
|
5
5
|
|
6
6
|
module LogSense
|
7
7
|
#
|
8
8
|
# Populate table of IP Locations from dbip-country-lite
|
9
9
|
#
|
10
10
|
module IpLocator
|
11
|
-
DB_FILE = File.join(File.dirname(__FILE__),
|
11
|
+
DB_FILE = File.join(File.dirname(__FILE__), "..", "..", "ip_locations", "dbip-country-lite.sqlite3")
|
12
12
|
|
13
13
|
def self.dbip_to_sqlite(db_location)
|
14
|
-
db = SQLite3::Database.new
|
15
|
-
db.execute
|
14
|
+
db = SQLite3::Database.new ":memory:"
|
15
|
+
db.execute "CREATE TABLE ip_location (
|
16
16
|
from_ip_n INTEGER,
|
17
17
|
from_ip TEXT,
|
18
18
|
to_ip TEXT,
|
19
19
|
country_code TEXT
|
20
|
-
)
|
20
|
+
)"
|
21
21
|
|
22
|
-
ins = db.prepare
|
22
|
+
ins = db.prepare "INSERT INTO ip_location(
|
23
|
+
from_ip_n, from_ip, to_ip, country_code)
|
24
|
+
values (?, ?, ?, ?)"
|
23
25
|
CSV.foreach(db_location) do |row|
|
26
|
+
# skip ip v6 addresses
|
27
|
+
next if row[0].include?(":")
|
28
|
+
|
24
29
|
ip = IPAddr.new row[0]
|
25
30
|
ins.execute(ip.to_i, row[0], row[1], row[2])
|
26
31
|
end
|
27
32
|
|
28
33
|
# persist to file
|
29
34
|
ddb = SQLite3::Database.new(DB_FILE)
|
30
|
-
b = SQLite3::Backup.new(ddb,
|
35
|
+
b = SQLite3::Backup.new(ddb, "main", db, "main")
|
31
36
|
b.step(-1) #=> DONE
|
32
37
|
b.finish
|
33
38
|
end
|
34
39
|
|
40
|
+
def merge(parser_db)
|
41
|
+
ipdb = Sqlite3::Database.open DB_FILE
|
42
|
+
parser_db
|
43
|
+
end
|
44
|
+
|
35
45
|
def self.load_db
|
36
46
|
SQLite3::Database.new DB_FILE
|
37
47
|
end
|
@@ -39,14 +49,16 @@ module LogSense
|
|
39
49
|
def self.locate_ip(ip, db)
|
40
50
|
return unless ip
|
41
51
|
|
42
|
-
query = db.prepare
|
52
|
+
query = db.prepare "SELECT * FROM ip_location
|
53
|
+
where from_ip_n <= ?
|
54
|
+
order by from_ip_n desc limit 1"
|
43
55
|
begin
|
44
56
|
ip_n = IPAddr.new(ip).to_i
|
45
57
|
result_set = query.execute ip_n
|
46
58
|
country_code = result_set.map { |x| x[3] }[0]
|
47
59
|
IsoCountryCodes.find(country_code).name
|
48
60
|
rescue IPAddr::InvalidAddressError
|
49
|
-
|
61
|
+
"INVALID IP"
|
50
62
|
rescue IsoCountryCodes::UnknownCodeError
|
51
63
|
country_code
|
52
64
|
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module LogSense
|
2
|
+
#
|
3
|
+
# Check options and return appropriate error if
|
4
|
+
# command arguments are wrong
|
5
|
+
#
|
6
|
+
module OptionsChecker
|
7
|
+
SUPPORTED_CHAINS = {
|
8
|
+
rails: %i[txt html sqlite3 ufw],
|
9
|
+
apache: %i[txt html sqlite3 ufw]
|
10
|
+
}.freeze
|
11
|
+
|
12
|
+
def self.compatible?(iformat, oformat)
|
13
|
+
(SUPPORTED_CHAINS[iformat.to_sym] || []).include? oformat.to_sym
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.chains_to_s
|
17
|
+
string = ""
|
18
|
+
SUPPORTED_CHAINS.each do |iformat, oformat|
|
19
|
+
string << "- #{iformat}: #{oformat.join(", ")}\n"
|
20
|
+
end
|
21
|
+
string
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -1,6 +1,7 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require "optparse"
|
2
|
+
require "optparse/date"
|
3
|
+
require "log_sense/version"
|
4
|
+
require "log_sense/options_checker"
|
4
5
|
|
5
6
|
module LogSense
|
6
7
|
module OptionsParser
|
@@ -9,94 +10,121 @@ module LogSense
|
|
9
10
|
#
|
10
11
|
def self.parse(options)
|
11
12
|
limit = 100
|
12
|
-
args = {}
|
13
|
+
args = {}
|
13
14
|
|
14
15
|
opt_parser = OptionParser.new do |opts|
|
15
|
-
opts.banner =
|
16
|
+
opts.banner = "Usage: log_sense [options] [logfile ...]"
|
16
17
|
|
17
|
-
opts.on(
|
18
|
-
|
18
|
+
opts.on("-tTITLE", "--title=TITLE",
|
19
|
+
String,
|
20
|
+
"Title to use in the report") do |optval|
|
21
|
+
args[:title] = optval
|
19
22
|
end
|
20
23
|
|
21
|
-
opts.on(
|
22
|
-
|
24
|
+
opts.on("-fFORMAT", "--input-format=FORMAT",
|
25
|
+
String,
|
26
|
+
"Input format (either rails or apache)") do |optval|
|
27
|
+
args[:input_format] = optval
|
23
28
|
end
|
24
29
|
|
25
|
-
opts.on(
|
26
|
-
|
30
|
+
opts.on("-iFORMAT", "--input-files=file,file,",
|
31
|
+
Array,
|
32
|
+
"Input files (can also be passed directly)") do |optval|
|
33
|
+
args[:input_filenames] = optval
|
27
34
|
end
|
28
35
|
|
29
|
-
opts.on(
|
30
|
-
|
36
|
+
opts.on("-tFORMAT", "--output-format=FORMAT",
|
37
|
+
String,
|
38
|
+
"Output format: html, org, txt, sqlite.") do |optval|
|
39
|
+
args[:output_format] = optval
|
31
40
|
end
|
32
41
|
|
33
|
-
opts.on(
|
34
|
-
|
42
|
+
opts.on("-oOUTPUT_FILE", "--output-file=OUTPUT_FILE",
|
43
|
+
String,
|
44
|
+
"Output file") do |n|
|
45
|
+
args[:output_filename] = n
|
35
46
|
end
|
36
47
|
|
37
|
-
opts.on(
|
38
|
-
|
48
|
+
opts.on("-bDATE", "--begin=DATE",
|
49
|
+
Date,
|
50
|
+
"Consider entries after or on DATE") do |optval|
|
51
|
+
args[:from_date] = optval
|
39
52
|
end
|
40
53
|
|
41
|
-
opts.on(
|
42
|
-
|
54
|
+
opts.on("-eDATE", "--end=DATE",
|
55
|
+
Date,
|
56
|
+
"Consider entries before or on DATE") do |optval|
|
57
|
+
args[:to_date] = optval
|
43
58
|
end
|
44
59
|
|
45
|
-
opts.on(
|
46
|
-
|
60
|
+
opts.on("-lN", "--limit=N",
|
61
|
+
Integer,
|
62
|
+
"Limit to the N most requested resources (defaults to #{limit})") do |optval|
|
63
|
+
args[:limit] = optval
|
47
64
|
end
|
48
65
|
|
49
|
-
opts.on(
|
50
|
-
|
66
|
+
opts.on("-wWIDTH", "--width=WIDTH",
|
67
|
+
Integer,
|
68
|
+
"Maximum width of long columns in textual reports") do |optval|
|
69
|
+
args[:width] = optval
|
51
70
|
end
|
52
71
|
|
53
|
-
opts.on(
|
54
|
-
|
72
|
+
opts.on("-rROWS", "--rows=ROWS",
|
73
|
+
Integer,
|
74
|
+
"Maximum number of rows for columns with multiple entries in textual reports") do |optval|
|
75
|
+
args[:inner_rows] = optval
|
55
76
|
end
|
56
77
|
|
57
|
-
opts.on(
|
58
|
-
|
59
|
-
|
78
|
+
opts.on("-pPATTERN", "--pattern=PATTERN",
|
79
|
+
String,
|
80
|
+
"Pattern to use with ufw report to decide IP to blacklist") do |optval|
|
81
|
+
args[:pattern] = optval
|
82
|
+
end
|
83
|
+
|
84
|
+
opts.on("-cPOLICY", "--crawlers=POLICY",
|
85
|
+
String,
|
86
|
+
"Decide what to do with crawlers (applies to Apache Logs)") do |optval|
|
87
|
+
case optval
|
88
|
+
when "only"
|
60
89
|
args[:only_crawlers] = true
|
61
|
-
when
|
90
|
+
when "ignore"
|
62
91
|
args[:ignore_crawlers] = true
|
63
92
|
end
|
64
93
|
end
|
65
94
|
|
66
|
-
opts.on(
|
95
|
+
opts.on("-ns", "--no-selfpoll",
|
96
|
+
"Ignore self poll entries (requests from ::1; applies to Apache Logs)") do
|
67
97
|
args[:no_selfpoll] = true
|
68
98
|
end
|
69
99
|
|
70
|
-
opts.on(
|
100
|
+
opts.on("-ng", "--no-geo",
|
101
|
+
"Do not geolocate entries") do
|
102
|
+
args[:geolocation] = false
|
103
|
+
end
|
104
|
+
|
105
|
+
opts.on("--verbose", "Inform about progress (output to STDERR)") do
|
71
106
|
args[:verbose] = true
|
72
107
|
end
|
73
108
|
|
74
|
-
opts.on(
|
109
|
+
opts.on("-v", "--version", "Prints version information") do
|
75
110
|
puts "log_sense version #{LogSense::VERSION}"
|
76
|
-
puts
|
77
|
-
puts
|
111
|
+
puts "Copyright (C) 2021 Shair.Tech"
|
112
|
+
puts "Distributed under the terms of the MIT license"
|
78
113
|
exit
|
79
114
|
end
|
80
115
|
|
81
|
-
opts.on(
|
116
|
+
opts.on("-h", "--help", "Prints this help") do
|
82
117
|
puts opts
|
83
|
-
puts
|
118
|
+
puts
|
84
119
|
puts "This is version #{LogSense::VERSION}"
|
85
120
|
|
86
|
-
puts
|
87
|
-
puts
|
88
|
-
|
89
|
-
templates = Dir.glob(pathname).select { |x| !File.basename(x).start_with?(/_|#/) && !File.basename(x).end_with?('~') }
|
90
|
-
components = templates.map { |x| File.basename(x).split '.' }.group_by { |x| x[0] }
|
91
|
-
components.each do |k, vs|
|
92
|
-
puts "#{k} parsing can produce the following outputs:"
|
93
|
-
puts ' - sqlite'
|
94
|
-
vs.each do |v|
|
95
|
-
puts " - #{v[1]}"
|
96
|
-
end
|
97
|
-
end
|
121
|
+
puts
|
122
|
+
puts "Output formats:"
|
123
|
+
puts
|
98
124
|
|
99
|
-
|
125
|
+
puts OptionsChecker.chains_to_s
|
126
|
+
|
127
|
+
exit 0
|
100
128
|
end
|
101
129
|
end
|
102
130
|
|
@@ -104,12 +132,14 @@ module LogSense
|
|
104
132
|
|
105
133
|
args[:limit] ||= limit
|
106
134
|
args[:input_filenames] ||= []
|
107
|
-
args[:input_format] ||=
|
108
|
-
args[:output_format] ||=
|
135
|
+
args[:input_format] ||= "apache"
|
136
|
+
args[:output_format] ||= "html"
|
109
137
|
args[:ignore_crawlers] ||= false
|
110
138
|
args[:only_crawlers] ||= false
|
111
139
|
args[:no_selfpoll] ||= false
|
112
140
|
args[:verbose] ||= false
|
141
|
+
# if set to false leave, otherwise set to true
|
142
|
+
args[:geolocation] = true unless args[:geolocation] == false
|
113
143
|
|
114
144
|
args
|
115
145
|
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
module LogSense
|
2
|
+
class RailsAggregator < Aggregator
|
3
|
+
def initialize(db, options = { limit: 900 })
|
4
|
+
@table = "Event"
|
5
|
+
@date_field = "started_at"
|
6
|
+
@url_field = "url"
|
7
|
+
|
8
|
+
@db = db
|
9
|
+
@options = options
|
10
|
+
end
|
11
|
+
|
12
|
+
#
|
13
|
+
# take a sqlite3 database and analyze data
|
14
|
+
#
|
15
|
+
# @ variables are automatically put in the returned data
|
16
|
+
#
|
17
|
+
def aggregate
|
18
|
+
aggregate_log_info
|
19
|
+
aggregate_statuses
|
20
|
+
aggregate_ips
|
21
|
+
|
22
|
+
@daily_distribution = @db.execute %(
|
23
|
+
SELECT date(started_at), #{human_readable_day}, count(started_at)
|
24
|
+
from Event
|
25
|
+
where #{filter}
|
26
|
+
group by date(started_at)).gsub("\n", "")
|
27
|
+
|
28
|
+
@time_distribution = @db.execute %(
|
29
|
+
SELECT strftime("%H", started_at), count(started_at) from Event
|
30
|
+
where #{filter}
|
31
|
+
group by strftime('%H', started_at)).gsub("\n", "")
|
32
|
+
|
33
|
+
@performance = @db.execute %(
|
34
|
+
SELECT distinct(controller),
|
35
|
+
count(controller),
|
36
|
+
printf("%.2f", min(duration_total_ms)),
|
37
|
+
printf("%.2f", avg(duration_total_ms)),
|
38
|
+
printf("%.2f", max(duration_total_ms))
|
39
|
+
from Event
|
40
|
+
where #{filter}
|
41
|
+
group by controller order by controller).gsub("\n", "")
|
42
|
+
|
43
|
+
@fatal = @db.execute %Q(
|
44
|
+
SELECT strftime("%Y-%m-%d %H:%M", started_at),
|
45
|
+
ip,
|
46
|
+
url,
|
47
|
+
error.description,
|
48
|
+
event.log_id
|
49
|
+
FROM Event JOIN Error
|
50
|
+
ON event.log_id == error.log_id
|
51
|
+
WHERE #{filter} and exit_status == 'F').gsub("\n", "") || [[]]
|
52
|
+
|
53
|
+
@internal_server_error = @db.execute %Q(
|
54
|
+
SELECT strftime("%Y-%m-%d %H:%M", started_at), status, ip, url,
|
55
|
+
error.description,
|
56
|
+
event.log_id
|
57
|
+
FROM Event JOIN Error
|
58
|
+
ON event.log_id == error.log_id
|
59
|
+
WHERE #{filter} and substr(status, 1, 1) == '5').gsub("\n", "") || [[]]
|
60
|
+
|
61
|
+
@error = @db.execute %Q(
|
62
|
+
SELECT filename, log_id, context, description, count(log_id)
|
63
|
+
FROM Error
|
64
|
+
GROUP BY description).gsub("\n", "") || [[]]
|
65
|
+
|
66
|
+
instance_vars_to_hash
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
@@ -1,68 +1,87 @@
|
|
1
|
-
require
|
1
|
+
require "sqlite3"
|
2
2
|
|
3
3
|
module LogSense
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
unique_visitor TEXT,
|
15
|
-
url TEXT,
|
16
|
-
controller TEXT,
|
17
|
-
html_verb TEXT,
|
18
|
-
status INTEGER,
|
19
|
-
duration_total_ms FLOAT,
|
20
|
-
duration_views_ms FLOAT,
|
21
|
-
duration_ar_ms FLOAT,
|
22
|
-
allocations INTEGER,
|
23
|
-
comment TEXT,
|
24
|
-
source_file TEXT,
|
25
|
-
line_number INTEGER
|
26
|
-
)'
|
4
|
+
#
|
5
|
+
# parse a Rails log file and return a SQLite3 DB
|
6
|
+
#
|
7
|
+
class RailsLogParser
|
8
|
+
#
|
9
|
+
# Tell users which format I can parse
|
10
|
+
#
|
11
|
+
def provide
|
12
|
+
[:rails]
|
13
|
+
end
|
27
14
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
15
|
+
def parse(streams, options = {})
|
16
|
+
db = SQLite3::Database.new ":memory:"
|
17
|
+
|
18
|
+
db.execute <<-EOS
|
19
|
+
CREATE TABLE IF NOT EXISTS Event(
|
20
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
21
|
+
exit_status TEXT,
|
22
|
+
started_at TEXT,
|
23
|
+
ended_at TEXT,
|
24
|
+
log_id TEXT,
|
25
|
+
ip TEXT,
|
26
|
+
unique_visitor TEXT,
|
27
|
+
url TEXT,
|
28
|
+
controller TEXT,
|
29
|
+
html_verb TEXT,
|
30
|
+
status INTEGER,
|
31
|
+
duration_total_ms FLOAT,
|
32
|
+
duration_views_ms FLOAT,
|
33
|
+
duration_ar_ms FLOAT,
|
34
|
+
allocations INTEGER,
|
35
|
+
comment TEXT,
|
36
|
+
source_file TEXT,
|
37
|
+
line_number INTEGER
|
38
|
+
)
|
39
|
+
EOS
|
40
|
+
|
41
|
+
ins = db.prepare <<-EOS
|
42
|
+
insert into Event(
|
43
|
+
exit_status,
|
44
|
+
started_at,
|
45
|
+
ended_at,
|
46
|
+
log_id,
|
47
|
+
ip,
|
48
|
+
unique_visitor,
|
49
|
+
url,
|
50
|
+
controller,
|
51
|
+
html_verb,
|
52
|
+
status,
|
53
|
+
duration_total_ms,
|
54
|
+
duration_views_ms,
|
55
|
+
duration_ar_ms,
|
56
|
+
allocations,
|
57
|
+
comment,
|
58
|
+
source_file,
|
59
|
+
line_number
|
60
|
+
)
|
61
|
+
values (#{Array.new(17, "?").join(", ")})
|
62
|
+
EOS
|
48
63
|
|
49
|
-
db.execute
|
64
|
+
db.execute <<-EOS
|
65
|
+
CREATE TABLE IF NOT EXISTS Error(
|
50
66
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
51
67
|
log_id TEXT,
|
52
68
|
context TEXT,
|
53
69
|
description TEXT,
|
54
70
|
filename TEXT,
|
55
71
|
line_number INTEGER
|
56
|
-
|
72
|
+
)
|
73
|
+
EOS
|
57
74
|
|
58
|
-
ins_error = db.prepare
|
75
|
+
ins_error = db.prepare <<-EOS
|
76
|
+
insert into Error(
|
59
77
|
log_id,
|
60
78
|
context,
|
61
79
|
description,
|
62
80
|
filename,
|
63
81
|
line_number
|
64
|
-
|
65
|
-
|
82
|
+
)
|
83
|
+
values (?, ?, ?, ?, ?)
|
84
|
+
EOS
|
66
85
|
|
67
86
|
# requests in the log might be interleaved.
|
68
87
|
#
|
@@ -93,7 +112,11 @@ module LogSense
|
|
93
112
|
|
94
113
|
data = match_and_process_error line
|
95
114
|
if data
|
96
|
-
ins_error.execute(data[:log_id],
|
115
|
+
ins_error.execute(data[:log_id],
|
116
|
+
data[:context],
|
117
|
+
data[:description],
|
118
|
+
filename,
|
119
|
+
line_number)
|
97
120
|
next
|
98
121
|
end
|
99
122
|
|
@@ -199,7 +222,7 @@ module LogSense
|
|
199
222
|
EXCEPTION = /[A-Za-z_0-9:]+(Error)?/
|
200
223
|
ERROR_REGEXP = /^\[#{ID}\] (?<context>#{EXCEPTION}) \((?<description>(#{EXCEPTION})?.*)\):/
|
201
224
|
|
202
|
-
def
|
225
|
+
def match_and_process_error line
|
203
226
|
matchdata = ERROR_REGEXP.match line
|
204
227
|
if matchdata
|
205
228
|
{
|
@@ -207,16 +230,13 @@ module LogSense
|
|
207
230
|
context: matchdata[:context],
|
208
231
|
description: matchdata[:description]
|
209
232
|
}
|
210
|
-
else
|
211
|
-
nil
|
212
233
|
end
|
213
234
|
end
|
214
235
|
|
215
|
-
|
216
236
|
# I, [2021-10-19T08:16:34.343858 #10477] INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
|
217
237
|
STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\] INFO -- : \[#{ID}\] Started #{VERB} "#{URL}" for #{IP} at/
|
218
238
|
|
219
|
-
def
|
239
|
+
def match_and_process_start line
|
220
240
|
matchdata = STARTED_REGEXP.match line
|
221
241
|
if matchdata
|
222
242
|
{
|
@@ -226,8 +246,6 @@ module LogSense
|
|
226
246
|
url: matchdata[:url],
|
227
247
|
ip: matchdata[:ip]
|
228
248
|
}
|
229
|
-
else
|
230
|
-
nil
|
231
249
|
end
|
232
250
|
end
|
233
251
|
|
@@ -237,7 +255,7 @@ module LogSense
|
|
237
255
|
# I, [2021-12-06T14:28:19.736545 #2804090] INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
|
238
256
|
COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\] INFO -- : \[#{ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/
|
239
257
|
|
240
|
-
def
|
258
|
+
def match_and_process_completed(line)
|
241
259
|
matchdata = (COMPLETED_REGEXP.match line)
|
242
260
|
# exit_status = matchdata[:status].to_i == 500 ? "E" : "I"
|
243
261
|
if matchdata
|
@@ -252,23 +270,19 @@ module LogSense
|
|
252
270
|
allocations: matchdata[:alloc],
|
253
271
|
comment: ""
|
254
272
|
}
|
255
|
-
else
|
256
|
-
nil
|
257
273
|
end
|
258
274
|
end
|
259
275
|
|
260
276
|
# I, [2021-10-19T08:16:34.345162 #10477] INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Processing by PeopleController#show as HTML
|
261
277
|
PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\] INFO -- : \[#{ID}\] Processing by (?<controller>[^ ]+) as/
|
262
278
|
|
263
|
-
def
|
279
|
+
def match_and_process_processing_by line
|
264
280
|
matchdata = PROCESSING_REGEXP.match line
|
265
281
|
if matchdata
|
266
282
|
{
|
267
283
|
log_id: matchdata[:id],
|
268
284
|
controller: matchdata[:controller]
|
269
285
|
}
|
270
|
-
else
|
271
|
-
nil
|
272
286
|
end
|
273
287
|
end
|
274
288
|
|
@@ -278,7 +292,7 @@ module LogSense
|
|
278
292
|
# F, [2021-12-04T00:34:05.839269 #2735058] FATAL -- : [3a16162e-a6a5-435e-a9d8-c4df5dc0f728] actionpack (5.2.4.4) lib/action_dispatch/middleware/debug_exceptions.rb:65:in `call'
|
279
293
|
FATAL_REGEXP = /F, \[#{TIMESTAMP} #[0-9]+\] FATAL -- : \[#{ID}\] (?<comment>.*)$/
|
280
294
|
|
281
|
-
def
|
295
|
+
def match_and_process_fatal(line)
|
282
296
|
matchdata = FATAL_REGEXP.match line
|
283
297
|
if matchdata
|
284
298
|
{
|
@@ -286,14 +300,14 @@ module LogSense
|
|
286
300
|
log_id: matchdata[:id],
|
287
301
|
comment: matchdata[:comment]
|
288
302
|
}
|
289
|
-
else
|
290
|
-
nil
|
291
303
|
end
|
292
304
|
end
|
293
305
|
|
294
306
|
# generate a unique visitor id from an event
|
295
|
-
def
|
296
|
-
|
307
|
+
def unique_visitor_id(event)
|
308
|
+
date = event[:started_at] || event[:ended_at] || "1970-01-01"
|
309
|
+
"#{DateTime.parse(date).strftime("%Y-%m-%d")} #{event[:ip]}"
|
297
310
|
end
|
298
311
|
end
|
299
312
|
end
|
313
|
+
|