log_sense 1.1.2 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d96a22ce71f0c0266811faa1853981d1211b93687bfd2e3077b91189feb5742b
|
4
|
+
data.tar.gz: 014ad1230a6a83b379310ba2b93b16bfd956db19a8f087bd2c216b5e77211620
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8fe6fb1a329b0cabae06199f13c70936c9c7aeaa6c99142a683414ac380bfc23fecb14d3588803f2f9afc24483bfb8f429f867633ce5029c3dc23df1c713dea4
|
7
|
+
data.tar.gz: 10fe88f045845a5573f8ba7aedde8d3475a0f390e6025746799525be685c6db11e5eb9148e9de3f8f05a7950ae9c912f2d67e454a285de08143789db374818c7
|
@@ -92,11 +92,12 @@ module LogSense
|
|
92
92
|
|
93
93
|
@statuses = db.execute "SELECT status, count(status) from Event where #{filter} group by status order by status"
|
94
94
|
|
95
|
+
@by_day_5xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '5' and #{filter} group by date(started_at)"
|
95
96
|
@by_day_4xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '4' and #{filter} group by date(started_at)"
|
96
97
|
@by_day_3xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '3' and #{filter} group by date(started_at)"
|
97
98
|
@by_day_2xx = db.execute "SELECT date(started_at), count(started_at) from Event where substr(status, 1,1) == '2' and #{filter} group by date(started_at)"
|
98
99
|
|
99
|
-
@statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx).group_by { |x| x[0] }.to_a.map { |x|
|
100
|
+
@statuses_by_day = (@by_day_2xx + @by_day_3xx + @by_day_4xx + @by_day_5xx).group_by { |x| x[0] }.to_a.map { |x|
|
100
101
|
[x[0], x[1].map { |y| y[1] }].flatten
|
101
102
|
}
|
102
103
|
|
@@ -104,8 +105,12 @@ module LogSense
|
|
104
105
|
|
105
106
|
@performance = db.execute "SELECT distinct(controller), count(controller), printf(\"%.2f\", min(duration_total_ms)), printf(\"%.2f\", avg(duration_total_ms)), printf(\"%.2f\", max(duration_total_ms)) from Event group by controller order by controller"
|
106
107
|
|
107
|
-
@fatal = db.execute "SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, log_id FROM Event WHERE exit_status == 'F'"
|
108
|
+
@fatal = db.execute ("SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE exit_status == 'F'") || [[]]
|
108
109
|
|
110
|
+
@internal_server_error = (db.execute "SELECT strftime(\"%Y-%m-%d %H:%M\", started_at), status, ip, url, error.description, event.log_id FROM Event JOIN Error ON event.log_id == error.log_id WHERE status is 500") || [[]]
|
111
|
+
|
112
|
+
@error = (db.execute "SELECT log_id, context, description, count(log_id) from Error GROUP BY description") || [[]]
|
113
|
+
|
109
114
|
data = {}
|
110
115
|
self.instance_variables.each do |variable|
|
111
116
|
var_as_symbol = variable.to_s[1..-1].to_sym
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'sqlite3'
|
2
|
+
require 'byebug'
|
2
3
|
|
3
4
|
module LogSense
|
4
5
|
module RailsLogParser
|
@@ -24,7 +25,7 @@ module LogSense
|
|
24
25
|
allocations INTEGER,
|
25
26
|
comment TEXT
|
26
27
|
)'
|
27
|
-
|
28
|
+
|
28
29
|
ins = db.prepare("insert into Event(
|
29
30
|
exit_status,
|
30
31
|
started_at,
|
@@ -44,6 +45,22 @@ module LogSense
|
|
44
45
|
)
|
45
46
|
values (#{Array.new(15, '?').join(', ')})")
|
46
47
|
|
48
|
+
|
49
|
+
db.execute 'CREATE TABLE IF NOT EXISTS Error(
|
50
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
51
|
+
log_id TEXT,
|
52
|
+
context TEXT,
|
53
|
+
description TEXT
|
54
|
+
)'
|
55
|
+
|
56
|
+
ins_error = db.prepare("insert into Error(
|
57
|
+
log_id,
|
58
|
+
context,
|
59
|
+
description
|
60
|
+
)
|
61
|
+
values (?, ?, ?)")
|
62
|
+
|
63
|
+
|
47
64
|
# requests in the log might be interleaved.
|
48
65
|
#
|
49
66
|
# We use the 'pending' variable to progressively store data
|
@@ -65,8 +82,14 @@ module LogSense
|
|
65
82
|
# Different requests might be interleaved, of course
|
66
83
|
|
67
84
|
File.readlines(filename).each do |line|
|
68
|
-
#
|
69
|
-
next if line[0] != 'I' and line[0] != 'F'
|
85
|
+
# I and F for completed requests, [ is for error messages
|
86
|
+
next if line[0] != 'I' and line[0] != 'F' and line[0] != '['
|
87
|
+
|
88
|
+
data = self.match_and_process_error line
|
89
|
+
if data
|
90
|
+
ins_error.execute(data[:log_id], data[:context], data[:description])
|
91
|
+
next
|
92
|
+
end
|
70
93
|
|
71
94
|
data = self.match_and_process_start line
|
72
95
|
if data
|
@@ -145,39 +168,6 @@ module LogSense
|
|
145
168
|
end
|
146
169
|
end
|
147
170
|
|
148
|
-
|
149
|
-
data = self.match_and_process_completed_no_alloc line
|
150
|
-
if data
|
151
|
-
id = data[:log_id]
|
152
|
-
|
153
|
-
# it might as well be that the first event started before
|
154
|
-
# the log. With this, we make sure we add only events whose
|
155
|
-
# start was logged and parsed
|
156
|
-
if pending[id]
|
157
|
-
event = data.merge (pending[id] || {})
|
158
|
-
|
159
|
-
ins.execute(
|
160
|
-
event[:exit_status],
|
161
|
-
event[:started_at],
|
162
|
-
event[:ended_at],
|
163
|
-
event[:log_id],
|
164
|
-
event[:ip],
|
165
|
-
"#{DateTime.parse(event[:ended_at]).strftime("%Y-%m-%d")} #{event[:ip]}",
|
166
|
-
event[:url],
|
167
|
-
event[:controller],
|
168
|
-
event[:html_verb],
|
169
|
-
event[:status],
|
170
|
-
event[:duration_total_ms],
|
171
|
-
event[:duration_views_ms],
|
172
|
-
event[:duration_ar_ms],
|
173
|
-
event[:allocations],
|
174
|
-
event[:comment]
|
175
|
-
)
|
176
|
-
|
177
|
-
pending.delete(id)
|
178
|
-
end
|
179
|
-
end
|
180
|
-
|
181
171
|
end
|
182
172
|
|
183
173
|
db
|
@@ -189,8 +179,30 @@ module LogSense
|
|
189
179
|
URL = /(?<url>[^"]+)/
|
190
180
|
IP = /(?<ip>[0-9.]+)/
|
191
181
|
STATUS = /(?<status>[0-9]+)/
|
182
|
+
STATUS_IN_WORDS = /(OK|Unauthorized|Found|Internal Server Error|Bad Request|Method Not Allowed|Request Timeout|Not Implemented|Bad Gateway|Service Unavailable)/
|
192
183
|
MSECS = /[0-9.]+/
|
193
184
|
|
185
|
+
# Error Messages
|
186
|
+
# [584cffcc-f1fd-4b5c-bb8b-b89621bd4921] ActionController::RoutingError (No route matches [GET] "/assets/foundation-icons.svg"):
|
187
|
+
# [fd8df8b5-83c9-48b5-a056-e5026e31bd5e] ActionView::Template::Error (undefined method `all_my_ancestor' for nil:NilClass):
|
188
|
+
# [d17ed55c-f5f1-442a-a9d6-3035ab91adf0] ActionView::Template::Error (undefined method `volunteer_for' for #<DonationsController:0x007f4864c564b8>
|
189
|
+
CONTEXT = /(?<context>[^ ]+Error)/
|
190
|
+
ERROR_REGEXP = /^\[#{ID}\] #{CONTEXT} \((?<description>.*)\):/
|
191
|
+
|
192
|
+
def self.match_and_process_error line
|
193
|
+
matchdata = ERROR_REGEXP.match line
|
194
|
+
if matchdata
|
195
|
+
{
|
196
|
+
log_id: matchdata[:id],
|
197
|
+
context: matchdata[:context],
|
198
|
+
description: matchdata[:description]
|
199
|
+
}
|
200
|
+
else
|
201
|
+
nil
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
|
194
206
|
# I, [2021-10-19T08:16:34.343858 #10477] INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Started GET "/grow/people/471" for 217.77.80.35 at 2021-10-19 08:16:34 +0000
|
195
207
|
STARTED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\] INFO -- : \[#{ID}\] Started #{VERB} "#{URL}" for #{IP} at/
|
196
208
|
|
@@ -209,11 +221,15 @@ module LogSense
|
|
209
221
|
end
|
210
222
|
end
|
211
223
|
|
224
|
+
# TODO: Add regexps for the performance data (Views ...). We have three cases (view, active records, allocations), (views, active records), (active records, allocations)
|
212
225
|
# I, [2021-10-19T08:16:34.712331 #10477] INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Completed 200 OK in 367ms (Views: 216.7ms | ActiveRecord: 141.3ms | Allocations: 168792)
|
213
|
-
|
226
|
+
# I, [2021-12-09T16:53:52.657727 #2735058] INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
|
227
|
+
# I, [2021-12-06T14:28:19.736545 #2804090] INFO -- : [34091cb5-3e7b-4042-aaf8-6c6510d3f14c] Completed 500 Internal Server Error in 66ms (ActiveRecord: 8.0ms | Allocations: 24885)
|
228
|
+
COMPLETED_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\] INFO -- : \[#{ID}\] Completed #{STATUS} #{STATUS_IN_WORDS} in (?<total>#{MSECS})ms \((Views: (?<views>#{MSECS})ms \| )?ActiveRecord: (?<arec>#{MSECS})ms( \| Allocations: (?<alloc>[0-9]+))?\)/
|
214
229
|
|
215
230
|
def self.match_and_process_completed line
|
216
231
|
matchdata = (COMPLETED_REGEXP.match line)
|
232
|
+
# exit_status = matchdata[:status].to_i == 500 ? "E" : "I"
|
217
233
|
if matchdata
|
218
234
|
{
|
219
235
|
exit_status: "I",
|
@@ -231,29 +247,6 @@ module LogSense
|
|
231
247
|
end
|
232
248
|
end
|
233
249
|
|
234
|
-
# I, [2021-12-09T16:53:52.657727 #2735058] INFO -- : [0064e403-9eb2-439d-8fe1-a334c86f5532] Completed 200 OK in 13ms (Views: 11.1ms | ActiveRecord: 1.2ms)
|
235
|
-
COMPLETED_NO_ALLOC_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\] INFO -- : \[#{ID}\] Completed #{STATUS} [^ ]+ in (?<total>#{MSECS})ms \(Views: (?<views>#{MSECS})ms \| ActiveRecord: (?<arec>#{MSECS})ms\)/
|
236
|
-
|
237
|
-
def self.match_and_process_completed_no_alloc line
|
238
|
-
matchdata = (COMPLETED_NO_ALLOC_REGEXP.match line)
|
239
|
-
if matchdata
|
240
|
-
{
|
241
|
-
exit_status: "I",
|
242
|
-
ended_at: matchdata[:timestamp],
|
243
|
-
log_id: matchdata[:id],
|
244
|
-
status: matchdata[:status],
|
245
|
-
duration_total_ms: matchdata[:total],
|
246
|
-
duration_views_ms: matchdata[:views],
|
247
|
-
duration_ar_ms: matchdata[:arec],
|
248
|
-
allocations: -1,
|
249
|
-
comment: ""
|
250
|
-
}
|
251
|
-
else
|
252
|
-
nil
|
253
|
-
end
|
254
|
-
end
|
255
|
-
|
256
|
-
|
257
250
|
# I, [2021-10-19T08:16:34.345162 #10477] INFO -- : [67103c0d-455d-4fe8-951e-87e97628cb66] Processing by PeopleController#show as HTML
|
258
251
|
PROCESSING_REGEXP = /I, \[#{TIMESTAMP} #[0-9]+\] INFO -- : \[#{ID}\] Processing by (?<controller>[^ ]+) as/
|
259
252
|
|
@@ -69,7 +69,7 @@
|
|
69
69
|
|
70
70
|
.card-divider {
|
71
71
|
padding: 0.2rem 0.4rem 0.2rem 0.4rem;
|
72
|
-
background: #
|
72
|
+
background: #D30001;
|
73
73
|
color: white;
|
74
74
|
}
|
75
75
|
|
@@ -156,19 +156,10 @@
|
|
156
156
|
"Log Structure",
|
157
157
|
"Daily Distribution",
|
158
158
|
"Time Distribution",
|
159
|
-
"Most Requested Pages",
|
160
|
-
"Most Requested Resources",
|
161
|
-
"404 on HTML Files",
|
162
|
-
"404 on other Resources",
|
163
|
-
"Attacks",
|
164
159
|
"Statuses",
|
165
|
-
"
|
166
|
-
"
|
167
|
-
"Platforms",
|
168
|
-
"Referers",
|
160
|
+
"Rails Performance",
|
161
|
+
"Fatal Events",
|
169
162
|
"IPs",
|
170
|
-
"Geolocation",
|
171
|
-
"Streaks",
|
172
163
|
"Command Invocation",
|
173
164
|
"Performance"
|
174
165
|
].each do |item| %>
|
@@ -262,11 +253,24 @@
|
|
262
253
|
},
|
263
254
|
{ title: "Rails Performance",
|
264
255
|
header: ['Controller', 'Hits', 'Min', 'Avg', 'Max'],
|
265
|
-
rows: @data[:performance]
|
256
|
+
rows: @data[:performance]
|
257
|
+
},
|
266
258
|
{ title: "Fatal Events",
|
267
|
-
header: ['Date', 'IP', 'URL', 'Log ID'], rows: @data[:fatal]
|
259
|
+
header: ['Date', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:fatal],
|
260
|
+
col: "small-12 cell"
|
261
|
+
},
|
262
|
+
{ title: "Internal Server Errors",
|
263
|
+
header: ['Date', 'Status', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:internal_server_error],
|
264
|
+
col: "small-12 cell"
|
265
|
+
},
|
266
|
+
{ title: "Errors",
|
267
|
+
header: ['Log ID', 'Context', 'Description', 'Count'], rows: @data[:error],
|
268
|
+
col: "small-12 cell"
|
269
|
+
},
|
270
|
+
{ title: "IPs",
|
271
|
+
header: ["IPs", "Hits", "Country"],
|
272
|
+
rows: data[:ips]
|
268
273
|
},
|
269
|
-
{ title: "IPs", header: ["IPs", "Hits", "Country"], rows: data[:ips] },
|
270
274
|
]
|
271
275
|
%>
|
272
276
|
<div class="grid-x grid-margin-x">
|
@@ -26,14 +26,6 @@ table.align_column(2, :right)
|
|
26
26
|
table
|
27
27
|
%>
|
28
28
|
|
29
|
-
** IP and Country
|
30
|
-
|
31
|
-
<%=
|
32
|
-
table = Terminal::Table.new headings: ['IP', 'Events', 'Country'], rows: @data[:ips]
|
33
|
-
table.align_column(1, :right)
|
34
|
-
table
|
35
|
-
%>
|
36
|
-
|
37
29
|
** Rails Performance
|
38
30
|
|
39
31
|
<%= table = Terminal::Table.new headings: ['Controller', 'Hits', 'Min', 'Avg', 'Max'], rows: @data[:performance]
|
@@ -46,7 +38,27 @@ table
|
|
46
38
|
|
47
39
|
** Fatal Events
|
48
40
|
|
49
|
-
<%= table = Terminal::Table.new headings: ['Date', 'IP', 'URL', 'Log ID'], rows: @data[:fatal]
|
41
|
+
<%= table = Terminal::Table.new headings: ['Date', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:fatal]
|
42
|
+
table
|
43
|
+
%>
|
44
|
+
|
45
|
+
** Internal Server Errors
|
46
|
+
|
47
|
+
<%= table = Terminal::Table.new headings: ['Date', 'Status', 'IP', 'URL', 'Description', 'Log ID'], rows: @data[:internal_server_error]
|
48
|
+
table
|
49
|
+
%>
|
50
|
+
|
51
|
+
** Errors
|
52
|
+
|
53
|
+
<%= table = Terminal::Table.new headings: ['Log ID', 'Context', 'Description', 'Count'], rows: @data[:error]
|
54
|
+
table
|
55
|
+
%>
|
56
|
+
|
57
|
+
** IPs
|
58
|
+
|
59
|
+
<%=
|
60
|
+
table = Terminal::Table.new headings: ['IP', 'Hits', 'Country'], rows: @data[:ips]
|
61
|
+
table.align_column(1, :right)
|
50
62
|
table
|
51
63
|
%>
|
52
64
|
|
data/lib/log_sense/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log_sense
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adolfo Villafiorita
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-12-
|
11
|
+
date: 2021-12-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: apache_log-parser
|