log_stats 0.4.2 → 0.4.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/example/log_stats +30 -12
- data/lib/log_stats/line_parser.rb +2 -1
- data/lib/log_stats/logger.rb +15 -1
- data/lib/log_stats/version.rb +1 -1
- data/lib/log_stats.rb +5 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a2d41ac522f442b77ac7626e31fb585bc79efde4
|
4
|
+
data.tar.gz: b1585a93945b1296d2c66a16dc2fb38d8b27317a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6839efa1ef45c97ac75ba8db6c5c4179dd0fb5ac45ebebeafb33d06d478db366da0232c77a33b8138320f5c2697c001534ae77d346ab3d57f9be2059be6d80a1
|
7
|
+
data.tar.gz: da3dda45aba36b4b0aa58e59f709e423783c3abd08949a1d62cf85cc39312e6749f75297c1402ab6056b09b2d31835866efb3d0daf08700688c7d3dfdca5d414
|
data/example/log_stats
CHANGED
@@ -4,16 +4,32 @@
|
|
4
4
|
# This example script shows custom parsing of API requests from a log file.
|
5
5
|
#############################################################################
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
require 'log_stats'
|
7
|
+
require "log_stats"
|
10
8
|
require "log_stats/config"
|
11
9
|
require "json"
|
12
10
|
require "uri"
|
13
11
|
|
14
12
|
parse_time = Proc.new { |line| line[/\b20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d/] }
|
13
|
+
|
15
14
|
custom_config = {
|
16
|
-
events:
|
15
|
+
events: {
|
16
|
+
requests: {
|
17
|
+
# NOTE: matches Heroku router lines. Also matches Papertrails slightly modified lines.
|
18
|
+
line_pattern: /\s(heroku\/router)|(heroku\[router\]:)\s/,
|
19
|
+
fields: [
|
20
|
+
{name: :time, parse: parse_time},
|
21
|
+
{name: :method},
|
22
|
+
{name: :host},
|
23
|
+
{name: :path},
|
24
|
+
{name: :status, numeric: true},
|
25
|
+
{name: :code, optional: true},
|
26
|
+
{name: :service, numeric: true}
|
27
|
+
],
|
28
|
+
top_list_limit: 100,
|
29
|
+
apdex: {tolerating: 500, frustrated: 2000},
|
30
|
+
apdex_goal: 0.9,
|
31
|
+
stats: false # Skip listing stats per request path to keep output size manageable
|
32
|
+
},
|
17
33
|
api_calls: {
|
18
34
|
# 2017-02-19T06:21:25.522274+00:00 app[worker.2]: [WARN] [Vac::Request] Slow response time for url=http://sumore02.cmore.dk/api/tve_web/search/categories/160145/assets/ method=get status=304 size= response_time=141
|
19
35
|
line_pattern: /\s\[Vac::Request\] Slow response time\s/,
|
@@ -26,20 +42,22 @@ custom_config = {
|
|
26
42
|
group_by: {
|
27
43
|
hostname: {
|
28
44
|
id: Proc.new { |api_call| URI(api_call[:url]).host }
|
29
|
-
},
|
30
|
-
method: {
|
31
|
-
id: Proc.new { |api_call| URI(api_call[:method]) }
|
32
45
|
}
|
33
|
-
|
46
|
+
# NOTE: Grouping by HTTP method doesn't really add much at the moment, so commenting out for now
|
47
|
+
# method: {
|
48
|
+
# id: Proc.new { |api_call| URI(api_call[:method]) }
|
49
|
+
# }
|
50
|
+
},
|
51
|
+
events: false, # Skip listing all individual API calls to keep output size manageable
|
52
|
+
limit: 5
|
34
53
|
}
|
35
|
-
}
|
36
|
-
verbose:
|
54
|
+
},
|
55
|
+
verbose: true
|
37
56
|
}
|
38
57
|
config = LogStats::Config.default_config.
|
39
58
|
merge(custom_config).
|
40
59
|
merge(LogStats::Config.env_config)
|
41
60
|
|
42
|
-
log_file_data = ARGF.
|
61
|
+
log_file_data = LogStats::Logger.elapsed(config, "Reading file") { ARGF.readlines }
|
43
62
|
data = LogStats.get_data(log_file_data, config)
|
44
|
-
data[:requests] = data[:requests][:kpi] # Only output KPI from requests data
|
45
63
|
puts JSON.pretty_generate(data)
|
@@ -2,7 +2,8 @@ module LogStats
|
|
2
2
|
module LineParser
|
3
3
|
def self.parse(log_data, config)
|
4
4
|
data = {}
|
5
|
-
log_data.split("\n")
|
5
|
+
lines = (log_data.is_a?(String) ? log_data.split("\n") : log_data)
|
6
|
+
lines.each do |line_string|
|
6
7
|
config[:events].each do |event, event_config|
|
7
8
|
if event_config[:line_pattern] =~ line_string
|
8
9
|
data[event] ||= []
|
data/lib/log_stats/logger.rb
CHANGED
@@ -1,7 +1,21 @@
|
|
1
1
|
module LogStats
|
2
2
|
class Logger
|
3
3
|
def self.info(config, message)
|
4
|
-
|
4
|
+
return if !config[:verbose]
|
5
|
+
puts(message)
|
6
|
+
end
|
7
|
+
|
8
|
+
def self.elapsed(config, message)
|
9
|
+
if config[:verbose]
|
10
|
+
start_at = Time.now
|
11
|
+
print(message + '...')
|
12
|
+
result = yield
|
13
|
+
elapsed = ((Time.now - start_at) * 1000.0).round
|
14
|
+
puts(" #{elapsed} ms")
|
15
|
+
result
|
16
|
+
else
|
17
|
+
yield
|
18
|
+
end
|
5
19
|
end
|
6
20
|
end
|
7
21
|
end
|
data/lib/log_stats/version.rb
CHANGED
data/lib/log_stats.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require "json"
|
2
|
+
require "log_stats/logger"
|
2
3
|
require "log_stats/version"
|
3
4
|
require "log_stats/line_parser"
|
4
5
|
require "log_stats/logger"
|
@@ -14,14 +15,15 @@ module LogStats
|
|
14
15
|
Requests::TextOutput.print(data[:requests], request_config)
|
15
16
|
end
|
16
17
|
if config[:output_format] == "json"
|
17
|
-
puts JSON.
|
18
|
+
puts JSON.pretty_generate(data)
|
18
19
|
end
|
19
20
|
data
|
20
21
|
end
|
21
22
|
|
22
23
|
def self.get_data(log_data, config)
|
23
|
-
Logger.
|
24
|
-
|
24
|
+
events = Logger.elapsed(config, "\nParsing #{log_data.length} log lines") do
|
25
|
+
LineParser.parse(log_data, config)
|
26
|
+
end
|
25
27
|
result = {}
|
26
28
|
if requests = events[:requests]
|
27
29
|
result[:requests] = get_requests_data(requests, config)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log_stats
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Marklund
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-05-
|
11
|
+
date: 2017-05-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|