log_stats 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +65 -4
- data/example/log_stats +45 -0
- data/example/papertrail_download +15 -0
- data/lib/log_stats/config.rb +3 -3
- data/lib/log_stats/logger.rb +2 -6
- data/lib/log_stats/stats.rb +72 -0
- data/lib/log_stats/version.rb +1 -1
- data/lib/log_stats.rb +36 -22
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a3f054eb3a7bbd2b720eca55eea18fce78ff20f8
|
4
|
+
data.tar.gz: a07e5505ee2287de50549855d4065e2b1c912191
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5838df03f26b19cdb7240237efcc8ecc2f41f049dae2d28a97ea18f02f8e7ed20050e68b4d585189d414a641c13b974875c3119e2353de8d6af8c98199d1f8a9
|
7
|
+
data.tar.gz: 1ccdcb9c71ade4b53d2662d70e291fac9aaf377c85d6dabbf888f5a39cf6c7581fb2e7f76ef85a0f06b2ed5696ff8b0238e341c4958e918837b0985501e4b7d1
|
data/README.md
CHANGED
@@ -10,17 +10,78 @@ gem install log_stats
|
|
10
10
|
log_stats /tmp/papertrail.log
|
11
11
|
```
|
12
12
|
|
13
|
-
To download longer timeperiods, like a whole day, download and gunzip a Papertrail log archive file
|
13
|
+
To download longer timeperiods, like a whole day, download and gunzip a Papertrail log archive file,
|
14
|
+
see [examples/papertrail_download](examples/papertrail_download).
|
15
|
+
|
16
|
+
You can extract not only web requests but any events you are interested in from your logs.
|
17
|
+
Here is the API call stats output from the [examples/log_stats](examples/log_stats) script:
|
18
|
+
|
19
|
+
```
|
20
|
+
"requests": {
|
21
|
+
"request_count": 265325,
|
22
|
+
"response_time_avg": 694.01984,
|
23
|
+
"response_time_95p": 2647,
|
24
|
+
"apdex": 0.7886,
|
25
|
+
"error_rate": 0.00024,
|
26
|
+
"timeout_rate": 0.00017
|
27
|
+
}
|
28
|
+
"api_calls": {
|
29
|
+
"count": 410207,
|
30
|
+
"fields": {
|
31
|
+
"response_time": {
|
32
|
+
"min": 9,
|
33
|
+
"max": 17016,
|
34
|
+
"avg": 151.11675324896942,
|
35
|
+
"median": 136,
|
36
|
+
"percentiles": {
|
37
|
+
"0.05": 100,
|
38
|
+
"0.1": 104,
|
39
|
+
"0.15": 108,
|
40
|
+
"0.2": 111,
|
41
|
+
"0.25": 115,
|
42
|
+
"0.3": 119,
|
43
|
+
"0.35": 122,
|
44
|
+
"0.4": 127,
|
45
|
+
"0.45": 131,
|
46
|
+
"0.5": 136,
|
47
|
+
"0.55": 140,
|
48
|
+
"0.6": 145,
|
49
|
+
"0.65": 149,
|
50
|
+
"0.7": 154,
|
51
|
+
"0.75": 162,
|
52
|
+
"0.8": 172,
|
53
|
+
"0.85": 188,
|
54
|
+
"0.9": 211,
|
55
|
+
"0.95": 244,
|
56
|
+
"0.99": 394,
|
57
|
+
"0.999": 1116
|
58
|
+
},
|
59
|
+
"events": [
|
60
|
+
{
|
61
|
+
"time": "2017-02-27T14:49:12",
|
62
|
+
"url": "https://account.example.se/operators?client=web&country_code=se",
|
63
|
+
"method": "get",
|
64
|
+
"response_time": 17016
|
65
|
+
},
|
66
|
+
{
|
67
|
+
"time": "2017-02-27T18:56:48",
|
68
|
+
"url": "http://sumore02.example.se/api/tve_web/user",
|
69
|
+
"method": "get",
|
70
|
+
"response_time": 15164
|
71
|
+
},
|
72
|
+
...
|
73
|
+
```
|
14
74
|
|
15
75
|
Example Heroku success log line:
|
16
76
|
|
17
77
|
```
|
18
|
-
768004272804798492 2017-02-14T03:41:49 2017-02-14T03:41:49Z 505641143
|
78
|
+
768004272804798492 2017-02-14T03:41:49 2017-02-14T03:41:49Z 505641143 example-web-prod 54.144.85.82 Local3 Info heroku/router at=info method=GET path="/filmer/med/glenn-erland-tosterud" host=www.example.se request_id=29cb0a66-23f9-4ef4-999a-65f8de089208 fwd="216.244.66.238,23.54.19.54" dyno=web.5 connect=0ms service=210ms status=200 bytes=52867
|
19
79
|
```
|
20
|
-
|
80
|
+
|
81
|
+
Example Heroku error log line:
|
21
82
|
|
22
83
|
```
|
23
|
-
768277650920906757 2017-02-14T21:48:07 2017-02-14T21:48:08Z 505641143
|
84
|
+
768277650920906757 2017-02-14T21:48:07 2017-02-14T21:48:08Z 505641143 example-web-prod 54.196.126.116 Local3 Info heroku/router at=error code=H12 desc="Request timeout" method=GET path="/serie/74763-alvinnn-og-gjengen-tv-serien/sesong-1/episode-2/3282212-alvinnn-og-gjengen-tv-serien-forelsket-i-rektor-norsk-tale" host=www.example.no request_id=f5c2921a-3974-4522-8fdf-d9bfff8b1db9 fwd="163.172.66.89,80.239.216.108" dyno=web.5 connect=0ms service=30001ms status=503 bytes=0
|
24
85
|
```
|
25
86
|
|
26
87
|
## License
|
data/example/log_stats
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
#############################################################################
|
4
|
+
# This example script shows custom parsing of API requests from a log file.
|
5
|
+
#############################################################################
|
6
|
+
|
7
|
+
$:.unshift(File.dirname(__FILE__) + '/../lib')
|
8
|
+
|
9
|
+
require 'log_stats'
|
10
|
+
require "log_stats/config"
|
11
|
+
require "json"
|
12
|
+
require "uri"
|
13
|
+
|
14
|
+
parse_time = Proc.new { |line| line[/\b20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d/] }
|
15
|
+
custom_config = {
|
16
|
+
events: LogStats::Config.default_config[:events].merge({
|
17
|
+
api_calls: {
|
18
|
+
# 2017-02-19T06:21:25.522274+00:00 app[worker.2]: [WARN] [Vac::Request] Slow response time for url=http://sumore02.cmore.dk/api/tve_web/search/categories/160145/assets/ method=get status=304 size= response_time=141
|
19
|
+
line_pattern: /\s\[Vac::Request\] Slow response time\s/,
|
20
|
+
fields: [
|
21
|
+
{name: :time, parse: parse_time},
|
22
|
+
{name: :url},
|
23
|
+
{name: :method},
|
24
|
+
{name: :response_time, numeric: true, events: true}
|
25
|
+
],
|
26
|
+
group_by: {
|
27
|
+
hostname: {
|
28
|
+
id: Proc.new { |api_call| URI(api_call[:url]).host }
|
29
|
+
},
|
30
|
+
method: {
|
31
|
+
id: Proc.new { |api_call| URI(api_call[:method]) }
|
32
|
+
}
|
33
|
+
}
|
34
|
+
}
|
35
|
+
}),
|
36
|
+
verbose: false
|
37
|
+
}
|
38
|
+
config = LogStats::Config.default_config.
|
39
|
+
merge(custom_config).
|
40
|
+
merge(LogStats::Config.env_config)
|
41
|
+
|
42
|
+
log_file_data = ARGF.read
|
43
|
+
data = LogStats.get_data(log_file_data, config)
|
44
|
+
data[:requests] = data[:requests][:kpi] # Only output KPI from requests data
|
45
|
+
puts JSON.pretty_generate(data)
|
@@ -0,0 +1,15 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
#
|
3
|
+
# Download papertrail log files. Usage:
|
4
|
+
#
|
5
|
+
# PAPERTRAIL_API_TOKEN=... examples/papertrail_download 2017-02-10
|
6
|
+
|
7
|
+
date=${1:-`date -v -1d "+%Y-%m-%d"`} # would be --date "1 day ago" on Linux
|
8
|
+
|
9
|
+
file=/tmp/papertrail-$date.tsv.gz
|
10
|
+
url=https://papertrailapp.com/api/v1/archives/$date/download
|
11
|
+
|
12
|
+
echo "Downloading $url to $file..."
|
13
|
+
|
14
|
+
curl -silent --no-include -o $file -L -H "X-Papertrail-Token: $PAPERTRAIL_API_TOKEN" $url
|
15
|
+
gunzip $file
|
data/lib/log_stats/config.rb
CHANGED
@@ -23,8 +23,8 @@ module LogStats
|
|
23
23
|
apdex_goal: 0.9,
|
24
24
|
}
|
25
25
|
},
|
26
|
-
|
27
|
-
|
26
|
+
output_format: "text",
|
27
|
+
verbose: true
|
28
28
|
}
|
29
29
|
end
|
30
30
|
|
@@ -41,7 +41,7 @@ module LogStats
|
|
41
41
|
def self.env_value(key)
|
42
42
|
env_key = key.to_s.upcase
|
43
43
|
value = ENV[env_key]
|
44
|
-
if !value.nil? && boolean?(default_config[key])
|
44
|
+
if !value.nil? && boolean?(default_config[key])
|
45
45
|
['1', true, 'true', 't', 'TRUE'].include?(value)
|
46
46
|
else
|
47
47
|
value
|
data/lib/log_stats/logger.rb
CHANGED
@@ -0,0 +1,72 @@
|
|
1
|
+
module LogStats
|
2
|
+
module Stats
|
3
|
+
def self.fields(events, event_config)
|
4
|
+
event_config[:fields].reduce({}) do |acc, field|
|
5
|
+
if field[:numeric]
|
6
|
+
acc[field[:name]] = field_stats_numeric(events, field)
|
7
|
+
end
|
8
|
+
acc
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.group_by(events, event_config)
|
13
|
+
event_config[:group_by].reduce({}) do |acc, (name, group_by)|
|
14
|
+
acc[name] = group_by_stats(events, group_by, event_config)
|
15
|
+
acc
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.field_stats_numeric(events, field)
|
20
|
+
sorted_events = events.sort_by { |event| event[field[:name]] }
|
21
|
+
percentile_levels = (5..95).step(5).map { |n| n/100.0 } + [0.99, 0.999]
|
22
|
+
percentiles = percentile_levels.reduce({}) do |acc, level|
|
23
|
+
acc[level] = percentile(sorted_events, field[:name], level)
|
24
|
+
acc
|
25
|
+
end
|
26
|
+
result = {
|
27
|
+
min: sorted_events[0][field[:name]],
|
28
|
+
max: sorted_events[-1][field[:name]],
|
29
|
+
avg: avg(events, field[:name]),
|
30
|
+
median: percentile(sorted_events, field[:name], 0.5),
|
31
|
+
percentiles: percentiles
|
32
|
+
}
|
33
|
+
if field[:events]
|
34
|
+
events_options = (field[:events].is_a?(Hash) ? field[:events] : {})
|
35
|
+
events_limit = events_options[:limit] || 100
|
36
|
+
result[:events] = if events_options[:sort] == "asc"
|
37
|
+
sorted_events[0, events_limit]
|
38
|
+
else
|
39
|
+
events_start_index = [0, sorted_events.size-events_limit].max
|
40
|
+
sorted_events[events_start_index..-1].reverse
|
41
|
+
end
|
42
|
+
end
|
43
|
+
result
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.group_by_stats(events, group_by, event_config)
|
47
|
+
total_count = events.size
|
48
|
+
events_by_group = events.group_by { |event| group_by[:id].call(event) }.select { |key, _| !key.nil? }
|
49
|
+
events_by_group.reduce({}) do |acc, (key, group_events)|
|
50
|
+
group_count = group_events.size
|
51
|
+
percent = (group_count.to_f*100/total_count).round(4)
|
52
|
+
acc[key] = {
|
53
|
+
count: group_count,
|
54
|
+
percent: percent,
|
55
|
+
fields: fields(group_events, event_config)
|
56
|
+
}
|
57
|
+
acc
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def self.avg(events, field_name)
|
62
|
+
sum = events.lazy.map { |event| event[field_name] }.reduce(&:+)
|
63
|
+
sum/events.size.to_f
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.percentile(sorted_events, field_name, level)
|
67
|
+
index = (sorted_events.size*level).round - 1
|
68
|
+
event = sorted_events[index]
|
69
|
+
event && event[field_name]
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
data/lib/log_stats/version.rb
CHANGED
data/lib/log_stats.rb
CHANGED
@@ -2,6 +2,7 @@ require "json"
|
|
2
2
|
require "log_stats/version"
|
3
3
|
require "log_stats/line_parser"
|
4
4
|
require "log_stats/logger"
|
5
|
+
require "log_stats/stats"
|
5
6
|
require "log_stats/requests/stats"
|
6
7
|
require "log_stats/requests/kpi"
|
7
8
|
require "log_stats/requests/text_output"
|
@@ -9,38 +10,51 @@ require "log_stats/requests/text_output"
|
|
9
10
|
module LogStats
|
10
11
|
def self.run(log_data, config)
|
11
12
|
data = get_data(log_data, config)
|
12
|
-
if config[:
|
13
|
+
if config[:output_format] == "text" && request_config = config[:events][:requests]
|
13
14
|
Requests::TextOutput.print(data[:requests], request_config)
|
14
15
|
end
|
15
|
-
if config[:
|
16
|
+
if config[:output_format] == "json"
|
16
17
|
puts JSON.generate(data)
|
17
18
|
end
|
18
19
|
data
|
19
20
|
end
|
20
21
|
|
21
22
|
def self.get_data(log_data, config)
|
22
|
-
|
23
|
-
|
24
|
-
data = LineParser.parse(log_data, config)
|
23
|
+
Logger.info(config, "\nParsing log lines...")
|
24
|
+
events = LineParser.parse(log_data, config)
|
25
25
|
result = {}
|
26
|
-
if requests =
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
requests_count: requests_count,
|
37
|
-
requests: requests,
|
38
|
-
stats: stats,
|
39
|
-
requests_by_status: requests.group_by { |request| request[:status] },
|
40
|
-
requests_by_code: requests.group_by { |request| request[:code] },
|
41
|
-
kpi: Requests::KPI.calculate(requests, stats)
|
26
|
+
if requests = events[:requests]
|
27
|
+
result[:requests] = get_requests_data(requests, config)
|
28
|
+
end
|
29
|
+
other_event_names = events.keys.reject { |k| k == :requests }
|
30
|
+
other_result = other_event_names.reduce({}) do |acc, event_name|
|
31
|
+
acc[event_name] = {
|
32
|
+
count: events[event_name].size,
|
33
|
+
fields: Stats.fields(events[event_name], config[:events][event_name]),
|
34
|
+
group_by: Stats.group_by(events[event_name], config[:events][event_name]),
|
35
|
+
events: events[event_name]
|
42
36
|
}
|
37
|
+
acc
|
43
38
|
end
|
44
|
-
result
|
39
|
+
result.merge(other_result)
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.get_requests_data(requests, config)
|
43
|
+
requests_count = requests.size
|
44
|
+
requests_config = config[:events][:requests]
|
45
|
+
Logger.info(config, "\nNumber of request lines: #{requests_count}")
|
46
|
+
Logger.info(config, "Start time: #{requests[0][:time]}")
|
47
|
+
Logger.info(config, "End time: #{requests[-1][:time]}")
|
48
|
+
|
49
|
+
Logger.info(config, "\nCalculating request stats...")
|
50
|
+
stats = Requests::Stats.stats(requests, requests_config)
|
51
|
+
{
|
52
|
+
requests_count: requests_count,
|
53
|
+
requests: requests,
|
54
|
+
stats: stats,
|
55
|
+
requests_by_status: requests.group_by { |request| request[:status] },
|
56
|
+
requests_by_code: requests.group_by { |request| request[:code] },
|
57
|
+
kpi: Requests::KPI.calculate(requests, stats)
|
58
|
+
}
|
45
59
|
end
|
46
60
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: log_stats
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Peter Marklund
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-05-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -67,6 +67,8 @@ files:
|
|
67
67
|
- README.md
|
68
68
|
- Rakefile
|
69
69
|
- bin/log_stats
|
70
|
+
- example/log_stats
|
71
|
+
- example/papertrail_download
|
70
72
|
- lib/log_stats.rb
|
71
73
|
- lib/log_stats/config.rb
|
72
74
|
- lib/log_stats/line_parser.rb
|
@@ -74,6 +76,7 @@ files:
|
|
74
76
|
- lib/log_stats/requests/kpi.rb
|
75
77
|
- lib/log_stats/requests/stats.rb
|
76
78
|
- lib/log_stats/requests/text_output.rb
|
79
|
+
- lib/log_stats/stats.rb
|
77
80
|
- lib/log_stats/version.rb
|
78
81
|
- log_stats.gemspec
|
79
82
|
homepage: https://github.com/peter/log_stats
|