log_stats 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0f874fbbcc589b39b6cd4b7146fa8f32bd4a57f1
4
- data.tar.gz: a9b0fab518d53e1bfe2718d3d2986277fc1a394b
3
+ metadata.gz: a3f054eb3a7bbd2b720eca55eea18fce78ff20f8
4
+ data.tar.gz: a07e5505ee2287de50549855d4065e2b1c912191
5
5
  SHA512:
6
- metadata.gz: b420cce2a9ffc8001403ba1213db7681a43ef56eeb00a1549f259165d61ac30722f09ccc4642838b05cb6b6d137366afed954f245aa8f97411b61f2c034f5727
7
- data.tar.gz: 1727b1363af36c20fdd66b4e4ac30e6d2c866d26430a1a0526ea10b278c922d0e496f9dc0a2b3bbd0c2d241659770298899ccba01d1730bce735fd52536cf5ec
6
+ metadata.gz: 5838df03f26b19cdb7240237efcc8ecc2f41f049dae2d28a97ea18f02f8e7ed20050e68b4d585189d414a641c13b974875c3119e2353de8d6af8c98199d1f8a9
7
+ data.tar.gz: 1ccdcb9c71ade4b53d2662d70e291fac9aaf377c85d6dabbf888f5a39cf6c7581fb2e7f76ef85a0f06b2ed5696ff8b0238e341c4958e918837b0985501e4b7d1
data/README.md CHANGED
@@ -10,17 +10,78 @@ gem install log_stats
10
10
  log_stats /tmp/papertrail.log
11
11
  ```
12
12
 
13
- To download longer timeperiods, like a whole day, download and gunzip a Papertrail log archive file.
13
+ To download longer timeperiods, like a whole day, download and gunzip a Papertrail log archive file,
14
+ see [examples/papertrail_download](examples/papertrail_download).
15
+
16
+ You can extract not only web requests but any events you are interested in from your logs.
17
+ Here is the API call stats output from the [examples/log_stats](examples/log_stats) script:
18
+
19
+ ```
20
+ "requests": {
21
+ "request_count": 265325,
22
+ "response_time_avg": 694.01984,
23
+ "response_time_95p": 2647,
24
+ "apdex": 0.7886,
25
+ "error_rate": 0.00024,
26
+ "timeout_rate": 0.00017
27
+ }
28
+ "api_calls": {
29
+ "count": 410207,
30
+ "fields": {
31
+ "response_time": {
32
+ "min": 9,
33
+ "max": 17016,
34
+ "avg": 151.11675324896942,
35
+ "median": 136,
36
+ "percentiles": {
37
+ "0.05": 100,
38
+ "0.1": 104,
39
+ "0.15": 108,
40
+ "0.2": 111,
41
+ "0.25": 115,
42
+ "0.3": 119,
43
+ "0.35": 122,
44
+ "0.4": 127,
45
+ "0.45": 131,
46
+ "0.5": 136,
47
+ "0.55": 140,
48
+ "0.6": 145,
49
+ "0.65": 149,
50
+ "0.7": 154,
51
+ "0.75": 162,
52
+ "0.8": 172,
53
+ "0.85": 188,
54
+ "0.9": 211,
55
+ "0.95": 244,
56
+ "0.99": 394,
57
+ "0.999": 1116
58
+ },
59
+ "events": [
60
+ {
61
+ "time": "2017-02-27T14:49:12",
62
+ "url": "https://account.example.se/operators?client=web&country_code=se",
63
+ "method": "get",
64
+ "response_time": 17016
65
+ },
66
+ {
67
+ "time": "2017-02-27T18:56:48",
68
+ "url": "http://sumore02.example.se/api/tve_web/user",
69
+ "method": "get",
70
+ "response_time": 15164
71
+ },
72
+ ...
73
+ ```
14
74
 
15
75
  Example Heroku success log line:
16
76
 
17
77
  ```
18
- 768004272804798492 2017-02-14T03:41:49 2017-02-14T03:41:49Z 505641143 cmore-web-prod 54.144.85.82 Local3 Info heroku/router at=info method=GET path="/filmer/med/glenn-erland-tosterud" host=www.cmore.se request_id=29cb0a66-23f9-4ef4-999a-65f8de089208 fwd="216.244.66.238,23.54.19.54" dyno=web.5 connect=0ms service=210ms status=200 bytes=52867
78
+ 768004272804798492 2017-02-14T03:41:49 2017-02-14T03:41:49Z 505641143 example-web-prod 54.144.85.82 Local3 Info heroku/router at=info method=GET path="/filmer/med/glenn-erland-tosterud" host=www.example.se request_id=29cb0a66-23f9-4ef4-999a-65f8de089208 fwd="216.244.66.238,23.54.19.54" dyno=web.5 connect=0ms service=210ms status=200 bytes=52867
19
79
  ```
20
- Example Heroku error log line:
80
+
81
+ Example Heroku error log line:
21
82
 
22
83
  ```
23
- 768277650920906757 2017-02-14T21:48:07 2017-02-14T21:48:08Z 505641143 cmore-web-prod 54.196.126.116 Local3 Info heroku/router at=error code=H12 desc="Request timeout" method=GET path="/serie/74763-alvinnn-og-gjengen-tv-serien/sesong-1/episode-2/3282212-alvinnn-og-gjengen-tv-serien-forelsket-i-rektor-norsk-tale" host=www.cmore.no request_id=f5c2921a-3974-4522-8fdf-d9bfff8b1db9 fwd="163.172.66.89,80.239.216.108" dyno=web.5 connect=0ms service=30001ms status=503 bytes=0
84
+ 768277650920906757 2017-02-14T21:48:07 2017-02-14T21:48:08Z 505641143 example-web-prod 54.196.126.116 Local3 Info heroku/router at=error code=H12 desc="Request timeout" method=GET path="/serie/74763-alvinnn-og-gjengen-tv-serien/sesong-1/episode-2/3282212-alvinnn-og-gjengen-tv-serien-forelsket-i-rektor-norsk-tale" host=www.example.no request_id=f5c2921a-3974-4522-8fdf-d9bfff8b1db9 fwd="163.172.66.89,80.239.216.108" dyno=web.5 connect=0ms service=30001ms status=503 bytes=0
24
85
  ```
25
86
 
26
87
  ## License
data/example/log_stats ADDED
@@ -0,0 +1,45 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ #############################################################################
4
+ # This example script shows custom parsing of API requests from a log file.
5
+ #############################################################################
6
+
7
+ $:.unshift(File.dirname(__FILE__) + '/../lib')
8
+
9
+ require 'log_stats'
10
+ require "log_stats/config"
11
+ require "json"
12
+ require "uri"
13
+
14
+ parse_time = Proc.new { |line| line[/\b20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d/] }
15
+ custom_config = {
16
+ events: LogStats::Config.default_config[:events].merge({
17
+ api_calls: {
18
+ # 2017-02-19T06:21:25.522274+00:00 app[worker.2]: [WARN] [Vac::Request] Slow response time for url=http://sumore02.cmore.dk/api/tve_web/search/categories/160145/assets/ method=get status=304 size= response_time=141
19
+ line_pattern: /\s\[Vac::Request\] Slow response time\s/,
20
+ fields: [
21
+ {name: :time, parse: parse_time},
22
+ {name: :url},
23
+ {name: :method},
24
+ {name: :response_time, numeric: true, events: true}
25
+ ],
26
+ group_by: {
27
+ hostname: {
28
+ id: Proc.new { |api_call| URI(api_call[:url]).host }
29
+ },
30
+ method: {
31
+ id: Proc.new { |api_call| URI(api_call[:method]) }
32
+ }
33
+ }
34
+ }
35
+ }),
36
+ verbose: false
37
+ }
38
+ config = LogStats::Config.default_config.
39
+ merge(custom_config).
40
+ merge(LogStats::Config.env_config)
41
+
42
+ log_file_data = ARGF.read
43
+ data = LogStats.get_data(log_file_data, config)
44
+ data[:requests] = data[:requests][:kpi] # Only output KPI from requests data
45
+ puts JSON.pretty_generate(data)
@@ -0,0 +1,15 @@
1
+ #!/bin/bash
2
+ #
3
+ # Download papertrail log files. Usage:
4
+ #
5
+ # PAPERTRAIL_API_TOKEN=... examples/papertrail_download 2017-02-10
6
+
7
+ date=${1:-`date -v -1d "+%Y-%m-%d"`} # would be --date "1 day ago" on Linux
8
+
9
+ file=/tmp/papertrail-$date.tsv.gz
10
+ url=https://papertrailapp.com/api/v1/archives/$date/download
11
+
12
+ echo "Downloading $url to $file..."
13
+
14
+ curl -silent --no-include -o $file -L -H "X-Papertrail-Token: $PAPERTRAIL_API_TOKEN" $url
15
+ gunzip $file
@@ -23,8 +23,8 @@ module LogStats
23
23
  apdex_goal: 0.9,
24
24
  }
25
25
  },
26
- verbose: true,
27
- stats_format: "text"
26
+ output_format: "text",
27
+ verbose: true
28
28
  }
29
29
  end
30
30
 
@@ -41,7 +41,7 @@ module LogStats
41
41
  def self.env_value(key)
42
42
  env_key = key.to_s.upcase
43
43
  value = ENV[env_key]
44
- if !value.nil? && boolean?(default_config[key])
44
+ if !value.nil? && boolean?(default_config[key])
45
45
  ['1', true, 'true', 't', 'TRUE'].include?(value)
46
46
  else
47
47
  value
@@ -1,11 +1,7 @@
1
1
  module LogStats
2
2
  class Logger
3
- def initialize(verbose)
4
- @verbose = verbose
5
- end
6
-
7
- def info(message)
8
- puts(message) if @verbose
3
+ def self.info(config, message)
4
+ puts(message) if config[:verbose]
9
5
  end
10
6
  end
11
7
  end
@@ -0,0 +1,72 @@
1
+ module LogStats
2
+ module Stats
3
+ def self.fields(events, event_config)
4
+ event_config[:fields].reduce({}) do |acc, field|
5
+ if field[:numeric]
6
+ acc[field[:name]] = field_stats_numeric(events, field)
7
+ end
8
+ acc
9
+ end
10
+ end
11
+
12
+ def self.group_by(events, event_config)
13
+ event_config[:group_by].reduce({}) do |acc, (name, group_by)|
14
+ acc[name] = group_by_stats(events, group_by, event_config)
15
+ acc
16
+ end
17
+ end
18
+
19
+ def self.field_stats_numeric(events, field)
20
+ sorted_events = events.sort_by { |event| event[field[:name]] }
21
+ percentile_levels = (5..95).step(5).map { |n| n/100.0 } + [0.99, 0.999]
22
+ percentiles = percentile_levels.reduce({}) do |acc, level|
23
+ acc[level] = percentile(sorted_events, field[:name], level)
24
+ acc
25
+ end
26
+ result = {
27
+ min: sorted_events[0][field[:name]],
28
+ max: sorted_events[-1][field[:name]],
29
+ avg: avg(events, field[:name]),
30
+ median: percentile(sorted_events, field[:name], 0.5),
31
+ percentiles: percentiles
32
+ }
33
+ if field[:events]
34
+ events_options = (field[:events].is_a?(Hash) ? field[:events] : {})
35
+ events_limit = events_options[:limit] || 100
36
+ result[:events] = if events_options[:sort] == "asc"
37
+ sorted_events[0, events_limit]
38
+ else
39
+ events_start_index = [0, sorted_events.size-events_limit].max
40
+ sorted_events[events_start_index..-1].reverse
41
+ end
42
+ end
43
+ result
44
+ end
45
+
46
+ def self.group_by_stats(events, group_by, event_config)
47
+ total_count = events.size
48
+ events_by_group = events.group_by { |event| group_by[:id].call(event) }.select { |key, _| !key.nil? }
49
+ events_by_group.reduce({}) do |acc, (key, group_events)|
50
+ group_count = group_events.size
51
+ percent = (group_count.to_f*100/total_count).round(4)
52
+ acc[key] = {
53
+ count: group_count,
54
+ percent: percent,
55
+ fields: fields(group_events, event_config)
56
+ }
57
+ acc
58
+ end
59
+ end
60
+
61
+ def self.avg(events, field_name)
62
+ sum = events.lazy.map { |event| event[field_name] }.reduce(&:+)
63
+ sum/events.size.to_f
64
+ end
65
+
66
+ def self.percentile(sorted_events, field_name, level)
67
+ index = (sorted_events.size*level).round - 1
68
+ event = sorted_events[index]
69
+ event && event[field_name]
70
+ end
71
+ end
72
+ end
@@ -1,3 +1,3 @@
1
1
  module LogStats
2
- VERSION = "0.3.0"
2
+ VERSION = "0.4.0"
3
3
  end
data/lib/log_stats.rb CHANGED
@@ -2,6 +2,7 @@ require "json"
2
2
  require "log_stats/version"
3
3
  require "log_stats/line_parser"
4
4
  require "log_stats/logger"
5
+ require "log_stats/stats"
5
6
  require "log_stats/requests/stats"
6
7
  require "log_stats/requests/kpi"
7
8
  require "log_stats/requests/text_output"
@@ -9,38 +10,51 @@ require "log_stats/requests/text_output"
9
10
  module LogStats
10
11
  def self.run(log_data, config)
11
12
  data = get_data(log_data, config)
12
- if config[:stats_format] == "text" && request_config = config[:events][:requests]
13
+ if config[:output_format] == "text" && request_config = config[:events][:requests]
13
14
  Requests::TextOutput.print(data[:requests], request_config)
14
15
  end
15
- if config[:stats_format] == "json"
16
+ if config[:output_format] == "json"
16
17
  puts JSON.generate(data)
17
18
  end
18
19
  data
19
20
  end
20
21
 
21
22
  def self.get_data(log_data, config)
22
- logger = Logger.new(config[:verbose])
23
- logger.info("\nParsing request lines...")
24
- data = LineParser.parse(log_data, config)
23
+ Logger.info(config, "\nParsing log lines...")
24
+ events = LineParser.parse(log_data, config)
25
25
  result = {}
26
- if requests = data[:requests]
27
- requests_count = requests.size
28
- requests_config = config[:events][:requests]
29
- logger.info("\nNumber of request lines: #{requests_count}")
30
- logger.info("Start time: #{requests[0][:time]}")
31
- logger.info("End time: #{requests[-1][:time]}")
32
-
33
- logger.info("\nCalculating request stats...")
34
- stats = Requests::Stats.stats(requests, requests_config)
35
- result[:requests] = {
36
- requests_count: requests_count,
37
- requests: requests,
38
- stats: stats,
39
- requests_by_status: requests.group_by { |request| request[:status] },
40
- requests_by_code: requests.group_by { |request| request[:code] },
41
- kpi: Requests::KPI.calculate(requests, stats)
26
+ if requests = events[:requests]
27
+ result[:requests] = get_requests_data(requests, config)
28
+ end
29
+ other_event_names = events.keys.reject { |k| k == :requests }
30
+ other_result = other_event_names.reduce({}) do |acc, event_name|
31
+ acc[event_name] = {
32
+ count: events[event_name].size,
33
+ fields: Stats.fields(events[event_name], config[:events][event_name]),
34
+ group_by: Stats.group_by(events[event_name], config[:events][event_name]),
35
+ events: events[event_name]
42
36
  }
37
+ acc
43
38
  end
44
- result
39
+ result.merge(other_result)
40
+ end
41
+
42
+ def self.get_requests_data(requests, config)
43
+ requests_count = requests.size
44
+ requests_config = config[:events][:requests]
45
+ Logger.info(config, "\nNumber of request lines: #{requests_count}")
46
+ Logger.info(config, "Start time: #{requests[0][:time]}")
47
+ Logger.info(config, "End time: #{requests[-1][:time]}")
48
+
49
+ Logger.info(config, "\nCalculating request stats...")
50
+ stats = Requests::Stats.stats(requests, requests_config)
51
+ {
52
+ requests_count: requests_count,
53
+ requests: requests,
54
+ stats: stats,
55
+ requests_by_status: requests.group_by { |request| request[:status] },
56
+ requests_by_code: requests.group_by { |request| request[:code] },
57
+ kpi: Requests::KPI.calculate(requests, stats)
58
+ }
45
59
  end
46
60
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: log_stats
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Peter Marklund
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-02-18 00:00:00.000000000 Z
11
+ date: 2017-05-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -67,6 +67,8 @@ files:
67
67
  - README.md
68
68
  - Rakefile
69
69
  - bin/log_stats
70
+ - example/log_stats
71
+ - example/papertrail_download
70
72
  - lib/log_stats.rb
71
73
  - lib/log_stats/config.rb
72
74
  - lib/log_stats/line_parser.rb
@@ -74,6 +76,7 @@ files:
74
76
  - lib/log_stats/requests/kpi.rb
75
77
  - lib/log_stats/requests/stats.rb
76
78
  - lib/log_stats/requests/text_output.rb
79
+ - lib/log_stats/stats.rb
77
80
  - lib/log_stats/version.rb
78
81
  - log_stats.gemspec
79
82
  homepage: https://github.com/peter/log_stats