log_stats 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f874fbbcc589b39b6cd4b7146fa8f32bd4a57f1
-  data.tar.gz: a9b0fab518d53e1bfe2718d3d2986277fc1a394b
+  metadata.gz: a3f054eb3a7bbd2b720eca55eea18fce78ff20f8
+  data.tar.gz: a07e5505ee2287de50549855d4065e2b1c912191
 SHA512:
-  metadata.gz: b420cce2a9ffc8001403ba1213db7681a43ef56eeb00a1549f259165d61ac30722f09ccc4642838b05cb6b6d137366afed954f245aa8f97411b61f2c034f5727
-  data.tar.gz: 1727b1363af36c20fdd66b4e4ac30e6d2c866d26430a1a0526ea10b278c922d0e496f9dc0a2b3bbd0c2d241659770298899ccba01d1730bce735fd52536cf5ec
+  metadata.gz: 5838df03f26b19cdb7240237efcc8ecc2f41f049dae2d28a97ea18f02f8e7ed20050e68b4d585189d414a641c13b974875c3119e2353de8d6af8c98199d1f8a9
+  data.tar.gz: 1ccdcb9c71ade4b53d2662d70e291fac9aaf377c85d6dabbf888f5a39cf6c7581fb2e7f76ef85a0f06b2ed5696ff8b0238e341c4958e918837b0985501e4b7d1

data/README.md

@@ -10,17 +10,78 @@ gem install log_stats
 log_stats /tmp/papertrail.log
 ```
 
-To download longer timeperiods, like a whole day, download and gunzip a Papertrail log archive file.
+To download longer timeperiods, like a whole day, download and gunzip a Papertrail log archive file,
+see [examples/papertrail_download](examples/papertrail_download).
+
+You can extract not only web requests but any events you are interested in from your logs.
+Here is the API call stats output from the [examples/log_stats](examples/log_stats) script:
+
+```
+"requests": {
+  "request_count": 265325,
+  "response_time_avg": 694.01984,
+  "response_time_95p": 2647,
+  "apdex": 0.7886,
+  "error_rate": 0.00024,
+  "timeout_rate": 0.00017
+}
+"api_calls": {
+    "count": 410207,
+    "fields": {
+      "response_time": {
+        "min": 9,
+        "max": 17016,
+        "avg": 151.11675324896942,
+        "median": 136,
+        "percentiles": {
+          "0.05": 100,
+          "0.1": 104,
+          "0.15": 108,
+          "0.2": 111,
+          "0.25": 115,
+          "0.3": 119,
+          "0.35": 122,
+          "0.4": 127,
+          "0.45": 131,
+          "0.5": 136,
+          "0.55": 140,
+          "0.6": 145,
+          "0.65": 149,
+          "0.7": 154,
+          "0.75": 162,
+          "0.8": 172,
+          "0.85": 188,
+          "0.9": 211,
+          "0.95": 244,
+          "0.99": 394,
+          "0.999": 1116
+        },
+        "events": [
+          {
+            "time": "2017-02-27T14:49:12",
+            "url": "https://account.example.se/operators?client=web&country_code=se",
+            "method": "get",
+            "response_time": 17016
+          },
+          {
+            "time": "2017-02-27T18:56:48",
+            "url": "http://sumore02.example.se/api/tve_web/user",
+            "method": "get",
+            "response_time": 15164
+          },
+          ...
+```
 
 Example Heroku success log line:
 
 ```
-768004272804798492	2017-02-14T03:41:49	2017-02-14T03:41:49Z	505641143	cmore-web-prod	54.144.85.82	Local3	Info	heroku/router	at=info method=GET path="/filmer/med/glenn-erland-tosterud" host=www.cmore.se request_id=29cb0a66-23f9-4ef4-999a-65f8de089208 fwd="216.244.66.238,23.54.19.54" dyno=web.5 connect=0ms service=210ms status=200 bytes=52867
+768004272804798492	2017-02-14T03:41:49	2017-02-14T03:41:49Z	505641143	example-web-prod	54.144.85.82	Local3	Info	heroku/router	at=info method=GET path="/filmer/med/glenn-erland-tosterud" host=www.example.se request_id=29cb0a66-23f9-4ef4-999a-65f8de089208 fwd="216.244.66.238,23.54.19.54" dyno=web.5 connect=0ms service=210ms status=200 bytes=52867
 ```
- Example Heroku error log line:
+
+Example Heroku error log line:
 
 ```
-768277650920906757	2017-02-14T21:48:07	2017-02-14T21:48:08Z	505641143	cmore-web-prod	54.196.126.116	Local3	Info	heroku/router	at=error code=H12 desc="Request timeout" method=GET path="/serie/74763-alvinnn-og-gjengen-tv-serien/sesong-1/episode-2/3282212-alvinnn-og-gjengen-tv-serien-forelsket-i-rektor-norsk-tale" host=www.cmore.no request_id=f5c2921a-3974-4522-8fdf-d9bfff8b1db9 fwd="163.172.66.89,80.239.216.108" dyno=web.5 connect=0ms service=30001ms status=503 bytes=0
+768277650920906757	2017-02-14T21:48:07	2017-02-14T21:48:08Z	505641143	example-web-prod	54.196.126.116	Local3	Info	heroku/router	at=error code=H12 desc="Request timeout" method=GET path="/serie/74763-alvinnn-og-gjengen-tv-serien/sesong-1/episode-2/3282212-alvinnn-og-gjengen-tv-serien-forelsket-i-rektor-norsk-tale" host=www.example.no request_id=f5c2921a-3974-4522-8fdf-d9bfff8b1db9 fwd="163.172.66.89,80.239.216.108" dyno=web.5 connect=0ms service=30001ms status=503 bytes=0
 ```
 
 ## License

data/example/log_stats

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+#############################################################################
+# This example script shows custom parsing of API requests from a log file.
+#############################################################################
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require 'log_stats'
+require "log_stats/config"
+require "json"
+require "uri"
+
+parse_time = Proc.new { |line| line[/\b20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d/] }
+custom_config = {
+  events: LogStats::Config.default_config[:events].merge({
+    api_calls: {
+      # 2017-02-19T06:21:25.522274+00:00 app[worker.2]: [WARN] [Vac::Request] Slow response time for url=http://sumore02.cmore.dk/api/tve_web/search/categories/160145/assets/ method=get status=304 size= response_time=141
+      line_pattern: /\s\[Vac::Request\] Slow response time\s/,
+      fields: [
+        {name: :time, parse: parse_time},
+        {name: :url},
+        {name: :method},
+        {name: :response_time, numeric: true, events: true}
+      ],
+      group_by: {
+        hostname: {
+          id: Proc.new { |api_call| URI(api_call[:url]).host }
+        },
+        method: {
+          id: Proc.new { |api_call| URI(api_call[:method]) }
+        }
+      }
+    }
+  }),
+  verbose: false
+}
+config = LogStats::Config.default_config.
+          merge(custom_config).
+          merge(LogStats::Config.env_config)
+
+log_file_data = ARGF.read
+data = LogStats.get_data(log_file_data, config)
+data[:requests] = data[:requests][:kpi] # Only output KPI from requests data
+puts JSON.pretty_generate(data)

data/example/papertrail_download

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# Download papertrail log files. Usage:
+#
+# PAPERTRAIL_API_TOKEN=... examples/papertrail_download 2017-02-10
+
+date=${1:-`date -v -1d "+%Y-%m-%d"`} # would be --date "1 day ago" on Linux
+
+file=/tmp/papertrail-$date.tsv.gz
+url=https://papertrailapp.com/api/v1/archives/$date/download
+
+echo "Downloading $url to $file..."
+
+curl -silent --no-include -o $file -L -H "X-Papertrail-Token: $PAPERTRAIL_API_TOKEN" $url
+gunzip $file

data/lib/log_stats/config.rb

@@ -23,8 +23,8 @@ module LogStats
             apdex_goal: 0.9,
           }
         },
-        verbose: true,
-        stats_format: "text"
+        output_format: "text",
+        verbose: true
       }
     end
 
@@ -41,7 +41,7 @@ module LogStats
     def self.env_value(key)
       env_key = key.to_s.upcase
       value = ENV[env_key]
-      if !value.nil? && boolean?(default_config[key]) 
+      if !value.nil? && boolean?(default_config[key])
         ['1', true, 'true', 't', 'TRUE'].include?(value)
       else
         value

data/lib/log_stats/logger.rb

@@ -1,11 +1,7 @@
 module LogStats
   class Logger
-    def initialize(verbose)
-      @verbose = verbose
-    end
-
-    def info(message)
-      puts(message) if @verbose
+    def self.info(config, message)
+      puts(message) if config[:verbose]
     end
   end
 end

data/lib/log_stats/stats.rb

@@ -0,0 +1,72 @@
+module LogStats
+  module Stats
+    def self.fields(events, event_config)
+      event_config[:fields].reduce({}) do |acc, field|
+        if field[:numeric]
+          acc[field[:name]] = field_stats_numeric(events, field)
+        end
+        acc
+      end
+    end
+
+    def self.group_by(events, event_config)
+      event_config[:group_by].reduce({}) do |acc, (name, group_by)|
+        acc[name] = group_by_stats(events, group_by, event_config)
+        acc
+      end
+    end
+
+    def self.field_stats_numeric(events, field)
+      sorted_events = events.sort_by { |event| event[field[:name]] }
+      percentile_levels = (5..95).step(5).map { |n| n/100.0 } + [0.99, 0.999]
+      percentiles = percentile_levels.reduce({}) do |acc, level|
+        acc[level] = percentile(sorted_events, field[:name], level)
+        acc
+      end
+      result = {
+        min: sorted_events[0][field[:name]],
+        max: sorted_events[-1][field[:name]],
+        avg: avg(events, field[:name]),
+        median: percentile(sorted_events, field[:name], 0.5),
+        percentiles: percentiles
+      }
+      if field[:events]
+        events_options = (field[:events].is_a?(Hash) ? field[:events] : {})
+        events_limit = events_options[:limit] || 100
+        result[:events] = if events_options[:sort] == "asc"
+                            sorted_events[0, events_limit]
+                          else
+                            events_start_index = [0, sorted_events.size-events_limit].max
+                            sorted_events[events_start_index..-1].reverse
+                          end
+      end
+      result
+    end
+
+    def self.group_by_stats(events, group_by, event_config)
+      total_count = events.size
+      events_by_group = events.group_by { |event| group_by[:id].call(event) }.select { |key, _| !key.nil? }
+      events_by_group.reduce({}) do |acc, (key, group_events)|
+        group_count = group_events.size
+        percent = (group_count.to_f*100/total_count).round(4)
+        acc[key] = {
+          count: group_count,
+          percent: percent,
+          fields: fields(group_events, event_config)
+        }
+        acc
+      end
+    end
+
+    def self.avg(events, field_name)
+      sum = events.lazy.map { |event| event[field_name] }.reduce(&:+)
+      sum/events.size.to_f
+    end
+
+    def self.percentile(sorted_events, field_name, level)
+      index = (sorted_events.size*level).round - 1
+      event = sorted_events[index]
+      event && event[field_name]
+    end
+  end
+end

data/lib/log_stats/version.rb

@@ -1,3 +1,3 @@
 module LogStats
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/log_stats.rb

@@ -2,6 +2,7 @@ require "json"
 require "log_stats/version"
 require "log_stats/line_parser"
 require "log_stats/logger"
+require "log_stats/stats"
 require "log_stats/requests/stats"
 require "log_stats/requests/kpi"
 require "log_stats/requests/text_output"
@@ -9,38 +10,51 @@ require "log_stats/requests/text_output"
 module LogStats
   def self.run(log_data, config)
     data = get_data(log_data, config)
-    if config[:stats_format] == "text" && request_config = config[:events][:requests]
+    if config[:output_format] == "text" && request_config = config[:events][:requests]
       Requests::TextOutput.print(data[:requests], request_config)
     end
-    if config[:stats_format] == "json"
+    if config[:output_format] == "json"
       puts JSON.generate(data)
     end
     data
   end
 
   def self.get_data(log_data, config)
-    logger = Logger.new(config[:verbose])
-    logger.info("\nParsing request lines...")
-    data = LineParser.parse(log_data, config)
+    Logger.info(config, "\nParsing log lines...")
+    events = LineParser.parse(log_data, config)
     result = {}
-    if requests = data[:requests]
-      requests_count = requests.size
-      requests_config = config[:events][:requests]
-      logger.info("\nNumber of request lines: #{requests_count}")
-      logger.info("Start time: #{requests[0][:time]}")
-      logger.info("End time: #{requests[-1][:time]}")
-
-      logger.info("\nCalculating request stats...")
-      stats = Requests::Stats.stats(requests, requests_config)
-      result[:requests] = {
-        requests_count: requests_count,
-        requests: requests,
-        stats: stats,
-        requests_by_status: requests.group_by { |request| request[:status] },
-        requests_by_code: requests.group_by { |request| request[:code] },
-        kpi: Requests::KPI.calculate(requests, stats)
+    if requests = events[:requests]
+      result[:requests] = get_requests_data(requests, config)
+    end
+    other_event_names = events.keys.reject { |k| k == :requests }
+    other_result = other_event_names.reduce({}) do |acc, event_name|
+      acc[event_name] = {
+        count: events[event_name].size,
+        fields: Stats.fields(events[event_name], config[:events][event_name]),
+        group_by: Stats.group_by(events[event_name], config[:events][event_name]),
+        events: events[event_name]
       }
+      acc
     end
-    result
+    result.merge(other_result)
+  end
+
+  def self.get_requests_data(requests, config)
+    requests_count = requests.size
+    requests_config = config[:events][:requests]
+    Logger.info(config, "\nNumber of request lines: #{requests_count}")
+    Logger.info(config, "Start time: #{requests[0][:time]}")
+    Logger.info(config, "End time: #{requests[-1][:time]}")
+
+    Logger.info(config, "\nCalculating request stats...")
+    stats = Requests::Stats.stats(requests, requests_config)
+    {
+      requests_count: requests_count,
+      requests: requests,
+      stats: stats,
+      requests_by_status: requests.group_by { |request| request[:status] },
+      requests_by_code: requests.group_by { |request| request[:code] },
+      kpi: Requests::KPI.calculate(requests, stats)
+    }
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_stats
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Peter Marklund
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-18 00:00:00.000000000 Z
+date: 2017-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -67,6 +67,8 @@ files:
 - README.md
 - Rakefile
 - bin/log_stats
+- example/log_stats
+- example/papertrail_download
 - lib/log_stats.rb
 - lib/log_stats/config.rb
 - lib/log_stats/line_parser.rb
@@ -74,6 +76,7 @@ files:
 - lib/log_stats/requests/kpi.rb
 - lib/log_stats/requests/stats.rb
 - lib/log_stats/requests/text_output.rb
+- lib/log_stats/stats.rb
 - lib/log_stats/version.rb
 - log_stats.gemspec
 homepage: https://github.com/peter/log_stats