log_stats 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac20eb5f46472b2dda48aa2373f9d5af71f4d5af
-  data.tar.gz: 71261fdd773a0f54031583206c5bb8fc7dc82d9a
+  metadata.gz: a2d41ac522f442b77ac7626e31fb585bc79efde4
+  data.tar.gz: b1585a93945b1296d2c66a16dc2fb38d8b27317a
 SHA512:
-  metadata.gz: a5fc2a3f20921465919ef87608eaa6b4923cf7e8254c23b7c3905a266f6dd2a78e87bab40e3ff02a182bedbd19ee6f3e6f22f51ec89552a60532a0f3f0a7e629
-  data.tar.gz: a635c7008eb76703b903b49ab3874cc57a053e3348e6ab74595b21314ee98f40617a1adf54c237ee4fc9a9e07b76c6831627031f998b79ef26d104e913e2e734
+  metadata.gz: 6839efa1ef45c97ac75ba8db6c5c4179dd0fb5ac45ebebeafb33d06d478db366da0232c77a33b8138320f5c2697c001534ae77d346ab3d57f9be2059be6d80a1
+  data.tar.gz: da3dda45aba36b4b0aa58e59f709e423783c3abd08949a1d62cf85cc39312e6749f75297c1402ab6056b09b2d31835866efb3d0daf08700688c7d3dfdca5d414

data/example/log_stats

@@ -4,16 +4,32 @@
 # This example script shows custom parsing of API requests from a log file.
 #############################################################################
 
-$:.unshift(File.dirname(__FILE__) + '/../lib')
-
-require 'log_stats'
+require "log_stats"
 require "log_stats/config"
 require "json"
 require "uri"
 
 parse_time = Proc.new { |line| line[/\b20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d/] }
+
 custom_config = {
-  events: LogStats::Config.default_config[:events].merge({
+  events: {
+    requests: {
+      # NOTE: matches Heroku router lines. Also matches Papertrails slightly modified lines.
+      line_pattern: /\s(heroku\/router)|(heroku\[router\]:)\s/,
+      fields: [
+        {name: :time, parse: parse_time},
+        {name: :method},
+        {name: :host},
+        {name: :path},
+        {name: :status, numeric: true},
+        {name: :code, optional: true},
+        {name: :service, numeric: true}
+      ],
+      top_list_limit: 100,
+      apdex: {tolerating: 500, frustrated: 2000},
+      apdex_goal: 0.9,
+      stats: false # Skip listing stats per request path to keep output size manageable
+    },
     api_calls: {
       # 2017-02-19T06:21:25.522274+00:00 app[worker.2]: [WARN] [Vac::Request] Slow response time for url=http://sumore02.cmore.dk/api/tve_web/search/categories/160145/assets/ method=get status=304 size= response_time=141
       line_pattern: /\s\[Vac::Request\] Slow response time\s/,
@@ -26,20 +42,22 @@ custom_config = {
       group_by: {
         hostname: {
           id: Proc.new { |api_call| URI(api_call[:url]).host }
-        },
-        method: {
-          id: Proc.new { |api_call| URI(api_call[:method]) }
         }
-      }
+        # NOTE: Grouping by HTTP method doesn't really add much at the moment, so commenting out for now
+        # method: {
+        #   id: Proc.new { |api_call| URI(api_call[:method]) }
+        # }
+      },
+      events: false, # Skip listing all individual API calls to keep output size manageable
+      limit: 5
     }
-  }),
-  verbose: false
+  },
+  verbose: true
 }
 config = LogStats::Config.default_config.
           merge(custom_config).
           merge(LogStats::Config.env_config)
 
-log_file_data = ARGF.read
+log_file_data = LogStats::Logger.elapsed(config, "Reading file") { ARGF.readlines }
 data = LogStats.get_data(log_file_data, config)
-data[:requests] = data[:requests][:kpi] # Only output KPI from requests data
 puts JSON.pretty_generate(data)

data/lib/log_stats/line_parser.rb

@@ -2,7 +2,8 @@ module LogStats
   module LineParser
     def self.parse(log_data, config)
       data = {}
-      log_data.split("\n").each do |line_string|
+      lines = (log_data.is_a?(String) ? log_data.split("\n") : log_data)
+      lines.each do |line_string|
         config[:events].each do |event, event_config|
           if event_config[:line_pattern] =~ line_string
             data[event] ||= []

data/lib/log_stats/logger.rb

@@ -1,7 +1,21 @@
 module LogStats
   class Logger
     def self.info(config, message)
-      puts(message) if config[:verbose]
+      return if !config[:verbose]
+      puts(message)
+    end
+
+    def self.elapsed(config, message)
+      if config[:verbose]
+        start_at = Time.now
+        print(message + '...')
+        result = yield
+        elapsed = ((Time.now - start_at) * 1000.0).round
+        puts(" #{elapsed} ms")
+        result
+      else
+        yield
+      end
     end
   end
 end

data/lib/log_stats/version.rb

@@ -1,3 +1,3 @@
 module LogStats
-  VERSION = "0.4.2"
+  VERSION = "0.4.3"
 end

data/lib/log_stats.rb

@@ -1,4 +1,5 @@
 require "json"
+require "log_stats/logger"
 require "log_stats/version"
 require "log_stats/line_parser"
 require "log_stats/logger"
@@ -14,14 +15,15 @@ module LogStats
       Requests::TextOutput.print(data[:requests], request_config)
     end
     if config[:output_format] == "json"
-      puts JSON.generate(data)
+      puts JSON.pretty_generate(data)
     end
     data
   end
 
   def self.get_data(log_data, config)
-    Logger.info(config, "\nParsing log lines...")
-    events = LineParser.parse(log_data, config)
+    events = Logger.elapsed(config, "\nParsing #{log_data.length} log lines") do
+      LineParser.parse(log_data, config)
+    end
     result = {}
     if requests = events[:requests]
       result[:requests] = get_requests_data(requests, config)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: log_stats
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - Peter Marklund
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-05 00:00:00.000000000 Z
+date: 2017-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler