alblogs 0.1.2 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d2aea6572a238675e0499c04e26341ea3cd3b8860b94c51095fe9c00cb4d806
-  data.tar.gz: 45a86b3b527c47bee978d0139c473c3aa491a1bddea41d2d10135093cc0cde4c
+  metadata.gz: 562382389696a749623d650ff1fa57aca2fea204b5140118172302fc078335eb
+  data.tar.gz: 1c4375cb342b53a396c42e09d500766fb308c57e334da6c62385b8d5595c02fa
 SHA512:
-  metadata.gz: 141b1919b75fc436dcb627fb93c20fd1312e57aa4d2b1525d1c7180f8e4921dba8c5af93614392f93162f9006756d4eba04a896f1ec18511033d4db7bdc7ae40
-  data.tar.gz: cc2c69a05b388d603c7e40cdf29759f57090b6894a4f9a001d1157a57a03692b419963e1f086efa6a87a35175ba4797b1a3121ea779c71a6f48bb92a46798f84
+  metadata.gz: 16115dc87afaea53aa9edb4b7f0b4adeaa7245c671dfb62667ce2a9c098fb13c5d13144543ca66327ce5a60d74257f002a093bb88dbc6a44aa09d94c4be2f146
+  data.tar.gz: a97e4761ade5678fb5b4333c87f88233a2924c02671f57c77700e5c09702814ed65be8b6325f6d7d5ba63f0dca61fffbc4e2e4770e711f78d8d420d0b24d6415

data/alblogs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'alblogs'
-  s.version     = '0.1.2'
+  s.version     = '0.1.7'
   s.summary     = 'ALB access log processing'
   s.description = 'Utility script for processing ALB access logs over a given time range'
   s.authors     = ['Doug Youch']

data/bin/alblogpp

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+# Purpose: to pretty print ALB log entries
+
+require 'optparse'
+require 'alblogs'
+require 'json'
+require 'csv'
+
+options = {
+  format: 'vertical'
+}
+OptionParser.new do |opts|
+  opts.banner = "Usage: alblogpp [options]"
+
+  opts.on("-f", "--format=FORMAT", "Output format vertical, json, csv, jsonl") do |v|
+    options[:format] = v
+  end
+end.parse!
+
+if options[:format] == 'csv'
+  print Alblogs::Entry.fields.to_csv
+end
+
+Alblogs::Entry.each_entry($stdin) do |entry|
+  case options[:format]
+  when 'csv'
+    print entry.to_a.to_csv
+  when 'json'
+    print JSON.pretty_generate(entry.to_h) + "\n"
+  when 'jsonl'
+    print entry.to_h.to_json + "\n"
+  else
+    entry.to_h.each do |key, value|
+      puts "#{key}: #{value}"
+    end
+  end
+end

data/bin/alblogs

@@ -1,9 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'optparse'
-require 'time'
-require 'shellwords'
-require 'json'
 require 'alblogs'
 
 started_at = Time.now.utc

data/bin/alblogs-analyze

@@ -0,0 +1,140 @@
+#!/usr/bin/env ruby
+
+require 'alblogs'
+
+LOG_FILE = ARGV.shift || raise('no log file specified')
+raise("#{LOG_FILE} not found") unless File.exist?(LOG_FILE)
+
+class RequestStat
+  attr_reader :method,
+              :path,
+              :num_requests,
+              :num_failed_requests,
+              :num_successful_requests,
+              :successful_requests_total_time,
+              :total_request_time,
+              :max_request_time,
+              :num_requests_over_a_second,
+              :response_codes,
+              :response_times
+
+  def initialize(method, path)
+    @method = method
+    @path = path
+    reset
+  end
+
+  def average_response_time
+    @total_request_time / (@num_requests - @num_failed_requests)
+  end
+
+  def average_successful_response_time
+    @num_successful_requests == 0 ? -1 : @successful_requests_total_time / @num_successful_requests
+  end
+
+  def num_500s
+    cnt = 0
+    @response_codes.each do |code, num_requests|
+      cnt += num_requests if code >= 500
+    end
+    cnt
+  end
+
+  def percentage_of_500s
+    num_500s.to_f / @num_requests.to_f
+  end
+
+  def update(entry)
+    @num_requests += 1
+
+    if (request_time = entry.target_processing_time)
+      if entry.elb_status_code >= 200 && entry.elb_status_code < 300
+        @num_successful_requests += 1
+        @successful_requests_total_time += request_time
+      end
+
+      @total_request_time += request_time
+      @max_request_time = request_time if request_time > @max_request_time
+      @num_requests_over_a_second += 1 if request_time >= 1.0
+
+      # break down requests times in buckets of 5 second intervals
+      if request_time < 1.0
+        @response_times['under_a_second'] += 1
+      elsif request_time < 2.0
+        @response_times['under_2_seconds'] += 1
+      elsif request_time < 5.0
+        @response_times['under_5_seconds'] += 1
+      elsif request_time < 10.0
+        @response_times['under_10_seconds'] += 1
+      else
+        @response_times['over_10_seconds'] += 1
+      end
+
+    else
+      @num_failed_requests += 1
+    end
+
+    @response_codes[entry.elb_status_code] += 1
+  end
+
+  def reset
+    @num_requests = 0
+    @num_failed_requests = 0
+    @total_request_time = 0.0
+    @max_request_time = 0.0
+    @num_requests_over_a_second = 0
+    @num_successful_requests = 0
+    @successful_requests_total_time = 0.0
+    @response_codes = Hash.new(0)
+    @response_times = Hash.new(0)
+  end
+end
+
+def normalize_path(entry)
+  entry
+    .request_uri
+    .path
+    .gsub(/(remote_id|external)\/[^\/]+/, '\1/:id')
+    .gsub(/\/\d+/, '/:id')
+end
+
+def generate_stat_key(entry)
+  "#{entry.request_method} #{normalize_path(entry)}"
+end
+
+def get_stats
+  stats = {}
+
+  File.open(LOG_FILE, 'rb') do |f|
+    Alblogs::Entry.each_entry(f) do |entry|
+      next if entry.request_method == 'OPTIONS'
+      stat = (stats[generate_stat_key(entry)] ||= RequestStat.new(entry.request_method, normalize_path(entry)))
+      stat.update(entry)
+    end
+  end
+
+  stats
+end
+
+def top_x_request_that_take_over_a_second(stats, top=20)
+  stats.values.sort { |a, b| b.num_requests_over_a_second <=> a.num_requests_over_a_second }[0, top]
+end
+
+def top_x_requested_pages(stats, top=20)
+  stats.values.sort { |a, b| b.num_requests <=> a.num_requests }[0, top]
+end
+
+def top_x_requests_that_take_over(stats, over='over_10_seconds', top=20)
+  stats.values.sort { |a, b| b.response_times[over].to_i <=> a.response_times[over].to_i }[0, top]
+end
+
+def top_x_requests_that_have_500s(stats, top=20)
+  stats.values.sort { |a, b| b.num_500s <=> a.num_500s }[0, top]
+end
+
+def top_x_worst_routes(stats, top=20)
+  stats.values.sort { |a, b| b.percentage_of_500s <=> a.percentage_of_500s }[0, top]
+end
+
+require 'irb'
+IRB.start

data/lib/alblogs/entry.rb

@@ -2,12 +2,55 @@ module Alblogs
   class Entry < Struct.new(:line, *::Alblogs::FIELDS.keys)
     REGEXP = Regexp.new(::Alblogs::FIELDS.values.join(' '))
 
+    def self.fields
+      ::Alblogs::FIELDS.keys
+    end
+
+    def request_parts
+      @request_parts ||= request.split(' ', 3)
+    end
+
+    def request_method
+      request_parts[0]
+    end
+
+    def request_url
+      request_parts[1]
+    end
+
+    def request_protocol
+      request_parts[2]
+    end
+
+    def request_uri
+      @request_uri ||= URI(request_url)
+    end
+
     def timestamp
       @timestamp ||= Time.iso8601(self[:timestamp])
     end
 
+    def to_a
+      ::Alblogs::FIELDS.keys.map { |k| send(k) }
+    end
+
+    def to_h
+      Hash[::Alblogs::FIELDS.keys.zip(to_a)]
+    end
+
+    # target_processing_time is in seconds with with millisecond precision
+    # -1 means target or server close connection or timed out
     def target_processing_time
-      self[:target_processing_time].to_f
+      self[:target_processing_time] == '-1' ? nil : self[:target_processing_time].to_f
+    end
+
+    # this is the response code the client received
+    def elb_status_code
+      self[:elb_status_code].to_i
+    end
+
+    def target_status_code
+      self[:target_status_code] == '-' ? nil : self[:target_status_code].to_i
     end
 
     def self.from_line(line)
@@ -17,5 +60,11 @@ module Alblogs
     def self.get_fields(line)
       REGEXP.match(line).to_a
     end
+
+    def self.each_entry(io)
+      while !io.eof?
+        yield from_line(io.readline)
+      end
+    end
   end
 end

data/lib/alblogs/iterator.rb

@@ -31,10 +31,10 @@ module Alblogs
 
         stats[:total_file_processing_time] += measure do
           File.open(tmp_file, 'rb') do |f|
-            while(! f.eof? && ! @stop)
-              stats[:lines] += 1
-              line = f.readline
-              entry = ::Alblogs::Entry.from_line(line)
+            ::Alblogs::Entry.each_entry(f) do |entry|
+              break if @stop
+
+              stats[:entries] += 1
               stats[:min_log_time] = ! stats[:min_log_time] || stats[:min_log_time] > entry.timestamp ? entry.timestamp : stats[:min_log_time]
               stats[:max_log_time] = ! stats[:max_log_time] || stats[:max_log_time] < entry.timestamp ? entry.timestamp : stats[:max_log_time]
               next if request_matcher && !request_matcher.match?(entry)

data/lib/alblogs.rb

@@ -1,3 +1,6 @@
+require 'time'
+require 'shellwords'
+
 module Alblogs
   autoload :Entry, 'alblogs/entry'
   autoload :Iterator, 'alblogs/iterator'

metadata

@@ -1,19 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: alblogs
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.7
 platform: ruby
 authors:
 - Doug Youch
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-08 00:00:00.000000000 Z
+date: 2023-09-01 00:00:00.000000000 Z
 dependencies: []
 description: Utility script for processing ALB access logs over a given time range
 email: dougyouch@gmail.com
 executables:
+- alblogpp
 - alblogs
+- alblogs-analyze
 extensions: []
 extra_rdoc_files: []
 files:
@@ -22,7 +24,9 @@ files:
 - ".ruby-version"
 - README.md
 - alblogs.gemspec
+- bin/alblogpp
 - bin/alblogs
+- bin/alblogs-analyze
 - lib/alblogs.rb
 - lib/alblogs/entry.rb
 - lib/alblogs/iterator.rb
@@ -34,7 +38,7 @@ files:
 homepage: https://github.com/dougyouch/alblogs
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -49,8 +53,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.0.9
+signing_key:
 specification_version: 4
 summary: ALB access log processing
 test_files: []