alblogs 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40a4878aed41640c3d06e5734de49c18c96121fecb4bfb981b360aae7c19e70a
-  data.tar.gz: 9150dbb3f1d1fb9085c75431c8756d2d318a36b49b8005179fab72baccc7f71f
+  metadata.gz: eefbd8b8cac015fc5eee191b6e8a73ff9fde3c465f8d1410b76774e686aa9623
+  data.tar.gz: 3dfce14461c67824dd0ad6c5bd1ecee0081767de256191bda00cad1a99c18d1f
 SHA512:
-  metadata.gz: a35fc4c33ceeb1be95779ec968f4d3fb74378a4f7978b9f19fe9bf9969ceac662a49feb8038823a09661214ad15b00459053b21ce6230ea0808dc41d68202ba1
-  data.tar.gz: ab6d34862e7a22a125fe323530ebfc058ec33d69e5ff65d7ca874a98e781853880749c1e4b56c248b82e71c88d3c874baed2b2425223b8330c737711fd84d1d5
+  metadata.gz: 9b17f53979c6c56d8909d62e1fcacf468edcd0294fc41928cd62a706d5d206fd8495e16453c339ea5dfd236aef9158c8ad9cfcb00522cc961e18ce82e98f7a4e
+  data.tar.gz: d5d792b11d17f82e30a58901f18ac7a1cdf1e3cea5d07201645475a63a255a06769f64772c947653f6ab2533657c23c84bdd2db4517382d484389e506f07079f

data/.gitignore

@@ -50,3 +50,4 @@ build-iPhoneSimulator/
 .rvmrc
 
 *~
+*.log

data/.ruby-gemset

@@ -0,0 +1 @@
+alblogs

data/.ruby-version

@@ -0,0 +1 @@
+2.6.3

data/README.md

@@ -2,6 +2,16 @@
 
 Utility script for processing ALB access logs over a given time range
 
+### Requirements
+
+Need to have the AWS CLI installed.  Can be found here https://aws.amazon.com/cli/
+
+### Install
+
+```
+gem install alblogs
+```
+
 ### Usage
 
 ```
@@ -25,3 +35,8 @@ Find all requests that took over 500ms to process in the last 12 hours.
 alblogs -b 's3://<my-aws-alb-bucket-name>/access_logs/AWSLogs/<aws-account-id>/elasticloadbalancing/<aws-region>' -s '12 hours' -o slow-requests.log --request-times-over 0.5
 ```
 
+### References
+
+AWS Documentaion: Access Logs for Your Application Load Balancer
+
+https://docs.aws.amazon.com/elasticloadbalancing/latest/application/load-balancer-access-logs.html

data/alblogs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'alblogs'
-  s.version     = '0.0.1'
+  s.version     = '0.1.0'
   s.summary     = 'ALB access log processing'
   s.description = 'Utility script for processing ALB access logs over a given time range'
   s.authors     = ['Doug Youch']

data/bin/alblogs

@@ -4,126 +4,7 @@ require 'optparse'
 require 'time'
 require 'shellwords'
 require 'json'
-
-def run_or_die(cmd)
-  res = `#{cmd}`
-  raise("command failed with #{$?}, #{cmd}") unless $?.success?
-  res
-end
-
-def parse_time_offset(str)
-  if str =~ /min/
-    str.sub(/ *min.*/, '').to_i * 60
-  elsif str =~ /hour/
-    str.sub(/ *hour.*/, '').to_i * 3600
-  elsif str =~ /day/
-    str.sub(/ *day.*/, '').to_i * 86400
-  else
-    nil
-  end
-end
-
-def time_ago(now, str)
-  if offset = parse_time_offset(str)
-    time = now - offset
-    time - (time.to_i % 60) # round to the start of the minute
-  else
-    Time.parse(str).utc
-  end
-end
-
-def get_s3_files(bucket, date_path, profile)
-  s3_url = "#{bucket}/#{date_path}/"
-  cmd = "aws"
-  cmd << " --profile #{Shellwords.escape(profile)}" if profile
-  cmd << " s3 ls #{Shellwords.escape(s3_url)}"
-  output = run_or_die(cmd)
-  output.split("\n").map do |line|
-    line =~ /(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}) +(\d+) +(.+)/
-    last_modified_at = Time.parse($1).utc
-    file_size = $2.to_i
-    file = $3
-    S3File.new("#{s3_url}#{file}", file_size, last_modified_at)
-  end
-end
-
-def get_s3_files_in_range(range, bucket, profile)
-  s3_files = {}
-  time = range.begin
-  while time < range.end
-    date_path = time.strftime('%Y/%m/%d')
-    get_s3_files(bucket, date_path, profile).each do |s3_file|
-      next unless s3_file.in_range?(range)
-      s3_files[s3_file.file] ||= s3_file
-    end
-    time += 86_400
-  end
-  s3_files
-end
-
-def download_s3_file(s3_file, dest, profile)
-  cmd = "aws"
-  cmd << " --profile #{Shellwords.escape(profile)}" if profile
-  cmd << " s3 cp #{Shellwords.escape(s3_file.file)} #{Shellwords.escape(dest)}.gz"
-  run_or_die(cmd)
-  cmd = "gzip -f -d #{Shellwords.escape(dest)}.gz"
-  run_or_die(cmd)
-end
-
-def alb_log_fields
-  @alb_log_fields ||=
-    begin
-      not_a_space = '([^ ]+)'
-      in_quotes = '"(.*?)"'
-      
-      {
-        type: not_a_space,
-        timestamp: not_a_space,
-        elb: not_a_space,
-        client_port: not_a_space,
-        target_port: not_a_space,
-        request_processing_time: not_a_space,
-        target_processing_time: not_a_space,
-        response_processing_time: not_a_space,
-        elb_status_code: not_a_space,
-        target_status_code: not_a_space,
-        received_bytes: not_a_space,
-        sent_bytes: not_a_space,
-        request: in_quotes,
-        user_agent: in_quotes,
-        ssl_cipher: not_a_space,
-        ssl_protocol: not_a_space,
-        target_group_arn: not_a_space,
-        trace_id: in_quotes,
-        domain_name: in_quotes,
-        chosen_cert_arn: in_quotes,
-        matched_rule_priority: not_a_space,
-        request_creation_time: not_a_space,
-        actions_executed: in_quotes,
-        redirect_url: in_quotes,
-        error_reason: in_quotes
-      }
-    end
-end
-
-def alb_log_fields_regex
-  @alb_log_fields_regex ||=
-    begin
-      Regexp.new alb_log_fields.values.join(' ')
-    end
-end
-
-def get_alb_log_fields(line)
-  matches  = alb_log_fields_regex.match(line).to_a
-  matches.shift
-  matches
-end
-
-def get_alb_log_entry(line)
-  entry = AlbLogEntry.new(*get_alb_log_fields(line))
-  entry.line = line
-  entry
-end
+require 'alblogs'
 
 def measure 
   start = Time.now
@@ -136,75 +17,10 @@ def display_stats(stats)
   $stderr.puts stats.inspect
 end
 
-class S3File
-  MINUTES_5 = 5 * 60
-
-  attr_reader :file,
-              :file_size,
-              :last_modified_at
-
-  def initialize(file, file_size, last_modified_at)
-    @file = file
-    @file_size = file_size
-    @last_modified_at = last_modified_at
-  end
-
-  def end_time
-    @end_time ||=
-      begin
-        unless @file =~ /_(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})Z_/
-          raise("unable to find time stamp in #{@file}")
-        end
-        Time.new($1, $2, $3, $4, $5, 0, 0)
-      end
-  end
-
-  def start_time
-    @start_time ||= (end_time - MINUTES_5)
-  end
-
-  def in_range?(range)
-    return false if end_time < range.begin
-    return false if start_time > range.end
-    true
-  end
-end
-
-class AlbLogEntry < Struct.new(*alb_log_fields.keys)
-  attr_accessor :line
-
-  def timestamp
-    @timestamp ||= Time.iso8601(self[:timestamp])
-  end
-
-  def target_processing_time
-    @target_processing_time ||= self[:target_processing_time].to_f
-  end
-end
-
-class RequestMatcher
-  attr_reader :range
-
-  def initialize(options)
-    @range = options[:start_time]..options[:end_time]
-    @exclude_filter = options[:exclude_filter]
-    @include_filter = options[:include_filter]
-    @request_times_over = options[:request_times_over]
-  end
-
-  def match?(entry)
-    return false unless @range.cover?(entry.timestamp)
-    return false if @include_filter && ! @include_filter.match?(entry.line)
-    return false if @exclude_filter && @exclude_filter.match?(entry.line)
-    return false if @request_times_over && @request_times_over > entry.target_processing_time
-    true
-  end
-end
-
 started_at = Time.now.utc
 
 options = {
-  start_time: time_ago(started_at, '30 min'),
+  start_time: Alblogs::Utils.time_ago(started_at, '30 min'),
   end_time: started_at,
   include_filter: nil,
   exclude_filter: nil,
@@ -218,11 +34,11 @@ OptionParser.new do |opts|
   opts.banner = "Usage: alblogs [options]"
 
   opts.on("-s", "--start=TIME_EXP", "Start time") do |v|
-    options[:start_time] = time_ago(started_at, v)
+    options[:start_time] = Alblogs::Utils.time_ago(started_at, v)
   end
 
   opts.on("-e", "--end=TIME_EXP", "End time") do |v|
-    options[:end_time] = time_ago(started_at, v)
+    options[:end_time] = Alblogs::Utils.time_ago(started_at, v)
   end
 
   opts.on("--include=REGEX", "Include filter") do |v|
@@ -264,7 +80,7 @@ if options[:end_time] && options[:end_time] < options[:start_time]
   options[:start_time], options[:end_time] = options[:end_time], options[:start_time]
 end
 
-request_matcher = RequestMatcher.new options
+request_matcher = Alblogs::RequestMatcher.new options
 
 stats = Hash.new(0)
 stats[:started_at] = started_at
@@ -282,11 +98,12 @@ File.unlink("#{tmp_file}.gz") if File.exists?("#{tmp_file}.gz")
 $stop = false
 trap("INT") { $stop = true }
 
-get_s3_files_in_range(request_matcher.range, options[:alb_s3_bucket], options[:aws_profile]).values.each do |s3_file|
+s3_bucket = Alblogs::S3Bucket.new(options[:alb_s3_bucket], options[:aws_profile])
+s3_bucket.get_s3_files_in_range(request_matcher.range).values.each do |s3_file|
   stats[:files] += 1
 
   stats[:total_download_time] += measure do
-    download_s3_file(s3_file, tmp_file, options[:aws_profile])
+    s3_bucket.download_s3_file(s3_file, tmp_file)
   end
 
   stats[:total_file_processing_time] += measure do
@@ -294,7 +111,7 @@ get_s3_files_in_range(request_matcher.range, options[:alb_s3_bucket], options[:a
       while(! f.eof? && ! $stop)
         stats[:lines] += 1
         line = f.readline
-        entry = get_alb_log_entry(line)
+        entry = Alblogs::Entry.from_line(line)
         stats[:min_log_time] = ! stats[:min_log_time] || stats[:min_log_time] > entry.timestamp ? entry.timestamp : stats[:min_log_time]
         stats[:max_log_time] = ! stats[:max_log_time] || stats[:max_log_time] < entry.timestamp ? entry.timestamp : stats[:max_log_time]
         next unless request_matcher.match?(entry)

data/lib/alblogs.rb

@@ -0,0 +1,41 @@
+module Alblogs
+  autoload :Entry, 'alblogs/entry'
+  autoload :RequestMatcher, 'alblogs/request_matcher'
+  autoload :S3Bucket, 'alblogs/s3_bucket'
+  autoload :S3File, 'alblogs/s3_file'
+  autoload :Utils, 'alblogs/utils'
+  
+  FIELDS =
+    begin
+      not_a_space = '([^ ]+)'
+      in_quotes = '"(.*?)"'
+      
+      {
+        type: not_a_space,
+        timestamp: not_a_space,
+        elb: not_a_space,
+        client_port: not_a_space,
+        target_port: not_a_space,
+        request_processing_time: not_a_space,
+        target_processing_time: not_a_space,
+        response_processing_time: not_a_space,
+        elb_status_code: not_a_space,
+        target_status_code: not_a_space,
+        received_bytes: not_a_space,
+        sent_bytes: not_a_space,
+        request: in_quotes,
+        user_agent: in_quotes,
+        ssl_cipher: not_a_space,
+        ssl_protocol: not_a_space,
+        target_group_arn: not_a_space,
+        trace_id: in_quotes,
+        domain_name: in_quotes,
+        chosen_cert_arn: in_quotes,
+        matched_rule_priority: not_a_space,
+        request_creation_time: not_a_space,
+        actions_executed: in_quotes,
+        redirect_url: in_quotes,
+        error_reason: in_quotes
+      }
+    end
+end

data/lib/alblogs/entry.rb

@@ -0,0 +1,21 @@
+module Alblogs
+  class Entry < Struct.new(:line, *::Alblogs::FIELDS.keys)
+    REGEXP = Regexp.new(::Alblogs::FIELDS.values.join(' '))
+
+    def timestamp
+      @timestamp ||= Time.iso8601(self[:timestamp])
+    end
+
+    def target_processing_time
+      self[:target_processing_time].to_f
+    end
+
+    def self.from_line(line)
+      new(*get_fields(line))
+    end
+
+    def self.get_fields(line)
+      REGEXP.match(line).to_a
+    end
+  end
+end

data/lib/alblogs/request_matcher.rb

@@ -0,0 +1,20 @@
+module Alblogs
+  class RequestMatcher
+    attr_reader :range
+
+    def initialize(options)
+      @range = options[:start_time]..options[:end_time]
+      @exclude_filter = options[:exclude_filter]
+      @include_filter = options[:include_filter]
+      @request_times_over = options[:request_times_over]
+    end
+
+    def match?(entry)
+      return false unless @range.cover?(entry.timestamp)
+      return false if @include_filter && ! @include_filter.match?(entry.line)
+      return false if @exclude_filter && @exclude_filter.match?(entry.line)
+      return false if @request_times_over && @request_times_over > entry.target_processing_time
+      true
+    end
+  end
+end

data/lib/alblogs/s3_bucket.rb

@@ -0,0 +1,49 @@
+module Alblogs
+  class S3Bucket
+    attr_reader :bucket,
+                :aws_profile
+
+    def initialize(bucket, aws_profile=nil)
+      @bucket = bucket
+      @aws_profile = aws_profile
+    end
+
+    def get_s3_files(date_path)
+      s3_url = "#{bucket}/#{date_path}/"
+      cmd = "aws"
+      cmd << " --profile #{Shellwords.escape(aws_profile)}" if aws_profile
+      cmd << " s3 ls #{Shellwords.escape(s3_url)}"
+      output = ::Alblogs::Utils.run_or_die(cmd)
+      output.split("\n").map do |line|
+        line =~ /(\d{4}\-\d{2}\-\d{2} \d{2}:\d{2}:\d{2}) +(\d+) +(.+)/
+        last_modified_at = Time.parse($1).utc
+        file_size = $2.to_i
+        file = $3
+        ::Alblogs::S3File.new("#{s3_url}#{file}", file_size, last_modified_at)
+      end
+    end
+
+    def get_s3_files_in_range(range)
+      s3_files = {}
+      time = range.begin
+      while time < range.end
+        date_path = time.strftime('%Y/%m/%d')
+        get_s3_files(date_path).each do |s3_file|
+          next unless s3_file.in_range?(range)
+          s3_files[s3_file.file] ||= s3_file
+        end
+        time += 86_400
+      end
+      s3_files
+    end
+
+    def download_s3_file(s3_file, dest)
+      cmd = "aws"
+      cmd << " --profile #{Shellwords.escape(aws_profile)}" if aws_profile
+      cmd << " s3 cp #{Shellwords.escape(s3_file.file)} #{Shellwords.escape(dest)}.gz"
+      ::Alblogs::Utils.run_or_die(cmd)
+      cmd = "gzip -f -d #{Shellwords.escape(dest)}.gz"
+      ::Alblogs::Utils.run_or_die(cmd)
+    end
+  end
+end

data/lib/alblogs/s3_file.rb

@@ -0,0 +1,35 @@
+module Alblogs
+  class S3File
+    MINUTES_5 = 5 * 60
+
+    attr_reader :file,
+                :file_size,
+                :last_modified_at
+
+    def initialize(file, file_size, last_modified_at)
+      @file = file
+      @file_size = file_size
+      @last_modified_at = last_modified_at
+    end
+
+    def end_time
+      @end_time ||=
+        begin
+          unless @file =~ /_(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})Z_/
+            raise("unable to find time stamp in #{@file}")
+          end
+          Time.new($1, $2, $3, $4, $5, 0, 0)
+        end
+    end
+
+    def start_time
+      @start_time ||= (end_time - MINUTES_5)
+    end
+
+    def in_range?(range)
+      return false if end_time < range.begin
+      return false if start_time > range.end
+      true
+    end
+  end
+end

data/lib/alblogs/utils.rb

@@ -0,0 +1,32 @@
+module Alblogs
+  module Utils
+    module_function
+
+    def parse_time_offset(str)
+      if str =~ /min/
+        str.sub(/ *min.*/, '').to_i * 60
+      elsif str =~ /hour/
+        str.sub(/ *hour.*/, '').to_i * 3600
+      elsif str =~ /day/
+        str.sub(/ *day.*/, '').to_i * 86400
+      else
+        nil
+      end
+    end
+
+    def time_ago(now, str)
+      if offset = parse_time_offset(str)
+        time = now - offset
+        time - (time.to_i % 60) # round to the start of the minute
+      else
+        Time.parse(str).utc
+      end
+    end
+
+    def run_or_die(cmd)
+      res = `#{cmd}`
+      raise("command failed with #{$?}, #{cmd}") unless $?.success?
+      res
+    end
+  end
+end

data/script/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH << File.expand_path('../lib', __dir__)
+require 'alblogs'
+require 'irb'
+IRB.start(__FILE__)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: alblogs
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Doug Youch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-06 00:00:00.000000000 Z
+date: 2019-10-08 00:00:00.000000000 Z
 dependencies: []
 description: Utility script for processing ALB access logs over a given time range
 email: dougyouch@gmail.com
@@ -18,9 +18,18 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
 - README.md
 - alblogs.gemspec
 - bin/alblogs
+- lib/alblogs.rb
+- lib/alblogs/entry.rb
+- lib/alblogs/request_matcher.rb
+- lib/alblogs/s3_bucket.rb
+- lib/alblogs/s3_file.rb
+- lib/alblogs/utils.rb
+- script/console
 homepage: https://github.com/dougyouch/alblogs
 licenses: []
 metadata: {}