alblogs 0.1.4 → 0.1.8

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 76c233cf10e2558be79ef68d0f92670181be1b7bd1b24eb7c5376a21264856bc
4
- data.tar.gz: f4bf7a47915a7a8763e9826f694e19a3c270440b047a93e73327a19c47a59684
3
+ metadata.gz: 7d30c169c414e1ce7cddb98c4021f39434c9dc216ad97e82335e643702429f91
4
+ data.tar.gz: 5b12b1ea6ee875280a10eafaf00b8bfd44919797d0db267c29d71f33d5303458
5
5
  SHA512:
6
- metadata.gz: 0de9e99529e3f13eddccf1f450bc11ee96a8e1b9c0827833343bd376f52e47d6de13aab8979bbc05273ddad67bfb3303235d3973c7ccdf50bb8388a2bfc76b34
7
- data.tar.gz: 8cf0c70772dd797c748b606243585704af31d15964be8eae00856bb470d4e5f4ff844b4b4566ffa9ce68f83b495cef4e89230a0a49343949b81e1d93017d01f8
6
+ metadata.gz: a660c6c7af77f7063176546204f59b7e7e66f2c25001950ebb705df272d05e4e4efece6eac1502e6f6c26d7b8cffed1a9954c707805f2440cb3f80a93d8f9fe1
7
+ data.tar.gz: 9ee39d7e180a9e68634fe8fddeaee4b3aa5b2f69e758293fed3c8a2018653a3e1f6ec3e1dd080fc165b4571303bf95c97d2f07a46b883e8fe5c9fad42601dbe6
data/alblogs.gemspec CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |s|
4
4
  s.name = 'alblogs'
5
- s.version = '0.1.4'
5
+ s.version = '0.1.8'
6
6
  s.summary = 'ALB access log processing'
7
7
  s.description = 'Utility script for processing ALB access logs over a given time range'
8
8
  s.authors = ['Doug Youch']
data/bin/alblogpp ADDED
@@ -0,0 +1,38 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Purpose: to pretty print ALB log entries
4
+
5
+ require 'optparse'
6
+ require 'alblogs'
7
+ require 'json'
8
+ require 'csv'
9
+
10
+ options = {
11
+ format: 'vertical'
12
+ }
13
+ OptionParser.new do |opts|
14
+ opts.banner = "Usage: alblogpp [options]"
15
+
16
+ opts.on("-f", "--format=FORMAT", "Output format vertical, json, csv, jsonl") do |v|
17
+ options[:format] = v
18
+ end
19
+ end.parse!
20
+
21
+ if options[:format] == 'csv'
22
+ print Alblogs::Entry.fields.to_csv
23
+ end
24
+
25
+ Alblogs::Entry.each_entry($stdin) do |entry|
26
+ case options[:format]
27
+ when 'csv'
28
+ print entry.to_a.to_csv
29
+ when 'json'
30
+ print JSON.pretty_generate(entry.to_h) + "\n"
31
+ when 'jsonl'
32
+ print entry.to_h.to_json + "\n"
33
+ else
34
+ entry.to_h.each do |key, value|
35
+ puts "#{key}: #{value}"
36
+ end
37
+ end
38
+ end
data/bin/alblogs CHANGED
@@ -1,9 +1,6 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
3
  require 'optparse'
4
- require 'time'
5
- require 'shellwords'
6
- require 'json'
7
4
  require 'alblogs'
8
5
 
9
6
  started_at = Time.now.utc
@@ -0,0 +1,140 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'alblogs'
4
+
5
+ LOG_FILE = ARGV.shift || raise('no log file specified')
6
+ raise("#{LOG_FILE} not found") unless File.exist?(LOG_FILE)
7
+
8
+ class RequestStat
9
+ attr_reader :method,
10
+ :path,
11
+ :num_requests,
12
+ :num_failed_requests,
13
+ :num_successful_requests,
14
+ :successful_requests_total_time,
15
+ :total_request_time,
16
+ :max_request_time,
17
+ :num_requests_over_a_second,
18
+ :response_codes,
19
+ :response_times
20
+
21
+ def initialize(method, path)
22
+ @method = method
23
+ @path = path
24
+ reset
25
+ end
26
+
27
+ def average_response_time
28
+ @total_request_time / (@num_requests - @num_failed_requests)
29
+ end
30
+
31
+ def average_successful_response_time
32
+ @num_successful_requests == 0 ? -1 : @successful_requests_total_time / @num_successful_requests
33
+ end
34
+
35
+ def num_500s
36
+ cnt = 0
37
+ @response_codes.each do |code, num_requests|
38
+ cnt += num_requests if code >= 500
39
+ end
40
+ cnt
41
+ end
42
+
43
+ def percentage_of_500s
44
+ num_500s.to_f / @num_requests.to_f
45
+ end
46
+
47
+ def update(entry)
48
+ @num_requests += 1
49
+
50
+ if (request_time = entry.target_processing_time)
51
+ if entry.elb_status_code >= 200 && entry.elb_status_code < 300
52
+ @num_successful_requests += 1
53
+ @successful_requests_total_time += request_time
54
+ end
55
+
56
+ @total_request_time += request_time
57
+ @max_request_time = request_time if request_time > @max_request_time
58
+ @num_requests_over_a_second += 1 if request_time >= 1.0
59
+
60
+ # break down requests times in buckets of 5 second intervals
61
+ if request_time < 1.0
62
+ @response_times['under_a_second'] += 1
63
+ elsif request_time < 2.0
64
+ @response_times['under_2_seconds'] += 1
65
+ elsif request_time < 5.0
66
+ @response_times['under_5_seconds'] += 1
67
+ elsif request_time < 10.0
68
+ @response_times['under_10_seconds'] += 1
69
+ else
70
+ @response_times['over_10_seconds'] += 1
71
+ end
72
+
73
+ else
74
+ @num_failed_requests += 1
75
+ end
76
+
77
+ @response_codes[entry.elb_status_code] += 1
78
+ end
79
+
80
+ def reset
81
+ @num_requests = 0
82
+ @num_failed_requests = 0
83
+ @total_request_time = 0.0
84
+ @max_request_time = 0.0
85
+ @num_requests_over_a_second = 0
86
+ @num_successful_requests = 0
87
+ @successful_requests_total_time = 0.0
88
+ @response_codes = Hash.new(0)
89
+ @response_times = Hash.new(0)
90
+ end
91
+ end
92
+
93
+ def normalize_path(entry)
94
+ entry
95
+ .request_uri
96
+ .path
97
+ .gsub(/(remote_id|external)\/[^\/]+/, '\1/:id')
98
+ .gsub(/\/\d+/, '/:id')
99
+ end
100
+
101
+ def generate_stat_key(entry)
102
+ "#{entry.request_method} #{normalize_path(entry)}"
103
+ end
104
+
105
+ def get_stats
106
+ stats = {}
107
+
108
+ File.open(LOG_FILE, 'rb') do |f|
109
+ Alblogs::Entry.each_entry(f) do |entry|
110
+ next if entry.request_method == 'OPTIONS'
111
+ stat = (stats[generate_stat_key(entry)] ||= RequestStat.new(entry.request_method, normalize_path(entry)))
112
+ stat.update(entry)
113
+ end
114
+ end
115
+
116
+ stats
117
+ end
118
+
119
+ def top_x_request_that_take_over_a_second(stats, top=20)
120
+ stats.values.sort { |a, b| b.num_requests_over_a_second <=> a.num_requests_over_a_second }[0, top]
121
+ end
122
+
123
+ def top_x_requested_pages(stats, top=20)
124
+ stats.values.sort { |a, b| b.num_requests <=> a.num_requests }[0, top]
125
+ end
126
+
127
+ def top_x_requests_that_take_over(stats, over='over_10_seconds', top=20)
128
+ stats.values.sort { |a, b| b.response_times[over].to_i <=> a.response_times[over].to_i }[0, top]
129
+ end
130
+
131
+ def top_x_requests_that_have_500s(stats, top=20)
132
+ stats.values.sort { |a, b| b.num_500s <=> a.num_500s }[0, top]
133
+ end
134
+
135
+ def top_x_worst_routes(stats, top=20)
136
+ stats.values.sort { |a, b| b.percentage_of_500s <=> a.percentage_of_500s }[0, top]
137
+ end
138
+
139
+ require 'irb'
140
+ IRB.start
data/lib/alblogs/entry.rb CHANGED
@@ -2,12 +2,55 @@ module Alblogs
2
2
  class Entry < Struct.new(:line, *::Alblogs::FIELDS.keys)
3
3
  REGEXP = Regexp.new(::Alblogs::FIELDS.values.join(' '))
4
4
 
5
+ def self.fields
6
+ ::Alblogs::FIELDS.keys
7
+ end
8
+
9
+ def request_parts
10
+ @request_parts ||= request.split(' ', 3)
11
+ end
12
+
13
+ def request_method
14
+ request_parts[0]
15
+ end
16
+
17
+ def request_url
18
+ request_parts[1]
19
+ end
20
+
21
+ def request_protocol
22
+ request_parts[2]
23
+ end
24
+
25
+ def request_uri
26
+ @request_uri ||= URI(request_url)
27
+ end
28
+
5
29
  def timestamp
6
30
  @timestamp ||= Time.iso8601(self[:timestamp])
7
31
  end
8
32
 
33
+ def to_a
34
+ ::Alblogs::FIELDS.keys.map { |k| send(k) }
35
+ end
36
+
37
+ def to_h
38
+ Hash[::Alblogs::FIELDS.keys.zip(to_a)]
39
+ end
40
+
41
+ # target_processing_time is in seconds with with millisecond precision
42
+ # -1 means target or server close connection or timed out
9
43
  def target_processing_time
10
- self[:target_processing_time].to_f
44
+ self[:target_processing_time] == '-1' ? nil : self[:target_processing_time].to_f
45
+ end
46
+
47
+ # this is the response code the client received
48
+ def elb_status_code
49
+ self[:elb_status_code].to_i
50
+ end
51
+
52
+ def target_status_code
53
+ self[:target_status_code] == '-' ? nil : self[:target_status_code].to_i
11
54
  end
12
55
 
13
56
  def self.from_line(line)
data/lib/alblogs.rb CHANGED
@@ -1,3 +1,6 @@
1
+ require 'time'
2
+ require 'shellwords'
3
+
1
4
  module Alblogs
2
5
  autoload :Entry, 'alblogs/entry'
3
6
  autoload :Iterator, 'alblogs/iterator'
metadata CHANGED
@@ -1,19 +1,21 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: alblogs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Doug Youch
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-10-08 00:00:00.000000000 Z
11
+ date: 2023-09-01 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Utility script for processing ALB access logs over a given time range
14
14
  email: dougyouch@gmail.com
15
15
  executables:
16
+ - alblogpp
16
17
  - alblogs
18
+ - alblogs-analyze
17
19
  extensions: []
18
20
  extra_rdoc_files: []
19
21
  files:
@@ -22,7 +24,9 @@ files:
22
24
  - ".ruby-version"
23
25
  - README.md
24
26
  - alblogs.gemspec
27
+ - bin/alblogpp
25
28
  - bin/alblogs
29
+ - bin/alblogs-analyze
26
30
  - lib/alblogs.rb
27
31
  - lib/alblogs/entry.rb
28
32
  - lib/alblogs/iterator.rb
@@ -34,7 +38,7 @@ files:
34
38
  homepage: https://github.com/dougyouch/alblogs
35
39
  licenses: []
36
40
  metadata: {}
37
- post_install_message:
41
+ post_install_message:
38
42
  rdoc_options: []
39
43
  require_paths:
40
44
  - lib
@@ -49,8 +53,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
49
53
  - !ruby/object:Gem::Version
50
54
  version: '0'
51
55
  requirements: []
52
- rubygems_version: 3.0.3
53
- signing_key:
56
+ rubygems_version: 3.0.9
57
+ signing_key:
54
58
  specification_version: 4
55
59
  summary: ALB access log processing
56
60
  test_files: []