alblogs 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eefbd8b8cac015fc5eee191b6e8a73ff9fde3c465f8d1410b76774e686aa9623
-  data.tar.gz: 3dfce14461c67824dd0ad6c5bd1ecee0081767de256191bda00cad1a99c18d1f
+  metadata.gz: 7d2aea6572a238675e0499c04e26341ea3cd3b8860b94c51095fe9c00cb4d806
+  data.tar.gz: 45a86b3b527c47bee978d0139c473c3aa491a1bddea41d2d10135093cc0cde4c
 SHA512:
-  metadata.gz: 9b17f53979c6c56d8909d62e1fcacf468edcd0294fc41928cd62a706d5d206fd8495e16453c339ea5dfd236aef9158c8ad9cfcb00522cc961e18ce82e98f7a4e
-  data.tar.gz: d5d792b11d17f82e30a58901f18ac7a1cdf1e3cea5d07201645475a63a255a06769f64772c947653f6ab2533657c23c84bdd2db4517382d484389e506f07079f
+  metadata.gz: 141b1919b75fc436dcb627fb93c20fd1312e57aa4d2b1525d1c7180f8e4921dba8c5af93614392f93162f9006756d4eba04a896f1ec18511033d4db7bdc7ae40
+  data.tar.gz: cc2c69a05b388d603c7e40cdf29759f57090b6894a4f9a001d1157a57a03692b419963e1f086efa6a87a35175ba4797b1a3121ea779c71a6f48bb92a46798f84

data/alblogs.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'alblogs'
-  s.version     = '0.1.0'
+  s.version     = '0.1.2'
   s.summary     = 'ALB access log processing'
   s.description = 'Utility script for processing ALB access logs over a given time range'
   s.authors     = ['Doug Youch']

data/bin/alblogs

@@ -6,17 +6,6 @@ require 'shellwords'
 require 'json'
 require 'alblogs'
 
-def measure 
-  start = Time.now
-  yield
-  Time.now - start
-end
-  
-def display_stats(stats)
-  stats[:elapsed_time] = Time.now.utc - stats[:started_at]
-  $stderr.puts stats.inspect
-end
-
 started_at = Time.now.utc
 
 options = {
@@ -81,52 +70,18 @@ if options[:end_time] && options[:end_time] < options[:start_time]
 end
 
 request_matcher = Alblogs::RequestMatcher.new options
-
-stats = Hash.new(0)
-stats[:started_at] = started_at
-stats[:range_starts_at] = request_matcher.range.begin
-stats[:range_ends_at] = request_matcher.range.end
-stats[:min_log_time] = nil
-stats[:max_log_time] = nil
-stats[:min_matched_log_time] = nil
-stats[:max_matched_log_time] = nil
-
-tmp_file = '.download.alblogs.log'
-File.unlink(tmp_file) if File.exists?(tmp_file)
-File.unlink("#{tmp_file}.gz") if File.exists?("#{tmp_file}.gz")
-
-$stop = false
-trap("INT") { $stop = true }
-
 s3_bucket = Alblogs::S3Bucket.new(options[:alb_s3_bucket], options[:aws_profile])
-s3_bucket.get_s3_files_in_range(request_matcher.range).values.each do |s3_file|
-  stats[:files] += 1
-
-  stats[:total_download_time] += measure do
-    s3_bucket.download_s3_file(s3_file, tmp_file)
-  end
 
-  stats[:total_file_processing_time] += measure do
-    File.open(tmp_file, 'rb') do |f|
-      while(! f.eof? && ! $stop)
-        stats[:lines] += 1
-        line = f.readline
-        entry = Alblogs::Entry.from_line(line)
-        stats[:min_log_time] = ! stats[:min_log_time] || stats[:min_log_time] > entry.timestamp ? entry.timestamp : stats[:min_log_time]
-        stats[:max_log_time] = ! stats[:max_log_time] || stats[:max_log_time] < entry.timestamp ? entry.timestamp : stats[:max_log_time]
-        next unless request_matcher.match?(entry)
-        stats[:matching_lines] += 1
-        stats[:min_matched_log_time] = ! stats[:min_matched_log_time] || stats[:min_matched_log_time] > entry.timestamp ? entry.timestamp : stats[:min_matched_log_time]
-        stats[:max_matched_log_time] = ! stats[:max_matched_log_time] || stats[:max_matched_log_time] < entry.timestamp ? entry.timestamp : stats[:max_matched_log_time]
-        options[:log_file].puts line
-      end
-    end
-  end
+iterator = Alblogs::Iterator.new(s3_bucket, request_matcher.range, request_matcher)
+iterator.display_stats_proc = Proc.new do |stats|
+  stats[:elapsed_time] = Time.now.utc - stats[:started_at]
+  $stderr.puts stats.inspect
+end
 
-  File.unlink(tmp_file)
+trap("INT") { iterator.stop! }
 
-  display_stats(stats) if options[:display_stats]
-  break if $stop
+iterator.each do |entry|
+  options[:log_file].puts entry.line
 end
 
 options[:log_file].close

data/lib/alblogs.rb

@@ -1,5 +1,6 @@
 module Alblogs
   autoload :Entry, 'alblogs/entry'
+  autoload :Iterator, 'alblogs/iterator'
   autoload :RequestMatcher, 'alblogs/request_matcher'
   autoload :S3Bucket, 'alblogs/s3_bucket'
   autoload :S3File, 'alblogs/s3_file'

data/lib/alblogs/iterator.rb

@@ -0,0 +1,88 @@
+module Alblogs
+  class Iterator
+    include Enumerable
+
+    attr_accessor :tmp_file,
+                  :display_stats_proc
+
+    attr_reader :s3_bucket,
+                :date_range,
+                :request_matcher,
+                :stats
+
+    def initialize(s3_bucket, date_range, request_matcher = nil)
+      @s3_bucket = s3_bucket
+      @date_range = date_range
+      @request_matcher = request_matcher
+      @tmp_file = '.download.alblogs.log'
+    end
+
+    def each
+      @stop = false
+      delete_tmp_file
+      init_stats
+
+      s3_bucket.get_s3_files_in_range(date_range).values.each do |s3_file|
+        stats[:files] += 1
+
+        stats[:total_download_time] += measure do
+          s3_bucket.download_s3_file(s3_file, tmp_file)
+        end
+
+        stats[:total_file_processing_time] += measure do
+          File.open(tmp_file, 'rb') do |f|
+            while(! f.eof? && ! @stop)
+              stats[:lines] += 1
+              line = f.readline
+              entry = ::Alblogs::Entry.from_line(line)
+              stats[:min_log_time] = ! stats[:min_log_time] || stats[:min_log_time] > entry.timestamp ? entry.timestamp : stats[:min_log_time]
+              stats[:max_log_time] = ! stats[:max_log_time] || stats[:max_log_time] < entry.timestamp ? entry.timestamp : stats[:max_log_time]
+              next if request_matcher && !request_matcher.match?(entry)
+              stats[:matching_lines] += 1
+              stats[:min_matched_log_time] = ! stats[:min_matched_log_time] || stats[:min_matched_log_time] > entry.timestamp ? entry.timestamp : stats[:min_matched_log_time]
+              stats[:max_matched_log_time] = ! stats[:max_matched_log_time] || stats[:max_matched_log_time] < entry.timestamp ? entry.timestamp : stats[:max_matched_log_time]
+              yield entry
+            end
+          end
+        end
+
+        File.unlink(tmp_file)
+
+        display_stats
+        break if @stop
+      end
+    end
+
+    def stop!
+      @stop = true
+    end
+
+    def display_stats
+      display_stats_proc && display_stats_proc.call(stats)
+    end
+
+    private
+
+    def measure 
+      start = Time.now
+      yield
+      Time.now - start
+    end
+
+    def delete_tmp_file
+      File.unlink(tmp_file) if File.exists?(tmp_file)
+      File.unlink("#{tmp_file}.gz") if File.exists?("#{tmp_file}.gz")
+    end
+
+    def init_stats
+      @stats = Hash.new(0)
+      @stats[:started_at] = Time.now.utc
+      @stats[:range_starts_at] = date_range.begin
+      @stats[:range_ends_at] = date_range.end
+      @stats[:min_log_time] = nil
+      @stats[:max_log_time] = nil
+      @stats[:min_matched_log_time] = nil
+      @stats[:max_matched_log_time] = nil
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: alblogs
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Doug Youch
@@ -25,6 +25,7 @@ files:
 - bin/alblogs
 - lib/alblogs.rb
 - lib/alblogs/entry.rb
+- lib/alblogs/iterator.rb
 - lib/alblogs/request_matcher.rb
 - lib/alblogs/s3_bucket.rb
 - lib/alblogs/s3_file.rb