ralf 1.1.0 → 1.1.1

data/.gitignore

@@ -0,0 +1,3 @@
+**.gem
+tmp/**
+rdoc

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+gemspec
+gem 'rspec'

data/Gemfile.lock

@@ -0,0 +1,38 @@
+PATH
+  remote: .
+  specs:
+    ralf (1.1.1)
+      chronic (~> 0.9.0)
+      logmerge (~> 1.0.3)
+      right_aws (~> 3.0.4)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    ZenTest (4.8.4)
+    autotest (4.4.6)
+      ZenTest (>= 4.4.1)
+    chronic (0.9.0)
+    diff-lcs (1.1.3)
+    fakeweb (1.3.0)
+    logmerge (1.0.3)
+    right_aws (3.0.4)
+      right_http_connection (>= 1.2.5)
+    right_http_connection (1.3.0)
+    rspec (2.12.0)
+      rspec-core (~> 2.12.0)
+      rspec-expectations (~> 2.12.0)
+      rspec-mocks (~> 2.12.0)
+    rspec-core (2.12.2)
+    rspec-expectations (2.12.1)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.12.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  autotest (~> 4.4.6)
+  fakeweb (~> 1.3.0)
+  ralf!
+  rspec

data/README.rdoc

@@ -1,3 +1,15 @@
+Release 1.1.1 [2013-02-11 13:14]
+
+* Update gemspec
+* extract the translator in it's own class
+* add option to recalculate the '206 Partial Content' issue on S3
+	(see https://forums.aws.amazon.com/thread.jspa?threadID=54214 for more details)
+
+Release 1.1.0 [2011-05-08]
+
+* Switched to Fileutils for 1.9 compatibility
+
+
 = Synopsis
 
 Download, merge and convert Amazon S3 bucket log files for a specified date or date range.

data/lib/ralf.rb

@@ -2,8 +2,11 @@ require 'rubygems'
 require 'right_aws'
 require 'logmerge'
 require 'fileutils'
+
+require 'ralf/version'
 require 'ralf/config'
 require 'ralf/bucket'
+require 'ralf/clf_translator'
 require 'chronic'
 require 'stringio'
 require 'date'
@@ -18,9 +21,6 @@ class Ralf
   ROOT_DEFAULT_CONFIG_FILE = '/etc/ralf.conf'
   USER_DEFAULT_CONFIG_FILE = '~/.ralf.conf'
 
-  AMAZON_LOG_FORMAT = Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" "([^"]*)"')
-  AMAZON_LOG_FORMAT_COPY = Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) (REST.COPY.OBJECT_GET) ([^ ]*) (-) ([^ ]*) (-) (-) ([^ ]*) (-) (-) (-) (-) (-)')
-  
   # The current configuration.
   attr_reader :config
   
@@ -90,7 +90,7 @@ class Ralf
       Ralf.merge(merged_log, log_files)
       
       # convert to common log format
-      Ralf.convert_to_common_log_format(merged_log, output_log)
+      Ralf.convert_to_common_log_format(merged_log, output_log, config.translate_options)
 
       puts "#{log_files.size} files" if config.debug?
     end
@@ -133,11 +133,11 @@ class Ralf
   end
 
   # Convert the input_log file to Apache Common Log Format into output_log
-  def self.convert_to_common_log_format(input_log, output_log)
+  def self.convert_to_common_log_format(input_log, output_log, options)
     out_file = File.open(output_log, 'w')
     File.open(input_log, 'r') do |in_file|
       while (line = in_file.gets)
-        if clf = translate_to_clf(line)
+        if clf = Ralf::ClfTranslator.new(line, options).to_s
           out_file.puts(clf)
         end
       end
@@ -145,18 +145,6 @@ class Ralf
     out_file.close
   end
 
-  def self.translate_to_clf(line)
-    if line =~ AMAZON_LOG_FORMAT
-      # host, date, ip, acl, request, status, bytes, agent, total_time_ms = $2, $3, $4, $5, $9, $10, $12, $17, $14
-      "%s - %s [%s] \"%s\" %s %s \"%s\" \"%s\" %d" % [$4, $5, $3, $9, $10, $12, $16, $17, ($14.to_i/1000.0).round]
-    elsif line =~ AMAZON_LOG_FORMAT_COPY
-      "%s - %s [%s] \"%s\" %s %s \"%s\" \"REST.COPY.OBJECT_GET\" %d" % [$4, $5, $3, "POST /#{$8} HTTP/1.1", $10, $12, $16, 0]
-    else
-      $stderr.puts "# ERROR: #{line}"
-      nil
-    end
-  end
-
   def load_config(cli_config_file)
     result = nil
     if cli_config_file

data/lib/ralf/clf_translator.rb

@@ -0,0 +1,97 @@
+class Ralf::ClfTranslator
+
+  AMAZON_LOG_FORMAT =      Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" "([^"]*)"')
+  AMAZON_LOG_COPY_FORMAT = Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) (REST.COPY.OBJECT_GET) ([^ ]*) (-) ([^ ]*) (-) (-) ([^ ]*) (-) (-) (-) (-) (-)')
+
+  attr :line
+  attr_reader :owner, :bucket, :timestamp, :remote_ip, :request_id, :operation, :key, :request_uri, :http_status, :s3_error_code, :bytes_sent, :object_size, :total_time_in_ms, :turn_around_time_in_ms, :referrer, :user_agent, :request_version_id, :duration
+  attr_reader :options
+
+  # options:
+  #   :recalculate_partial_content => false (default)
+  #     If request is '206 Partial Content' estimate the actual bytes when apparent bandwidth has exceeded 2Mbit/sec.
+  #     S3 caches content to edge servers with a burst which never reaches the client
+
+  def initialize(line, options = {})
+    @options = options
+    @error = false
+    @line = line
+    @translate_successfull = translate
+  end
+
+  def to_s
+    if @translate_successfull
+      "%s - %s [%s] \"%s\" %s %s \"%s\" \"%s\" %d" % [remote_ip, requester, timestamp, request_uri, http_status, bytes_sent, referrer, user_agent, duration]
+    else
+      nil
+    end
+  end
+
+private
+
+  def requester
+    @requester[0..9]
+  end
+
+  def translate
+    if line =~ AMAZON_LOG_FORMAT
+      @owner, @bucket, @timestamp, @remote_ip, @requester, @request_id, @operation, @key, @request_uri, @http_status, @s3_error_code, @bytes_sent, @object_size, @total_time_in_ms, @turn_around_time_in_ms, @referrer, @user_agent, @request_version_id = $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18
+
+      if options[:recalculate_partial_content] && 206 == http_status.to_i && ((bytes_sent.to_i*8)/total_time_in_ms.to_i > 2000)
+        @bytes_sent = [ 128 * 1024 + 3 * total_time_in_ms.to_i, bytes_sent.to_i ].min # 128 K buffer + 3 bytes/msec = 3 kbytes/sec = 24 kbit/sec
+      end
+      @duration = (total_time_in_ms.to_i/1000.0).round
+
+    elsif line =~ AMAZON_LOG_COPY_FORMAT
+      @owner, @bucket, @timestamp, @remote_ip, @requester, @request_id, @operation, @key, @request_uri, @http_status, @s3_error_code, @bytes_sent, @object_size, @total_time_in_ms, @turn_around_time_in_ms, @referrer, @user_agent, @request_version_id = $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18
+      @operation == 'REST.COPY.OBJECT_GET'
+      @user_agent = @operation
+      @duration = 0
+      @request_uri = "POST /#{@key} HTTP/1.1"
+
+    else
+      $stderr.puts "# ERROR: #{line}"
+      false
+    end
+  end
+
+end
+
+# convert the format as specified in http://docs.aws.amazon.com/AmazonS3/latest/dev/LogFormat.html
+# 
+# 1 Bucket Owner  
+#   The canonical user id of the owner of the source bucket.
+# 2 Bucket  
+#   The name of the bucket that the request was processed against. If the system receives a malformed request and cannot determine the bucket, the request will not appear in any server access log.
+# 3 Time  
+#   The time at which the request was received. The format, using strftime() terminology, is [%d/%b/%Y:%H:%M:%S %z]
+# 4 Remote IP 
+#   The apparent Internet address of the requester. Intermediate proxies and firewalls might obscure the actual address of the machine making the request.
+# 5 Requester 
+#   The canonical user id of the requester, or the string "Anonymous" for unauthenticated requests. This identifier is the same one used for access control purposes.
+# 6 Request ID  
+#   The request ID is a string generated by Amazon S3 to uniquely identify each request.
+# 7 Operation 
+#   Either SOAP.operation, REST.HTTP_method.resource_type or WEBSITE.HTTP_method.resource_type
+# 8 Key 
+#   The "key" part of the request, URL encoded, or "-" if the operation does not take a key parameter.
+# 9 Request-URI 
+#   The Request-URI part of the HTTP request message.
+# 10 HTTP status 
+#   The numeric HTTP status code of the response.
+# 11 Error Code  
+#   The Amazon S3 Error Code, or "-" if no error occurred.
+# 12 Bytes Sent  
+#   The number of response bytes sent, excluding HTTP protocol overhead, or "-" if zero.
+# 13 Object Size 
+#   The total size of the object in question.
+# 14 Total Time  
+#   The number of milliseconds the request was in flight from the server's perspective. This value is measured from the time your request is received to the time that the last byte of the response is sent. Measurements made from the client's perspective might be longer due to network latency.
+# 15 Turn-Around Time  
+#   The number of milliseconds that Amazon S3 spent processing your request. This value is measured from the time the last byte of your request was received until the time the first byte of the response was sent.
+# 16 Referrer  
+#   The value of the HTTP Referrer header, if present. HTTP user-agents (e.g. browsers) typically set this header to the URL of the linking or embedding page when making a request.
+# 17 User-Agent  
+#   The value of the HTTP User-Agent header.
+# 18 Version Id
+#   The version ID in the request, or "-" if the operation does not take a versionId parameter.

data/lib/ralf/config.rb

@@ -22,6 +22,7 @@ class Ralf::Config
     :cache_dir    # reader interpolates format
     
   attr_reader :errors
+  attr_reader :translate_options
   
   protected
   
@@ -40,6 +41,7 @@ class Ralf::Config
     @options[:now]       ||= nil
     @options[:range]     ||= 'today'
     @options[:cache_dir] ||= (0 == Process.uid ? ROOT_DEFAULT_CACHE_DIR : File.expand_path(USER_DEFAULT_CACHE_DIR))
+    @options[:translate_options] ||= {}
 
     assign_options(@options)
   end
@@ -102,6 +104,10 @@ class Ralf::Config
     @range = range
   end
   
+  def translate_options=(opts)
+    @translate_options = {:recalculate_partial_content => false}.merge(opts)
+  end
+
   def output_file(variables)
     Ralf::Interpolation.interpolate(@output_file, variables)
   end

data/lib/ralf/version.rb

@@ -0,0 +1,3 @@
+class Ralf #:nodoc:
+  VERSION = "1.1.1"
+end

data/ralf.gemspec

@@ -1,77 +1,38 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "ralf/version"
 
 Gem::Specification.new do |s|
   s.name = %q{ralf}
-  s.version = "1.1.0"
+  s.version = Ralf::VERSION
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Klaas Jan Wierenga", "Leon Berenschot"]
-  s.date = %q{2011-05-09}
+  s.platform           = Gem::Platform::RUBY
+  s.authors            = ["Klaas Jan Wierenga", "Leon Berenschot"]
+  s.email              = ["k.j.wierenga@gmail.com", "leonb@beriedata.nl"]
+  s.homepage           = %q{http://github.com/kjwierenga/ralf}
+  s.summary            = %q{Retrieve Amazon Log Files}
+  s.description        = %q{ Download logfiles from Amazon S3 buckets to local disk and combine them in one Apache CLF per bucket }
+
+  s.files              = `git ls-files`.split("\n")
+  s.test_files         = `git ls-files -- {spec,features}/*`.split("\n")
+  s.executables        = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths      = ["lib"]
+
+  s.date               = %q{2013-02-06}
   s.default_executable = %q{ralf}
-  s.description = %q{      Download logfiles from Amazon S3 buckets to local disk and combine them in one Apache CLF per bucket
-}
-  s.email = ["k.j.wierenga@gmail.com", "leonb@beriedata.nl"]
-  s.executables = ["ralf"]
   s.extra_rdoc_files = [
     "README.rdoc"
   ]
-  s.files = [
-    ".rvmrc",
-    "README.rdoc",
-    "Rakefile",
-    "VERSION",
-    "bin/ralf",
-    "lib/ralf.rb",
-    "lib/ralf/bucket.rb",
-    "lib/ralf/config.rb",
-    "lib/ralf/interpolation.rb",
-    "lib/ralf/log.rb",
-    "lib/ralf/option_parser.rb",
-    "ralf.gemspec",
-    "spec/fixtures/apache.log",
-    "spec/fixtures/example_buckets.yaml",
-    "spec/ralf/bucket_spec.rb",
-    "spec/ralf/config_spec.rb",
-    "spec/ralf/interpolation_spec.rb",
-    "spec/ralf/log_spec.rb",
-    "spec/ralf/option_parser_spec.rb",
-    "spec/ralf_spec.rb",
-    "spec/spec.opts",
-    "spec/spec_helper.rb",
-    "spec/support/fakeweb.rb"
-  ]
-  s.homepage = %q{http://github.com/kjwierenga/ralf}
-  s.rdoc_options = ["--exclude", "."]
-  s.require_paths = ["lib"]
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Retrieve Amazon Log Files}
 
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  s.rdoc_options       = ["--exclude", "."]
+
+  s.add_development_dependency "rspec", "~> 2"
+  s.add_development_dependency "autotest", '~> 4.4.6'
+  s.add_development_dependency "fakeweb", "~> 1.3.0"
+
+  s.add_runtime_dependency "right_aws", "~> 3.0.4"
+  s.add_runtime_dependency "logmerge",  "~> 1.0.3"
+  s.add_runtime_dependency "chronic",   "~> 0.9.0"
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_development_dependency(%q<rspec>, ["~> 1.3.0"])
-      s.add_development_dependency(%q<fakeweb>, ["~> 1.2.8"])
-      s.add_runtime_dependency(%q<right_aws>, ["~> 1.10.0"])
-      s.add_runtime_dependency(%q<logmerge>, ["~> 1.0.2"])
-      s.add_runtime_dependency(%q<chronic>, [">= 0.2.3"])
-    else
-      s.add_dependency(%q<rspec>, ["~> 1.3.0"])
-      s.add_dependency(%q<fakeweb>, ["~> 1.2.8"])
-      s.add_dependency(%q<right_aws>, ["~> 1.10.0"])
-      s.add_dependency(%q<logmerge>, ["~> 1.0.2"])
-      s.add_dependency(%q<chronic>, [">= 0.2.3"])
-    end
-  else
-    s.add_dependency(%q<rspec>, ["~> 1.3.0"])
-    s.add_dependency(%q<fakeweb>, ["~> 1.2.8"])
-    s.add_dependency(%q<right_aws>, ["~> 1.10.0"])
-    s.add_dependency(%q<logmerge>, ["~> 1.0.2"])
-    s.add_dependency(%q<chronic>, [">= 0.2.3"])
-  end
 end
 

data/spec/fixtures/winamp.txt

@@ -0,0 +1,24 @@
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:57:29 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 AE5C72112DFB80DE REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.0" 200 - 4215272 18708200 5165 47 "-" "WinampMPEG/5.56, Ultravox/2.1" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:57:34 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 4C59A68A6BC2D5D3 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4210177 18708200 4342 25 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:57:38 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 ECBB95CD19EFEDAB REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4200071 18708200 3794 20 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:57:42 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 58BB2C8DF33D62B5 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4233474 18708200 2803 22 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:57:45 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 77D288D238143DC6 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4230605 18708200 13013 16 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:57:58 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 74212905FEB7883D REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 3920952 18708200 3448 26 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:01 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 060140E915EE005A REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4226689 18708200 3016 14 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:04 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 C2B6BD79E706A583 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4217974 18708200 2508 23 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:07 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 C4DB180DBBE7D2C0 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4227033 18708200 2666 224 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:10 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 6822B2F51B7390BF REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4229800 18708200 6314 21 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:16 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 5802133D0E709A69 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4169752 18708200 3117 21 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:19 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 7AEA132936905D0F REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4221938 18708200 3086 23 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:22 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 CFB5E41CD315DA46 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4116176 18708200 3547 21 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:26 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 2AA4F763E5B2DB4C REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 3134615 18708200 141 41 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:33 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 E9441C619413CE17 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 2546843 18708200 266 19 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:37 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 B74D95B8EF02DA55 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 1861439 18708200 53 19 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:41 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 0B9744B24E0B5359 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 1273664 18708200 74 40 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:44 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 DA589AAA642D33D7 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 783806 18708200 38 18 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:48 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 F533853F09AC5567 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 3428645 18708200 94 34 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:51 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 69FE22F49D5F6741 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4232818 18708200 3758 43 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:58:55 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 C34AFF770557888B REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4308854 18708200 5224 20 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:59:00 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 F59924F875AC10CF REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4238246 18708200 5691 70 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:59:06 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 D612FB39D6C637D3 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4280986 18708200 3307 14 "-" "WinampMPEG/5.56" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:59:09 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 D35647E03C769516 REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4205237 18708200 10806 28 "-" "WinampMPEG/5.56" -

data/spec/ralf/clf_translator_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+require 'ralf'
+require 'ralf/bucket'
+
+describe Ralf::ClfTranslator do
+
+  it "should return nil when it's an invalid line" do
+    $stderr = StringIO.new
+    translated_object = Ralf::ClfTranslator.new('invalid_line')
+    translated_object.to_s.should be_nil
+    $stderr.string.should eql("# ERROR: invalid_line\n")
+  end
+
+  it "should translate a amazon logline to a Apache CombinedLogFile" do
+    aws_line = '2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [03/Nov/2010:15:57:29 +0000] 84.82.12.240 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 AE5C72112DFB80DE REST.GET.OBJECT 10122150/2010-10-31-0930.mp3 "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.0" 200 - 4215272 18708200 5165 47 "-" "WinampMPEG/5.56, Ultravox/2.1" -'
+    clf_line = '84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:57:29 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.0" 200 4215272 "-" "WinampMPEG/5.56, Ultravox/2.1" 5'
+    translated_object = Ralf::ClfTranslator.new(aws_line)
+    translated_object.to_s.should eql(clf_line)
+  end
+
+  it "should mark invalid lines with '# ERROR: '" do
+    $stderr = StringIO.new
+    invalid_line = "this is an invalid log line"
+    Ralf::ClfTranslator.new(invalid_line)
+    $stderr.string.should eql("# ERROR: #{invalid_line}\n")
+  end
+  
+  describe "#option{:recalculate_partial_content}" do
+    describe "set to false (default)" do
+      it "should add rounded total_time in seconds" do
+        $stderr = StringIO.new
+        input_line = '2010-10-02-19-15-45-6879098C3140BE9D:2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [02/Oct/2010:18:29:16 +0000] 82.168.113.55 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 4F911681022807C6 REST.GET.OBJECT 10028050/2010-09-26-1830.mp3 "GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4194304 17537676 1600 12 "-" "VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team" -'
+        Ralf::ClfTranslator.new(input_line).to_s.should eql("82.168.113.55 - 2cf7e6b063 [02/Oct/2010:18:29:16 +0000] \"GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1\" 206 4194304 \"-\" \"VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team\" 2")
+        $stderr.string.should be_empty
+      end
+    end
+    describe "set to true" do
+      it "should add rounded total_time in seconds" do
+        $stderr = StringIO.new
+        input_line = '2010-10-02-19-15-45-6879098C3140BE9D:2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [02/Oct/2010:18:29:16 +0000] 82.168.113.55 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 4F911681022807C6 REST.GET.OBJECT 10028050/2010-09-26-1830.mp3 "GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4194304 17537676 1600 12 "-" "VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team" -'
+        Ralf::ClfTranslator.new(input_line, :recalculate_partial_content => true).to_s.should eql("82.168.113.55 - 2cf7e6b063 [02/Oct/2010:18:29:16 +0000] \"GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1\" 206 135872 \"-\" \"VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team\" 2")
+        $stderr.string.should be_empty
+      end
+    end
+  end
+
+ [
+    ['2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:17:01 +0000] 10.32.219.38 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 784FD457838EFF42 REST.GET.ACL - "GET /?acl HTTP/1.1" 200 - 1384 - 399 - "-" "Jakarta Commons-HttpClient/3.0" -',
+     '10.32.219.38 - 3272ee65a9 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 1384 "-" "Jakarta Commons-HttpClient/3.0" 0'],
+    ['2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:17:02 +0000] 10.32.219.38 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 6E239BC5A4AC757C SOAP.PUT.OBJECT logs/2010-02-10-07-17-02-F6EFD00DAB9A08B6 "POST /soap/ HTTP/1.1" 200 - 797 686 63 31 "-" "Axis/1.3" -',
+     '10.32.219.38 - 3272ee65a9 [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 797 "-" "Axis/1.3" 0'],
+    ['2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:24:40 +0000] 10.217.37.15 - 0B76C90B3634290B REST.GET.ACL - "GET /?acl HTTP/1.1" 307 TemporaryRedirect 488 - 7 - "-" "Jakarta Commons-HttpClient/3.0" -',
+     '10.217.37.15 - - [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 488 "-" "Jakarta Commons-HttpClient/3.0" 0'],
+    ['2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.staging.kerkdienstgemist.nl [17/Sep/2010:13:38:36 +0000] 85.113.244.146 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 71F7D2AAA93B0A05 REST.COPY.OBJECT_GET 10010150/2010-08-29-0930.mp3 - 200 - - 13538337 - - - - -',
+     '85.113.244.146 - 2cf7e6b063 [17/Sep/2010:13:38:36 +0000] "POST /10010150/2010-08-29-0930.mp3 HTTP/1.1" 200 - "-" "REST.COPY.OBJECT_GET" 0'],
+    ['2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.staging.kerkdienstgemist.nl [17/Sep/2010:13:38:37 +0000] 85.113.244.146 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 7CC5E3D09AE78CAE REST.COPY.OBJECT_GET 10010150/2010-09-05-1000.mp3 - 200 - - 9860402 - - - - -',
+     '85.113.244.146 - 2cf7e6b063 [17/Sep/2010:13:38:37 +0000] "POST /10010150/2010-09-05-1000.mp3 HTTP/1.1" 200 - "-" "REST.COPY.OBJECT_GET" 0'],
+  ].each do |aws_line,clf_line|
+    it "converts #{aws_line} into #{clf_line}" do
+      Ralf::ClfTranslator.new(aws_line).to_s.should eql(clf_line)
+    end
+  end
+end

data/spec/ralf/config_spec.rb

@@ -128,4 +128,15 @@ describe Ralf::Config do
     $stdout.string.should eql("Warning: invalid configuration variable: unknown_variable\n")
   end
 
+  describe "#translate_options" do
+    it "sets {:recalculate_partial_content => false} by default" do
+      config = Ralf::Config.new
+      config.translate_options.should eql({:recalculate_partial_content => false})
+    end
+    it "sets {:recalculate_partial_content => true}" do
+      config = Ralf::Config.new(:translate_options => {:recalculate_partial_content => true})
+      config.translate_options.should eql({:recalculate_partial_content => true})
+    end
+  end
+
 end

data/spec/ralf_spec.rb

@@ -52,14 +52,13 @@ describe Ralf do
       :cache_dir   => '/var/log/amazon/ralf_cache/:year/:month/:bucket',
     }.merge(@aws_credentials)
     
-    # File = mock('File')
+  end
+
+  before(:each) do
     @s3_mock = mock('s3_mock')
     Ralf::Bucket.s3 = @s3_mock
-
     @example_buckets = load_example_bucket_mocks
-  end
 
-  before(:each) do
     RightAws::S3.should_receive(:new).any_number_of_times.and_return(@s3_mock)
   end
 
@@ -283,7 +282,7 @@ describe Ralf do
       end
 
       Ralf.should_receive(:convert_to_common_log_format).with(
-        "/var/log/s3/test1.log.alf", "/var/log/s3/test1.log")
+        "/var/log/s3/test1.log.alf", "/var/log/s3/test1.log", {:recalculate_partial_content => false})
       
       LogMerge::Merger.should_receive(:merge).with(alfio,
         *@example_buckets['test1'].keys.map{|k| "/var/log/s3/cache/test1/#{k.name.gsub('logs/', '')}"})
@@ -324,11 +323,11 @@ describe Ralf do
 EOF_INPUT
 
       clf_log =<<EOF_OUTPUT
-10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 1384 "-" "Jakarta Commons-HttpClient/3.0" 0
-10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 797 "-" "Axis/1.3" 0
+10.32.219.38 - 3272ee65a9 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 1384 "-" "Jakarta Commons-HttpClient/3.0" 0
+10.32.219.38 - 3272ee65a9 [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 797 "-" "Axis/1.3" 0
 10.217.37.15 - - [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 488 "-" "Jakarta Commons-HttpClient/3.0" 0
-85.113.244.146 - 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 [17/Sep/2010:13:38:36 +0000] "POST /10010150/2010-08-29-0930.mp3 HTTP/1.1" 200 - "-" "REST.COPY.OBJECT_GET" 0
-85.113.244.146 - 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 [17/Sep/2010:13:38:37 +0000] "POST /10010150/2010-09-05-1000.mp3 HTTP/1.1" 200 - "-" "REST.COPY.OBJECT_GET" 0
+85.113.244.146 - 2cf7e6b063 [17/Sep/2010:13:38:36 +0000] "POST /10010150/2010-08-29-0930.mp3 HTTP/1.1" 200 - "-" "REST.COPY.OBJECT_GET" 0
+85.113.244.146 - 2cf7e6b063 [17/Sep/2010:13:38:37 +0000] "POST /10010150/2010-09-05-1000.mp3 HTTP/1.1" 200 - "-" "REST.COPY.OBJECT_GET" 0
 EOF_OUTPUT
 
       output_log = StringIO.new
@@ -339,22 +338,48 @@ EOF_OUTPUT
       # $stderr.should_receive(:puts).with("# ERROR: 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.staging.kerkdienstgemist.nl [17/Sep/2010:13:38:36 +0000] 85.113.244.146 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 71F7D2AAA93B0A05 REST.COPY.OBJECT_GET 10010150/2010-08-29-0930.mp3 - 200 - - 13538337 - - - - -\n")
       # $stderr.should_receive(:puts).with("# ERROR: 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.staging.kerkdienstgemist.nl [17/Sep/2010:13:38:37 +0000] 85.113.244.146 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 7CC5E3D09AE78CAE REST.COPY.OBJECT_GET 10010150/2010-09-05-1000.mp3 - 200 - - 9860402 - - - - -\n")
 
-      Ralf.convert_to_common_log_format('input_file', 'output_file')
+      Ralf.convert_to_common_log_format('input_file', 'output_file', {})
       output_log.string.should eql(clf_log)
     end
 
-    it "should mark invalid lines with '# ERROR: '" do
-      $stderr = StringIO.new
-      invalid_line = "this is an invalid log line"
-      Ralf.translate_to_clf(invalid_line)
-      $stderr.string.should eql("# ERROR: #{invalid_line}\n")
-    end
+    it "should estimate Actual Bytes Sent of transfers from Bytes Sent and Total Time" do
+      output_log = StringIO.new
+      winamp_log = File.open(File.join(File.dirname(__FILE__), 'fixtures', 'winamp.txt'), 'r')
+
+      File.should_receive(:open).with('output_file', 'w').and_return(output_log)
+      File.should_receive(:open).with('input_file',  'r').and_yield(winamp_log)
+
+      Ralf.convert_to_common_log_format('input_file', 'output_file', {:recalculate_partial_content => true})
+
+      expected_clf_log =<<EOF_OUTPUT
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:57:29 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.0" 200 4215272 "-" "WinampMPEG/5.56, Ultravox/2.1" 5
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:57:34 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 144098 "-" "WinampMPEG/5.56" 4
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:57:38 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 142454 "-" "WinampMPEG/5.56" 4
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:57:42 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 139481 "-" "WinampMPEG/5.56" 3
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:57:45 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 170111 "-" "WinampMPEG/5.56" 13
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:57:58 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 141416 "-" "WinampMPEG/5.56" 3
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:01 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 140120 "-" "WinampMPEG/5.56" 3
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:04 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 138596 "-" "WinampMPEG/5.56" 3
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:07 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 139070 "-" "WinampMPEG/5.56" 3
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:10 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 150014 "-" "WinampMPEG/5.56" 6
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:16 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 140423 "-" "WinampMPEG/5.56" 3
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:19 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 140330 "-" "WinampMPEG/5.56" 3
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:22 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 141713 "-" "WinampMPEG/5.56" 4
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:26 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 131495 "-" "WinampMPEG/5.56" 0
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:33 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 131870 "-" "WinampMPEG/5.56" 0
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:37 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 131231 "-" "WinampMPEG/5.56" 0
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:41 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 131294 "-" "WinampMPEG/5.56" 0
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:44 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 131186 "-" "WinampMPEG/5.56" 0
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:48 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 131354 "-" "WinampMPEG/5.56" 0
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:51 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 142346 "-" "WinampMPEG/5.56" 4
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:58:55 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 146744 "-" "WinampMPEG/5.56" 5
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:59:00 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 148145 "-" "WinampMPEG/5.56" 6
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:59:06 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 140993 "-" "WinampMPEG/5.56" 3
+84.82.12.240 - 2cf7e6b063 [03/Nov/2010:15:59:09 +0000] "GET /10122150/2010-10-31-0930.mp3?Signature=5n2%2B8hrDvgSbP6OJRP1vVav42uU%3D&Expires=1288807041&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 163490 "-" "WinampMPEG/5.56" 11
+EOF_OUTPUT
 
-    it "should add rounded total_time in seconds" do
-      $stderr = StringIO.new
-      input_line = '2010-10-02-19-15-45-6879098C3140BE9D:2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [02/Oct/2010:18:29:16 +0000] 82.168.113.55 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 4F911681022807C6 REST.GET.OBJECT 10028050/2010-09-26-1830.mp3 "GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4194304 17537676 1600 12 "-" "VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team" -'
-      Ralf.translate_to_clf(input_line).should eql("82.168.113.55 - 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 [02/Oct/2010:18:29:16 +0000] \"GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1\" 206 4194304 \"-\" \"VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team\" 2")
-      $stderr.string.should be_empty
+      output_log.string.should == expected_clf_log
+      winamp_log.close
     end
 
   end

data/spec/spec_helper.rb

@@ -1,14 +1,11 @@
 require 'rubygems'
+require 'bundler/setup'
+
 require 'right_aws' # load RightHttpConnection before FakeWeb otherwise we get buggy
 require 'fakeweb'
 
-require 'spec/autorun'
+RSpec.configure do |config|
 
-Spec::Runner.configure do |config|
-  # == Notes
-  #
-  # For more information take a look at Spec::Runner::Configuration and Spec::Runner
-  
   def load_example_bucket_mocks
     buckets = YAML.load(File.open(File.join(File.dirname(__FILE__), 'fixtures', 'example_buckets.yaml')))
     buckets = buckets.inject({}) do |memo, info|
@@ -20,5 +17,5 @@ Spec::Runner.configure do |config|
     end
     buckets
   end
-end
 
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ralf
 version: !ruby/object:Gem::Version 
-  hash: 19
-  prerelease: false
+  hash: 17
+  prerelease: 
   segments: 
   - 1
   - 1
-  - 0
-  version: 1.1.0
+  - 1
+  version: 1.1.1
 platform: ruby
 authors: 
 - Klaas Jan Wierenga
@@ -16,8 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-09 00:00:00 +02:00
-default_executable: ralf
+date: 2013-02-06 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
@@ -27,79 +26,93 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 27
+        hash: 7
         segments: 
-        - 1
-        - 3
-        - 0
-        version: 1.3.0
+        - 2
+        version: "2"
   type: :development
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: fakeweb
+  name: autotest
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 15
+        hash: 35
         segments: 
-        - 1
-        - 2
-        - 8
-        version: 1.2.8
+        - 4
+        - 4
+        - 6
+        version: 4.4.6
   type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: right_aws
+  name: fakeweb
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 63
+        hash: 27
         segments: 
         - 1
-        - 10
+        - 3
         - 0
-        version: 1.10.0
-  type: :runtime
+        version: 1.3.0
+  type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: logmerge
+  name: right_aws
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 19
+        hash: 15
         segments: 
-        - 1
+        - 3
         - 0
-        - 2
-        version: 1.0.2
+        - 4
+        version: 3.0.4
   type: :runtime
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: chronic
+  name: logmerge
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
         hash: 17
         segments: 
+        - 1
         - 0
-        - 2
         - 3
-        version: 0.2.3
+        version: 1.0.3
   type: :runtime
   version_requirements: *id005
-description: "      Download logfiles from Amazon S3 buckets to local disk and combine them in one Apache CLF per bucket\n"
+- !ruby/object:Gem::Dependency 
+  name: chronic
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 59
+        segments: 
+        - 0
+        - 9
+        - 0
+        version: 0.9.0
+  type: :runtime
+  version_requirements: *id006
+description: " Download logfiles from Amazon S3 buckets to local disk and combine them in one Apache CLF per bucket "
 email: 
 - k.j.wierenga@gmail.com
 - leonb@beriedata.nl
@@ -110,21 +123,28 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
+- .gitignore
+- .rspec
 - .rvmrc
+- Gemfile
+- Gemfile.lock
 - README.rdoc
 - Rakefile
-- VERSION
 - bin/ralf
 - lib/ralf.rb
 - lib/ralf/bucket.rb
+- lib/ralf/clf_translator.rb
 - lib/ralf/config.rb
 - lib/ralf/interpolation.rb
 - lib/ralf/log.rb
 - lib/ralf/option_parser.rb
+- lib/ralf/version.rb
 - ralf.gemspec
-- spec/fixtures/apache.log
+- spec/fixtures/apache.txt
 - spec/fixtures/example_buckets.yaml
+- spec/fixtures/winamp.txt
 - spec/ralf/bucket_spec.rb
+- spec/ralf/clf_translator_spec.rb
 - spec/ralf/config_spec.rb
 - spec/ralf/interpolation_spec.rb
 - spec/ralf/log_spec.rb
@@ -133,7 +153,6 @@ files:
 - spec/spec.opts
 - spec/spec_helper.rb
 - spec/support/fakeweb.rb
-has_rdoc: true
 homepage: http://github.com/kjwierenga/ralf
 licenses: []
 
@@ -164,9 +183,21 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: Retrieve Amazon Log Files
-test_files: []
-
+test_files: 
+- spec/fixtures/apache.txt
+- spec/fixtures/example_buckets.yaml
+- spec/fixtures/winamp.txt
+- spec/ralf/bucket_spec.rb
+- spec/ralf/clf_translator_spec.rb
+- spec/ralf/config_spec.rb
+- spec/ralf/interpolation_spec.rb
+- spec/ralf/log_spec.rb
+- spec/ralf/option_parser_spec.rb
+- spec/ralf_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/support/fakeweb.rb

data/VERSION

@@ -1 +0,0 @@
-1.1.0