ralf 1.0.0 → 1.1.0

data/.rvmrc

@@ -0,0 +1 @@
+rvm --create ruby-1.8.7-p249@ralf

data/README.rdoc

@@ -123,4 +123,6 @@ install the ralf gem.
 = Authors
 Authors: {Leon Berenschot}[http://github.com/LeipeLeon] and {K.J. Wierenga}[http://github.com/kjwierenga]
 
+Contributers: {Victor Castell}[http://github.com/victorcoder]
+
 This program is used for {kerkdienstgemist.nl}[http://kerkdienstgemist.nl] {Amazon S3}[http://aws.amazon.com/s3/] log file processing.

data/Rakefile

@@ -15,11 +15,11 @@ begin
     gem.homepage = "http://github.com/kjwierenga/ralf"
     gem.authors = ["Klaas Jan Wierenga", "Leon Berenschot"]
 
-    gem.add_development_dependency 'rspec',   '>= 1.3.0'
-    gem.add_development_dependency 'fakeweb', '>= 1.2.8'
+    gem.add_development_dependency 'rspec',   '~> 1.3.0'
+    gem.add_development_dependency 'fakeweb', '~> 1.2.8'
 
-    gem.add_dependency 'right_aws', '>= 1.10.0'
-    gem.add_dependency 'logmerge',  '>= 1.0.2'
+    gem.add_dependency 'right_aws', '~> 1.10.0'
+    gem.add_dependency 'logmerge',  '~> 1.0.2'
     gem.add_dependency 'chronic',   '>= 0.2.3'
 
     gem.rdoc_options << '--exclude' << '.'
@@ -27,24 +27,30 @@ begin
   end
   Jeweler::GemcutterTasks.new
 rescue LoadError
-  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+  puts "Warning: Jeweler (or a dependency) not available."
+  puts "         Check dependencies with `rake check_dependencies`."
 end
 
-require 'spec/rake/spectask'
-Spec::Rake::SpecTask.new(:spec) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.spec_files = FileList['spec/**/*_spec.rb']
-end
+begin
+  require 'spec/rake/spectask'
+  Spec::Rake::SpecTask.new(:spec) do |spec|
+    spec.libs << 'lib' << 'spec'
+    spec.spec_files = FileList['spec/**/*_spec.rb']
+  end
 
-Spec::Rake::SpecTask.new(:rcov) do |spec|
-  spec.libs << 'lib' << 'spec'
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
+  Spec::Rake::SpecTask.new(:rcov) do |spec|
+    spec.libs << 'lib' << 'spec'
+    spec.pattern = 'spec/**/*_spec.rb'
+    spec.rcov = true
+  end
 
-task :spec => :check_dependencies
+  task :spec => :check_dependencies
 
-task :default => :spec
+  task :default => :spec
+rescue LoadError
+  puts "Warning: Rspec not available."
+  puts "         Check dependencies with `rake check_dependencies`."
+end
 
 require 'rake/rdoctask'
 Rake::RDocTask.new do |rdoc|

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0

data/lib/ralf.rb

@@ -1,10 +1,12 @@
 require 'rubygems'
 require 'right_aws'
 require 'logmerge'
-require 'ftools'
+require 'fileutils'
 require 'ralf/config'
 require 'ralf/bucket'
 require 'chronic'
+require 'stringio'
+require 'date'
 
 class Ralf
 
@@ -17,6 +19,7 @@ class Ralf
   USER_DEFAULT_CONFIG_FILE = '~/.ralf.conf'
 
   AMAZON_LOG_FORMAT = Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" "([^"]*)"')
+  AMAZON_LOG_FORMAT_COPY = Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) (REST.COPY.OBJECT_GET) ([^ ]*) (-) ([^ ]*) (-) (-) ([^ ]*) (-) (-) (-) (-) (-)')
   
   # The current configuration.
   attr_reader :config
@@ -81,7 +84,7 @@ class Ralf
       merged_log =  output_log + ".alf"
 
       # create directory for output file
-      File.makedirs(File.dirname(merged_log))
+      FileUtils.mkdir_p(File.dirname(merged_log))
 
       # merge the log files
       Ralf.merge(merged_log, log_files)
@@ -109,7 +112,7 @@ class Ralf
   
   # Download log files for +bucket+ and +date+ to +dir+.
   def self.download_logs(bucket, date, dir)
-    File.makedirs(dir)
+    FileUtils.mkdir_p(dir)
 
     # iterate over the available log files, saving them to disk and 
     log_files = []
@@ -134,7 +137,9 @@ class Ralf
     out_file = File.open(output_log, 'w')
     File.open(input_log, 'r') do |in_file|
       while (line = in_file.gets)
-        out_file.puts(translate_to_clf(line))
+        if clf = translate_to_clf(line)
+          out_file.puts(clf)
+        end
       end
     end
     out_file.close
@@ -142,10 +147,13 @@ class Ralf
 
   def self.translate_to_clf(line)
     if line =~ AMAZON_LOG_FORMAT
-      # host, date, ip, acl, request, status, bytes, agent = $2, $3, $4, $5, $9, $10, $12, $17
-      "%s - %s [%s] \"%s\" %d %s \"%s\" \"%s\"" % [$4, $5, $3, $9, $10, $12, $16, $17]
+      # host, date, ip, acl, request, status, bytes, agent, total_time_ms = $2, $3, $4, $5, $9, $10, $12, $17, $14
+      "%s - %s [%s] \"%s\" %s %s \"%s\" \"%s\" %d" % [$4, $5, $3, $9, $10, $12, $16, $17, ($14.to_i/1000.0).round]
+    elsif line =~ AMAZON_LOG_FORMAT_COPY
+      "%s - %s [%s] \"%s\" %s %s \"%s\" \"REST.COPY.OBJECT_GET\" %d" % [$4, $5, $3, "POST /#{$8} HTTP/1.1", $10, $12, $16, 0]
     else
       $stderr.puts "# ERROR: #{line}"
+      nil
     end
   end
 

data/lib/ralf/config.rb

@@ -1,4 +1,5 @@
 require 'ralf/interpolation'
+require 'yaml'
 
 class Ralf::Config
   

data/ralf.gemspec

@@ -0,0 +1,77 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{ralf}
+  s.version = "1.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Klaas Jan Wierenga", "Leon Berenschot"]
+  s.date = %q{2011-05-09}
+  s.default_executable = %q{ralf}
+  s.description = %q{      Download logfiles from Amazon S3 buckets to local disk and combine them in one Apache CLF per bucket
+}
+  s.email = ["k.j.wierenga@gmail.com", "leonb@beriedata.nl"]
+  s.executables = ["ralf"]
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".rvmrc",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/ralf",
+    "lib/ralf.rb",
+    "lib/ralf/bucket.rb",
+    "lib/ralf/config.rb",
+    "lib/ralf/interpolation.rb",
+    "lib/ralf/log.rb",
+    "lib/ralf/option_parser.rb",
+    "ralf.gemspec",
+    "spec/fixtures/apache.log",
+    "spec/fixtures/example_buckets.yaml",
+    "spec/ralf/bucket_spec.rb",
+    "spec/ralf/config_spec.rb",
+    "spec/ralf/interpolation_spec.rb",
+    "spec/ralf/log_spec.rb",
+    "spec/ralf/option_parser_spec.rb",
+    "spec/ralf_spec.rb",
+    "spec/spec.opts",
+    "spec/spec_helper.rb",
+    "spec/support/fakeweb.rb"
+  ]
+  s.homepage = %q{http://github.com/kjwierenga/ralf}
+  s.rdoc_options = ["--exclude", "."]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{Retrieve Amazon Log Files}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, ["~> 1.3.0"])
+      s.add_development_dependency(%q<fakeweb>, ["~> 1.2.8"])
+      s.add_runtime_dependency(%q<right_aws>, ["~> 1.10.0"])
+      s.add_runtime_dependency(%q<logmerge>, ["~> 1.0.2"])
+      s.add_runtime_dependency(%q<chronic>, [">= 0.2.3"])
+    else
+      s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+      s.add_dependency(%q<fakeweb>, ["~> 1.2.8"])
+      s.add_dependency(%q<right_aws>, ["~> 1.10.0"])
+      s.add_dependency(%q<logmerge>, ["~> 1.0.2"])
+      s.add_dependency(%q<chronic>, [">= 0.2.3"])
+    end
+  else
+    s.add_dependency(%q<rspec>, ["~> 1.3.0"])
+    s.add_dependency(%q<fakeweb>, ["~> 1.2.8"])
+    s.add_dependency(%q<right_aws>, ["~> 1.10.0"])
+    s.add_dependency(%q<logmerge>, ["~> 1.0.2"])
+    s.add_dependency(%q<chronic>, [">= 0.2.3"])
+  end
+end
+

data/spec/ralf_spec.rb

@@ -255,7 +255,7 @@ describe Ralf do
         @example_buckets[name]
       end
       
-      File.stub(:makedirs)
+      FileUtils.stub(:mkdir_p)
       ralf = Ralf.new(@valid_options.merge(:cache_dir   => '/var/log/s3/cache/:bucket',
                                            :output_file => '/var/log/s3/:bucket.log',
                                            :buckets     => 'test1',
@@ -313,22 +313,31 @@ describe Ralf do
   
     it "should convert output files to common_log_format" do
       input_log = StringIO.new
+      
+      # last two lines are ignored because they don't match
       input_log.string = <<EOF_INPUT
 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:17:01 +0000] 10.32.219.38 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 784FD457838EFF42 REST.GET.ACL - "GET /?acl HTTP/1.1" 200 - 1384 - 399 - "-" "Jakarta Commons-HttpClient/3.0" -
 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:17:02 +0000] 10.32.219.38 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 6E239BC5A4AC757C SOAP.PUT.OBJECT logs/2010-02-10-07-17-02-F6EFD00DAB9A08B6 "POST /soap/ HTTP/1.1" 200 - 797 686 63 31 "-" "Axis/1.3" -
 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:24:40 +0000] 10.217.37.15 - 0B76C90B3634290B REST.GET.ACL - "GET /?acl HTTP/1.1" 307 TemporaryRedirect 488 - 7 - "-" "Jakarta Commons-HttpClient/3.0" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.staging.kerkdienstgemist.nl [17/Sep/2010:13:38:36 +0000] 85.113.244.146 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 71F7D2AAA93B0A05 REST.COPY.OBJECT_GET 10010150/2010-08-29-0930.mp3 - 200 - - 13538337 - - - - -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.staging.kerkdienstgemist.nl [17/Sep/2010:13:38:37 +0000] 85.113.244.146 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 7CC5E3D09AE78CAE REST.COPY.OBJECT_GET 10010150/2010-09-05-1000.mp3 - 200 - - 9860402 - - - - -
 EOF_INPUT
 
       clf_log =<<EOF_OUTPUT
-10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 1384 "-" "Jakarta Commons-HttpClient/3.0"
-10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 797 "-" "Axis/1.3"
-10.217.37.15 - - [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 488 "-" "Jakarta Commons-HttpClient/3.0"
+10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 1384 "-" "Jakarta Commons-HttpClient/3.0" 0
+10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 797 "-" "Axis/1.3" 0
+10.217.37.15 - - [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 488 "-" "Jakarta Commons-HttpClient/3.0" 0
+85.113.244.146 - 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 [17/Sep/2010:13:38:36 +0000] "POST /10010150/2010-08-29-0930.mp3 HTTP/1.1" 200 - "-" "REST.COPY.OBJECT_GET" 0
+85.113.244.146 - 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 [17/Sep/2010:13:38:37 +0000] "POST /10010150/2010-09-05-1000.mp3 HTTP/1.1" 200 - "-" "REST.COPY.OBJECT_GET" 0
 EOF_OUTPUT
 
       output_log = StringIO.new
       
       File.should_receive(:open).with('input_file', 'r').and_yield(input_log)
       File.should_receive(:open).with('output_file', 'w').and_return(output_log)
+      
+      # $stderr.should_receive(:puts).with("# ERROR: 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.staging.kerkdienstgemist.nl [17/Sep/2010:13:38:36 +0000] 85.113.244.146 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 71F7D2AAA93B0A05 REST.COPY.OBJECT_GET 10010150/2010-08-29-0930.mp3 - 200 - - 13538337 - - - - -\n")
+      # $stderr.should_receive(:puts).with("# ERROR: 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.staging.kerkdienstgemist.nl [17/Sep/2010:13:38:37 +0000] 85.113.244.146 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 7CC5E3D09AE78CAE REST.COPY.OBJECT_GET 10010150/2010-09-05-1000.mp3 - 200 - - 9860402 - - - - -\n")
 
       Ralf.convert_to_common_log_format('input_file', 'output_file')
       output_log.string.should eql(clf_log)
@@ -341,6 +350,13 @@ EOF_OUTPUT
       $stderr.string.should eql("# ERROR: #{invalid_line}\n")
     end
 
+    it "should add rounded total_time in seconds" do
+      $stderr = StringIO.new
+      input_line = '2010-10-02-19-15-45-6879098C3140BE9D:2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 media.kerkdienstgemist.nl [02/Oct/2010:18:29:16 +0000] 82.168.113.55 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 4F911681022807C6 REST.GET.OBJECT 10028050/2010-09-26-1830.mp3 "GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1" 206 - 4194304 17537676 1600 12 "-" "VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team" -'
+      Ralf.translate_to_clf(input_line).should eql("82.168.113.55 - 2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 [02/Oct/2010:18:29:16 +0000] \"GET /10028050/2010-09-26-1830.mp3?Signature=E3ehd6nkXjNg7vr%2F4b3LtxCWads%3D&Expires=1286051333&AWSAccessKeyId=AKIAI3XHXJPFSJW2UQAQ HTTP/1.1\" 206 4194304 \"-\" \"VLC media player - version 1.0.5 Goldeneye - (c) 1996-2010 the VideoLAN team\" 2")
+      $stderr.string.should be_empty
+    end
+
   end
   
 end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ralf
 version: !ruby/object:Gem::Version 
+  hash: 19
   prerelease: false
   segments: 
   - 1
+  - 1
   - 0
-  - 0
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors: 
 - Klaas Jan Wierenga
@@ -15,16 +16,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-21 00:00:00 +02:00
+date: 2011-05-09 00:00:00 +02:00
 default_executable: ralf
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
+        hash: 27
         segments: 
         - 1
         - 3
@@ -36,9 +39,11 @@ dependencies:
   name: fakeweb
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
+        hash: 15
         segments: 
         - 1
         - 2
@@ -50,9 +55,11 @@ dependencies:
   name: right_aws
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
+        hash: 63
         segments: 
         - 1
         - 10
@@ -64,9 +71,11 @@ dependencies:
   name: logmerge
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
+        hash: 19
         segments: 
         - 1
         - 0
@@ -78,9 +87,11 @@ dependencies:
   name: chronic
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 17
         segments: 
         - 0
         - 2
@@ -99,7 +110,7 @@ extensions: []
 extra_rdoc_files: 
 - README.rdoc
 files: 
-- .gitignore
+- .rvmrc
 - README.rdoc
 - Rakefile
 - VERSION
@@ -110,6 +121,7 @@ files:
 - lib/ralf/interpolation.rb
 - lib/ralf/log.rb
 - lib/ralf/option_parser.rb
+- ralf.gemspec
 - spec/fixtures/apache.log
 - spec/fixtures/example_buckets.yaml
 - spec/ralf/bucket_spec.rb
@@ -127,38 +139,34 @@ licenses: []
 
 post_install_message: 
 rdoc_options: 
-- --charset=UTF-8
 - --exclude
 - .
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Retrieve Amazon Log Files
-test_files: 
-- spec/ralf/bucket_spec.rb
-- spec/ralf/config_spec.rb
-- spec/ralf/interpolation_spec.rb
-- spec/ralf/log_spec.rb
-- spec/ralf/option_parser_spec.rb
-- spec/ralf_spec.rb
-- spec/spec_helper.rb
-- spec/support/fakeweb.rb
+test_files: []
+

data/.gitignore

@@ -1,4 +0,0 @@
-**.gem
-tmp/**
-ralf.gemspec
-rdoc