ralf 0.1.5 → 1.0.0

data/spec/ralf_spec.rb

@@ -5,274 +5,342 @@ require 'ralf'
 describe Ralf do
 
   before(:all) do
-    @key1 = {:name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT', :data => 'This is content for key 1'}
-    @key2 = {:name => 'log/access_log-2010-02-10-00-07-28-EFREUTERGRSGDH', :data => 'This is content for key 2'}
-    @key3 = {:name => 'log/access_log-2010-02-11-00-09-32-SDHTFTFHDDDDDH', :data => 'This is content for key 3'}
+    # make sure we don't accidentally use actual credentials during test
+    ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'] = nil, nil
+    
+    @key1 = {
+      :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT',
+      :data => 'This is content for key 1'
+    }
+    @key2 = {
+      :name => 'log/access_log-2010-02-10-00-07-28-EFREUTERGRSGDH',
+      :data => 'This is content for key 2'
+    }
+    @key3 = {
+      :name => 'log/access_log-2010-02-11-00-09-32-SDHTFTFHDDDDDH',
+      :data => 'This is content for key 3'
+    }
+    
+    @aws_credentials = {
+      :aws_access_key_id     => 'the_aws_access_key_id',
+      :aws_secret_access_key => 'the_secret_access_key',
+    }
+    
+    @valid_options = {
+      :config_file => '',
+      :output_file => './ralf/:year/:month/:day/:bucket.log',
+    }.merge(@aws_credentials)
+    
+    @cli_config_path = 'my_ralf.conf'
+    @cli_config = {
+      :range       => '2010-02-10',
+      :output_file => './ralf/:year/:month/:day/:bucket.log',
+      :cache_dir   => '/tmp/ralf_cache/:bucket',
+    }.merge(@aws_credentials)
+
+    @tilde_config_path = File.expand_path('~/.ralf.conf')
+    @tilde_config = {
+      :range       => '2010-02-11',
+      :output_file => '~/ralf/:year/:month/:day/:bucket.log',
+      :cache_dir   => '~/ralf/cache/:bucket',
+    }.merge(@aws_credentials)
+
+    @etc_config_path = '/etc/ralf.conf'
+    @etc_config = {
+      :range       => '2010-02-12',
+      :output_file => '/var/log/amazon/:year/:month/:day/:bucket.log',
+      :cache_dir   => '/var/log/amazon/ralf_cache/:year/:month/:bucket',
+    }.merge(@aws_credentials)
+    
+    # File = mock('File')
+    @s3_mock = mock('s3_mock')
+    Ralf::Bucket.s3 = @s3_mock
+
+    @example_buckets = load_example_bucket_mocks
   end
 
   before(:each) do
-    @default_params = {
-      :config => File.dirname(__FILE__) + '/fixtures/config.yaml',
-      :out_seperator => ':year/:month/:day',
-      :date => '2010-02-10'
-    }
+    RightAws::S3.should_receive(:new).any_number_of_times.and_return(@s3_mock)
   end
 
-  it "should initialize properly" do
-    ralf = Ralf.new(@default_params)
-    ralf.class.should eql(Ralf)
-  end
+  describe "Configuration Options" do
 
-  describe "Preferences" do
+    it "should initialize properly" do
+      YAML.should_not_receive(:load_file)
+      ralf = Ralf.new({:output_file => 'here', :config_file => ''}.merge(@aws_credentials))
+      ralf.class.should eql(Ralf)
+    end
+    
+    it "should read config file specified on command-line" do
+      YAML.should_receive(:load_file).with(@cli_config_path).and_return(@cli_config)
+      YAML.should_not_receive(:load_file).with(@tilde_config_path)
+      YAML.should_not_receive(:load_file).with(@etc_config_path)
+      ralf = Ralf.new(:config_file => @cli_config_path)
+      ralf.config.should == Ralf::Config.new(@cli_config)
+    end
+    
+    it "should read config file from '~/.ralf.conf' if not running as root." do
+      Process.should_receive(:uid).and_return(1)
+      File.should_receive(:exist?).with(@tilde_config_path).and_return(true)
+      YAML.should_receive(:load_file).with(@tilde_config_path).and_return(@tilde_config)
+      ralf = Ralf.new
+      ralf.config.should == Ralf::Config.new(@tilde_config)
+    end
+    
+    it "should read config file from '/etc/ralf.conf' if running as root." do
+      Process.should_receive(:uid).and_return(0)
+      File.should_receive(:exist?).with(@etc_config_path).and_return(true)
+      YAML.should_receive(:load_file).with(@etc_config_path).and_return(@etc_config)
+      ralf = Ralf.new
+      ralf.config.should == Ralf::Config.new(@etc_config)
+    end
 
+    it "should read config file from '~/.ralf.conf' if running as root and /etc/ralf doesn't exist." do
+      Process.should_receive(:uid).and_return(0)
+      File.should_receive(:exist?).with(@etc_config_path).and_return(true)
+      YAML.should_receive(:load_file).with(@etc_config_path).and_return(@etc_config)
+      ralf = Ralf.new
+      ralf.config.should == Ralf::Config.new(@etc_config)
+    end
+    
+    it "should have only required option when :config_file is empty string" do
+      YAML.should_not_receive(:load_file)
+      File.should_not_receive(:exist?)
+      ralf = Ralf.new({ :config_file => ''}.merge(@aws_credentials))
+      ralf.config.should == Ralf::Config.new(@aws_credentials)
+    end
+    
+    it "command-line options should override config file options" do
+      File.should_receive(:exist?).with(@tilde_config_path).and_return(true)
+      YAML.should_receive(:load_file).with(@tilde_config_path).and_return(@tilde_config)
+      ralf = Ralf.new(@cli_config)
+      ralf.config.should == Ralf::Config.new(@cli_config)
+    end
+  
     it "should raise an error when an nonexistent config file is given" do
+      missing_file = 'the_missing_file.conf'
+      File.should_receive(:open).with(missing_file).and_raise(Errno::ENOENT)
       lambda {
-        ralf = Ralf.new(:config => '~/a_non_existen_file.yaml')
-      }.should  raise_error(Ralf::NoConfigFile)
+        ralf = Ralf.new(:config_file => missing_file)
+      }.should raise_error(Errno::ENOENT)
     end
 
     it "should set the preferences" do
-      ralf = Ralf.new(@default_params)
-      ralf.config[:aws_access_key_id].should      eql('access_key')
-      ralf.config[:aws_secret_access_key].should  eql('secret')
-      ralf.config[:out_path].should               eql('/Users/berl/S3')
-      # ralf.config.should eql({:aws_access_key_id => 'access_key', :aws_secret_access_key => 'secret'})
+      YAML.should_not_receive(:load_file)
+      ralf = Ralf.new(@cli_config.merge(:config_file => ''))
+      ralf.config.should == Ralf::Config.new(@cli_config)
     end
-
-    it "should look for default configurations" do
-      File.should_receive(:expand_path).once.with('~/.ralf.yaml').and_return('/Users/berl/.ralf.yaml')
-      File.should_receive(:expand_path).twice.with('/Users/berl/.ralf.yaml').and_return('/Users/berl/.ralf.yaml')
-      File.should_receive(:expand_path).once.with('/etc/ralf.yaml').and_return('/etc/ralf.yaml')
-      File.should_receive(:exists?).once.with('/etc/ralf.yaml').and_return(false)
-      File.should_receive(:exists?).twice.with('/Users/berl/.ralf.yaml').and_return(true)
-      YAML.should_receive(:load_file).with('/Users/berl/.ralf.yaml').and_return({
-        :aws_access_key_id      => 'access_key',
-        :aws_secret_access_key  => 'secret',
-        :out_path               => '/Users/berl/S3',
-        :out_prefix             => 's3_combined'
-      })
-
-      ralf = Ralf.new()
+    
+    it "should raise Ralf::Config::ConfigurationError when --output-file not specified" do
+      YAML.should_not_receive(:load_file)
+      lambda {
+        Ralf.new(:output_file => nil, :config_file => '')
+      }.should raise_error(Ralf::Config::ConfigurationError)
     end
 
     it "should use AWS credentials provided in ENV" do
-      ENV['AWS_ACCESS_KEY_ID']     = 'access_key'
-      ENV['AWS_SECRET_ACCESS_KEY'] = 'secret'
-      File.should_receive(:exists?).once.with('/etc/ralf.yaml').and_return(false)
-      File.should_receive(:exists?).once.with('/Users/berl/.ralf.yaml').and_return(false)
-
+      YAML.should_not_receive(:load_file)
       lambda {
-        Ralf.new(:out_path => '/Users/berl/S3')
-      }.should_not raise_error(Ralf::ConfigIncomplete)
+        Ralf.new(:config_file => '')
+      }.should raise_error(Ralf::Config::ConfigurationError, 'aws_access_key_id missing, aws_secret_access_key missing')
 
+      ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'] = 'aws_access_key', 'secret'
+      lambda {
+        Ralf.new(:config_file => '')
+      }.should_not raise_error(Ralf::Config::ConfigurationError)
+      
+      # reset
+      ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'] = nil, nil
     end
 
   end
 
-  describe "Date handling" do
+  describe "Range handling" do
+    
+    it "should set range to today if unspecified" do
+      YAML.should_not_receive(:load_file)
+      now = Time.now
+      Time.should_receive(:now).any_number_of_times.and_return(now)
+      ralf = Ralf.new(@valid_options)
+      date = now.strftime("%Y-%m-%d")
 
-    it "should set the date to today" do
-      ralf = Ralf.new()
-      date = Date.today
-      ralf.date.should  eql("%4d-%02d-%02d" % [date.year, date.month, date.day])
+      ralf.config.range.to_s.should eql("#{date}..#{date}")
     end
 
-    it "should set the date to the date given" do
-      ralf = Ralf.new(@default_params.merge(:date => '2010-02-01'))
-      ralf.date.should  eql('2010-02-01')
+    it "should set the range when single date given" do
+      YAML.should_not_receive(:load_file)
+      ralf = Ralf.new(@valid_options.merge(:range => '2010-02-01'))
+      ralf.config.range.to_s.should eql('2010-02-01..2010-02-01')
     end
 
     it "should raise error when invalid date given" do
+      YAML.should_not_receive(:load_file)
       lambda {
-        ralf = Ralf.new(@default_params.merge(:date => 'someday'))
-        ralf.date.should  be_nil
-      }.should raise_error(Ralf::InvalidDate, "someday is an invalid value.")
+        ralf = Ralf.new(@valid_options.merge(:range => 'someday'))
+        ralf.range.should be_nil
+      }.should raise_error(Ralf::Config::RangeError, "invalid expression 'someday'")
     end
 
     it "should accept a range of 2 dates" do
-      ralf = Ralf.new(@default_params.merge(:date => nil, :range => ['2010-02-10', '2010-02-12']))
-      ralf.range.to_s.should eql('2010-02-10..2010-02-12')
+      YAML.should_not_receive(:load_file)
+      ralf = Ralf.new(@valid_options.merge(:range => ['2010-02-10', '2010-02-12']))
+      ralf.config.range.to_s.should eql('2010-02-10..2010-02-12')
+    end
+    
+    it "should raise error for range array with more than 2 items" do
+      YAML.should_not_receive(:load_file)
+      lambda {
+        ralf = Ralf.new(@valid_options.merge(:range => ['2010-02-10', '2010-02-12', '2010-02-13']))
+      }.should raise_error(ArgumentError, 'too many range items')
     end
 
-    it "should accept a range starting with 1 date" do
-      Date.should_receive(:today).any_number_of_times.and_return(Date.strptime('2010-02-17'))
-      ralf = Ralf.new(@default_params.merge(:date => nil, :range => '2010-02-10'))
-      ralf.range.to_s.should eql('2010-02-10..2010-02-17')
+    it "should treat a range with 1 date as a single date" do
+      YAML.should_not_receive(:load_file)
+      ralf = Ralf.new(@valid_options.merge(:range => '2010-02-10'))
+      ralf.config.range.to_s.should eql('2010-02-10..2010-02-10')
+    end
 
-      ralf = Ralf.new(@default_params.merge(:date => nil, :range => ['2010-02-10']))
-      ralf.range.to_s.should eql('2010-02-10..2010-02-17')
+    it "should accept a range array with 1 date" do
+      YAML.should_not_receive(:load_file)
+      ralf = Ralf.new(@valid_options.merge(:range => ['2010-02-10']))
+      ralf.config.range.to_s.should eql('2010-02-10..2010-02-10')
     end
 
     it "should accept a range defined by words" do
-      Date.should_receive(:today).any_number_of_times.and_return(Date.strptime('2010-02-17'))
-      Chronic.should_receive(:parse).once.with('2 days ago', {:guess=>false, :context=>:past}).and_return(
-        Chronic::Span.new(Time.parse('Mon Feb 15 00:00:00 +0100 2010'),Time.parse('Tue Feb 16 00:00:00 +0100 2010'))
-      )
-
-      ralf = Ralf.new(@default_params.merge(:date => nil, :range => '2 days ago'))
-      ralf.range.to_s.should eql('2010-02-15..2010-02-17')
+      YAML.should_not_receive(:load_file)
+      Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Mon Feb 17 09:41:00 +0100 2010'))
+      ralf = Ralf.new(@valid_options.merge(:range => '2 days ago'))
+      ralf.config.range.to_s.should eql('2010-02-15..2010-02-15')
     end
 
     it "should accept a month and convert it to a range" do
-      ralf = Ralf.new(@default_params.merge(:date => nil, :range => 'january'))
-      ralf.range.to_s.should  eql('2010-01-01..2010-01-31')
+      YAML.should_not_receive(:load_file)
+      Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Mon Feb 17 09:41:00 +0100 2010'))
+      ralf = Ralf.new(@valid_options.merge(:range => 'january'))
+      ralf.config.range.to_s.should  eql('2010-01-01..2010-01-31')
     end
-
-  end
-
-  describe "Handle Buckets" do
-
-    before(:each) do
-      @ralf = Ralf.new(@default_params)
-      @bucket1 = {:name => 'bucket1'}
-      @bucket1.should_receive(:logging_info).any_number_of_times.and_return({ :enabled => true, :targetprefix => "log/access_log-", :targetbucket => @bucket1[:name] })
-      @bucket1.should_receive(:name).any_number_of_times.and_return(@bucket1[:name])
-      @bucket2 = {:name => 'bucket2'}
-      @bucket2.should_receive(:logging_info).any_number_of_times.and_return({ :enabled => false, :targetprefix => "log/", :targetbucket => @bucket2[:name] })
-      @bucket2.should_receive(:name).any_number_of_times.and_return(@bucket2[:name])
+    
+    it "should allow 'this month' with base 'yesterday'" do
+      YAML.should_not_receive(:load_file)
+      Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
+      ralf = Ralf.new(@valid_options.merge(:range => 'this month', :now => 'yesterday'))
+      ralf.config.range.to_s.should eql('2010-04-01..2010-04-30')
     end
-
-    it "should find buckets with logging enabled" do
-      @ralf.s3.should_receive(:buckets).once.and_return([@bucket1, @bucket2])
-
-      @ralf.find_buckets_with_logging.should  eql([@bucket1, @bucket2])
-      @ralf.buckets_with_logging.should       eql([@bucket1])
+    
+    it "should support setting range first then change now (1st day of month)" do
+      YAML.should_not_receive(:load_file)
+      Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
+      ralf = Ralf.new(@valid_options.merge(:range => 'this month'))
+      ralf.config.range.to_s.should eql('2010-05-01..2010-05-01')
+      ralf.config.merge!(:now => 'yesterday')
+      ralf.config.range.to_s.should eql('2010-04-01..2010-04-30')
     end
 
-    it "should return the new organized path" do
-      @key1.should_receive(:name).and_return(@key1[:name])
-      @ralf.s3_organized_log_file(@bucket1, @key1).should eql('log/2010/02/10/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
+    it "should support setting range first then change now" do
+      YAML.should_not_receive(:load_file)
+      Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 08 16:31:00 +0100 2010'))
+      ralf = Ralf.new(@valid_options.merge(:range => 'this month'))
+      ralf.config.range.to_s.should eql('2010-05-01..2010-05-07')
+      ralf.config.merge!(:now => '2010-05-06')
+      ralf.config.range.to_s.should eql('2010-05-01..2010-05-06')
     end
 
-    describe "logging" do
-
-      before(:each) do
-        @key1.should_receive(:name).any_number_of_times.and_return(@key1[:name])
-        @key2.should_receive(:name).any_number_of_times.and_return(@key2[:name])
-        @key1.should_receive(:data).any_number_of_times.and_return(@key1[:data])
-        @key2.should_receive(:data).any_number_of_times.and_return(@key2[:data])
-      end
-
-      it "should save logging to disk" do
-        @bucket1.should_receive(:keys).any_number_of_times.and_return([@key1, @key2])
-
-        File.should_receive(:makedirs).twice.with('/Users/berl/S3/bucket1/log/2010/02/10').and_return(true)
-        File.should_receive(:exists?).once.with(  '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-07-28-EFREUTERGRSGDH').and_return(true)
-        File.should_receive(:exists?).once.with(  '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT').and_return(false)
-        File.should_receive(:open).once.with(     '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT', "w").and_return(true)
-
-        @ralf.save_logging_to_local_disk(@bucket1, '2010-02-10').should eql([@key1, @key2])
-      end
-
-      it "should save logging for range to disk" do
-        @bucket1.should_receive(:keys).any_number_of_times.and_return([@key1, @key2], [@key3], [])
-        @key3.should_receive(:name).any_number_of_times.and_return(@key3[:name])
-        @key3.should_receive(:data).any_number_of_times.and_return(@key3[:data])
-
-        @ralf.date = nil
-        @ralf.range = ['2010-02-10', '2010-02-12']
+  end
 
-        File.should_receive(:exists?).once.with(  '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT').and_return(false)
-        File.should_receive(:exists?).once.with(  '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-07-28-EFREUTERGRSGDH').and_return(true)
-        File.should_receive(:exists?).once.with(  '/Users/berl/S3/bucket1/log/2010/02/11/access_log-2010-02-11-00-09-32-SDHTFTFHDDDDDH').and_return(false)
-        File.should_receive(:open).once.with(     '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT', "w").and_return(true)
-        File.should_receive(:open).once.with(     '/Users/berl/S3/bucket1/log/2010/02/11/access_log-2010-02-11-00-09-32-SDHTFTFHDDDDDH', "w").and_return(true)
+  describe "Handle Buckets" do
 
-        @ralf.save_logging(@bucket1).class.should  eql(Range)
+    it "should download, merge and convert logfiles" do
+      YAML.should_not_receive(:load_file)
+      @s3_mock.should_receive(:bucket).any_number_of_times do |name|
+        @example_buckets[name]
       end
-
-      it "should save logging if a different targetbucket is given" do
-        @ralf.s3.should_receive(:bucket).and_return(@bucket1)
-        @bucket3 = {:name => 'bucket3'}
-        @bucket3.should_receive(:logging_info).any_number_of_times.and_return({ :enabled => false, :targetprefix => "log/", :targetbucket => 'bucket1' })
-        @bucket3.should_receive(:name).any_number_of_times.and_return(@bucket3[:name])
-        @bucket1.should_receive(:keys).any_number_of_times.and_return([@key1, @key2])
-
-        @ralf.save_logging_to_local_disk(@bucket3, '2010-02-10').should eql([@key1, @key2])
+      
+      File.stub(:makedirs)
+      ralf = Ralf.new(@valid_options.merge(:cache_dir   => '/var/log/s3/cache/:bucket',
+                                           :output_file => '/var/log/s3/:bucket.log',
+                                           :buckets     => 'test1',
+                                           :range       => ['2010-02-01', '2010-02-12']))
+      
+      alfio = StringIO.new
+      File.should_receive(:open).with('/var/log/s3/test1.log.alf', 'w').and_yield(alfio)
+      
+      Ralf.should_receive(:download_logs).any_number_of_times do |bucket, date, dir|
+        expected_bucket = Ralf::Bucket.new(@example_buckets['test1'])
+        [ :name, :logging_enabled?, :targetbucket, :targetprefix ].each do |attr|
+          bucket.send(attr).should eql(expected_bucket.send(attr))
+        end
+        ralf.config.range.should include(date)
+        dir.should eql("/var/log/s3/cache/test1")
+        
+        log_files = []
+        case date
+        when Date.new(2010,02,10)
+          log_files << '/var/log/s3/cache/test1/2010-02-10-00-05-32-ZDRFGTCKUYVJCT'
+        when Date.new(2010,02,11)
+          log_files << '/var/log/s3/cache/test1/2010-02-11-00-05-32-ZDRFGTCKUYVJCT'
+        end
+        log_files
       end
 
+      Ralf.should_receive(:convert_to_common_log_format).with(
+        "/var/log/s3/test1.log.alf", "/var/log/s3/test1.log")
+      
+      LogMerge::Merger.should_receive(:merge).with(alfio,
+        *@example_buckets['test1'].keys.map{|k| "/var/log/s3/cache/test1/#{k.name.gsub('logs/', '')}"})
+      
+      ralf.run()
     end
-
-    it "should merge all logs" do
-      out_string = StringIO.new
-
-      Dir.should_receive(:glob).with('/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10*').and_return(
-          ['/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT',
-           '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-07-28-EFREUTERGRSGDH'])
-
-      File.should_receive(:open).with('/Users/berl/S3/s3_combined_bucket1_2010-02-10.alf', "w").and_yield(out_string)
-
-      LogMerge::Merger.should_receive(:merge).with(
-        out_string, 
-        '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT',
-        '/Users/berl/S3/bucket1/log/2010/02/10/access_log-2010-02-10-00-07-28-EFREUTERGRSGDH'
-      )
-
-      @ralf.merge_to_combined(@bucket1)
-
-      out_string.string.should eql('')
-    end
-
-    it "should save logs which have a targetprefix containing a '/'" do
-      @ralf.local_log_dirname(@bucket1).should  eql('/Users/berl/S3/bucket1/log/2010/02/10')
-      @ralf.local_log_dirname(@bucket2).should  eql('/Users/berl/S3/bucket2/log/2010/02/10')
-    end
-
-    it "should save to a subdir when a out_seperator is given" do
-      @ralf.local_log_dirname(@bucket1).should  eql('/Users/berl/S3/bucket1/log/2010/02/10')
-
-      @ralf.out_seperator = ':year/w:week'
-      @ralf.local_log_dirname(@bucket1).should  eql('/Users/berl/S3/bucket1/log/2010/w06')
+    
+    it "should raise error when output_file option is missing" do
+      YAML.should_not_receive(:load_file)
+      ralf = Ralf.new(@aws_credentials.merge(:config_file => ''))
+      lambda {
+        ralf.run
+      }.should raise_error(ArgumentError, "--output-file required")
     end
-
-    it "should get the proper directories" do
-      @key1.should_receive(:name).and_return('log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
-      @ralf.local_log_file_basename_prefix(@bucket1).should   eql('access_log-')
-      @ralf.local_log_file_basename(@bucket1, @key1).should   eql('access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
-      @ralf.local_log_dirname(@bucket1).should                eql('/Users/berl/S3/bucket1/log/2010/02/10')
-
-      @key1.should_receive(:name).and_return('log/2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
-      @ralf.local_log_file_basename_prefix(@bucket2).should   eql('')
-      @ralf.local_log_file_basename(@bucket2, @key1).should   eql('2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
-      @ralf.local_log_dirname(@bucket2).should                eql('/Users/berl/S3/bucket2/log/2010/02/10')
+    
+    it "should raise error when output_file option requires :bucket variable" do
+      YAML.should_not_receive(:load_file)
+      ralf = Ralf.new(@aws_credentials.merge(:output_file => '/tmp/ralf/ralf.log', :config_file => ''))
+      lambda {
+        ralf.run
+      }.should raise_error(ArgumentError, "--output-file requires ':bucket' variable")
     end
 
   end
 
-  describe "Conversion" do
-
-    before(:each) do
-      @ralf    = Ralf.new(@default_params)
-      @bucket1 = {:name => 'bucket1'}
-      @bucket1.should_receive(:name).any_number_of_times.and_return('bucket1')
-    end
-
-    it "should convert the alf to clf" do
-      File.should_receive(:open).once.with("/Users/berl/S3/s3_combined_bucket1_2010-02-10.log", "w").and_return(File)
-      File.should_receive(:open).once.with("/Users/berl/S3/s3_combined_bucket1_2010-02-10.alf", "r").and_return(File)
-      File.should_receive(:close).once.and_return(true)
-      @ralf.convert_alt_to_clf(@bucket1).should eql(true)
-    end
-
-    it "should find the proper values in a line" do
-      [ [
-        '2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:17:01 +0000] 10.32.219.38 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 784FD457838EFF42 REST.GET.ACL - "GET /?acl HTTP/1.1" 200 - 1384 - 399 - "-" "Jakarta Commons-HttpClient/3.0" -                        ',
-        '10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 1384 "-" "Jakarta Commons-HttpClient/3.0"'
-      ],[
-        '2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:17:02 +0000] 10.32.219.38 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 6E239BC5A4AC757C SOAP.PUT.OBJECT logs/2010-02-10-07-17-02-F6EFD00DAB9A08B6 "POST /soap/ HTTP/1.1" 200 - 797 686 63 31 "-" "Axis/1.3" -',
-        '10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 797 "-" "Axis/1.3"'
-      ],[
-        '2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:24:40 +0000] 10.217.37.15 - 0B76C90B3634290B REST.GET.ACL - "GET /?acl HTTP/1.1" 307 TemporaryRedirect 488 - 7 - "-" "Jakarta Commons-HttpClient/3.0" -                                                                          ',
-        '10.217.37.15 - - [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 488 "-" "Jakarta Commons-HttpClient/3.0"'
-      ] ].each do |alf,clf|
-        @ralf.translate_to_clf(alf).should eql(clf)
-      end
+  describe "Conversion of Amazon Log Format to Common Log Format" do
+  
+    it "should convert output files to common_log_format" do
+      input_log = StringIO.new
+      input_log.string = <<EOF_INPUT
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:17:01 +0000] 10.32.219.38 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 784FD457838EFF42 REST.GET.ACL - "GET /?acl HTTP/1.1" 200 - 1384 - 399 - "-" "Jakarta Commons-HttpClient/3.0" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:17:02 +0000] 10.32.219.38 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 6E239BC5A4AC757C SOAP.PUT.OBJECT logs/2010-02-10-07-17-02-F6EFD00DAB9A08B6 "POST /soap/ HTTP/1.1" 200 - 797 686 63 31 "-" "Axis/1.3" -
+2cf7e6b06335c0689c6d29163df5bb001c96870cd78609e3845f1ed76a632621 assets.staging.kerkdienstgemist.nl [10/Feb/2010:07:24:40 +0000] 10.217.37.15 - 0B76C90B3634290B REST.GET.ACL - "GET /?acl HTTP/1.1" 307 TemporaryRedirect 488 - 7 - "-" "Jakarta Commons-HttpClient/3.0" -
+EOF_INPUT
+
+      clf_log =<<EOF_OUTPUT
+10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 1384 "-" "Jakarta Commons-HttpClient/3.0"
+10.32.219.38 - 3272ee65a908a7677109fedda345db8d9554ba26398b2ca10581de88777e2b61 [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 797 "-" "Axis/1.3"
+10.217.37.15 - - [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 488 "-" "Jakarta Commons-HttpClient/3.0"
+EOF_OUTPUT
+
+      output_log = StringIO.new
+      
+      File.should_receive(:open).with('input_file', 'r').and_yield(input_log)
+      File.should_receive(:open).with('output_file', 'w').and_return(output_log)
+
+      Ralf.convert_to_common_log_format('input_file', 'output_file')
+      output_log.string.should eql(clf_log)
     end
 
     it "should mark invalid lines with '# ERROR: '" do
-      @ralf.translate_to_clf('An invalid line in the logfile').should match(/^# ERROR/)
+      $stderr = StringIO.new
+      invalid_line = "this is an invalid log line"
+      Ralf.translate_to_clf(invalid_line)
+      $stderr.string.should eql("# ERROR: #{invalid_line}\n")
     end
 
   end
-
+  
 end

data/spec/spec.opts

@@ -0,0 +1,5 @@
+--colour
+--format specdoc
+--loadby mtime
+--drb
+--backtrace

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+require 'rubygems'
+require 'right_aws' # load RightHttpConnection before FakeWeb otherwise we get buggy
+require 'fakeweb'
+
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  # == Notes
+  #
+  # For more information take a look at Spec::Runner::Configuration and Spec::Runner
+  
+  def load_example_bucket_mocks
+    buckets = YAML.load(File.open(File.join(File.dirname(__FILE__), 'fixtures', 'example_buckets.yaml')))
+    buckets = buckets.inject({}) do |memo, info|
+      info[:keys].map! do |key|
+        mock(key[:name], key)
+      end
+      memo.merge!(info[:name] => mock(info[:name], info))
+      memo
+    end
+    buckets
+  end
+end
+

data/spec/support/fakeweb.rb

@@ -0,0 +1,4 @@
+
+FakeWeb.allow_net_connect = false
+
+FakeWeb.register_uri(:any, 'https://s3.amazonaws.com:443', :body => '')