ralf 0.1.5 → 1.0.0

data/lib/ralf.rb

@@ -0,0 +1,173 @@
+require 'rubygems'
+require 'right_aws'
+require 'logmerge'
+require 'ftools'
+require 'ralf/config'
+require 'ralf/bucket'
+require 'chronic'
+
+class Ralf
+
+  private
+  RLIMIT_NOFILE_HEADROOM = 100 # number of file descriptors to allocate above number of logfiles
+
+  public
+
+  ROOT_DEFAULT_CONFIG_FILE = '/etc/ralf.conf'
+  USER_DEFAULT_CONFIG_FILE = '~/.ralf.conf'
+
+  AMAZON_LOG_FORMAT = Regexp.new('([^ ]*) ([^ ]*) \[([^\]]*)\] ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) "([^"]*)" "([^"]*)"')
+  
+  # The current configuration.
+  attr_reader :config
+  
+  # Instance of RightAws::S3 used by Ralf to talk to Amazon S3.
+  attr_reader :s3
+  
+  # Create instance and run with the specified parameters
+  def self.run(options)
+    list = options.delete(:list)
+
+    ralf = Ralf.new(options)
+    if list
+      ralf.list
+    else
+      ralf.run
+    end
+  end
+  
+  # Create new Ralf instance
+  def initialize(options = {})
+    initial_options = options.dup
+
+    @config = load_config(initial_options.delete(:config_file))
+    
+    config.merge!(initial_options)
+    config.validate!
+    
+    RightAws::RightAwsBaseInterface.caching = true # enable caching to speed up
+    Bucket.s3 = RightAws::S3.new(config.aws_access_key_id, config.aws_secret_access_key,
+      :protocol => 'http', :port => 80,
+      :logger => Logger.new('aws' == config.debug? ? $stdout : StringIO.new)
+    )
+  end
+
+  # For all buckets for all dates in the configured range download the available
+  # log files. After downloading, merge the log files and convert the format
+  # from Amazon Log Format to Apache Common Log Format.
+  def run(output_file = nil)
+    config.output_file = output_file unless output_file.nil?
+    
+    raise ArgumentError.new("--output-file required") if config.output_file_missing?
+    raise ArgumentError.new("--output-file requires ':bucket' variable") if (config.buckets.nil? || config.buckets.size > 1) and !(config.output_file_format =~ /:bucket/)
+    
+    puts "Processing range #{config.range}" if config.debug?
+
+    # iterate over all buckets
+    Bucket.each(config.buckets) do |bucket|
+      
+      print "#{bucket.name}: " if config.debug?
+
+      # iterate over the full range
+      log_files = []
+      config.range.each do |date|
+        dir = config.cache_dir(:bucket => bucket.name, :date => date)
+        log_files << Ralf.download_logs(bucket, date, dir)
+      end
+      log_files.flatten!
+
+      # determine output file names
+      output_log = config.output_file(:date => config.range.end, :bucket => bucket.name)
+      merged_log =  output_log + ".alf"
+
+      # create directory for output file
+      File.makedirs(File.dirname(merged_log))
+
+      # merge the log files
+      Ralf.merge(merged_log, log_files)
+      
+      # convert to common log format
+      Ralf.convert_to_common_log_format(merged_log, output_log)
+
+      puts "#{log_files.size} files" if config.debug?
+    end
+  end
+  
+  # List all buckets with the logging info.
+  def list(with_logging = false)
+    puts "Listing buckets..." if config.debug?
+    
+    Bucket.each(config.buckets, with_logging) do |bucket|
+      print "#{bucket.name}"
+      puts bucket.logging_enabled? ? " [#{bucket.targetbucket}/#{bucket.targetprefix}]" : " [-]"
+    end
+
+    nil
+  end
+
+  private
+  
+  # Download log files for +bucket+ and +date+ to +dir+.
+  def self.download_logs(bucket, date, dir)
+    File.makedirs(dir)
+
+    # iterate over the available log files, saving them to disk and 
+    log_files = []
+    bucket.each_log(date) do |log|
+      log_files << log.save_to_dir(dir)
+    end
+    log_files
+  end
+  
+  # Takes an array of log file names and merges them on ascending timestamp
+  # into +output_file+ name. Assumes the +log_files+ are sorted
+  # on ascending timestamp.
+  def self.merge(output_file, log_files)
+    update_rlimit_nofile(log_files.size)
+    File.open(output_file, 'w') do |out_file|
+      LogMerge::Merger.merge out_file, *log_files
+    end
+  end
+
+  # Convert the input_log file to Apache Common Log Format into output_log
+  def self.convert_to_common_log_format(input_log, output_log)
+    out_file = File.open(output_log, 'w')
+    File.open(input_log, 'r') do |in_file|
+      while (line = in_file.gets)
+        out_file.puts(translate_to_clf(line))
+      end
+    end
+    out_file.close
+  end
+
+  def self.translate_to_clf(line)
+    if line =~ AMAZON_LOG_FORMAT
+      # host, date, ip, acl, request, status, bytes, agent = $2, $3, $4, $5, $9, $10, $12, $17
+      "%s - %s [%s] \"%s\" %d %s \"%s\" \"%s\"" % [$4, $5, $3, $9, $10, $12, $16, $17]
+    else
+      $stderr.puts "# ERROR: #{line}"
+    end
+  end
+
+  def load_config(cli_config_file)
+    result = nil
+    if cli_config_file
+      result = Ralf::Config.load_file(cli_config_file) unless cli_config_file.empty?
+    else
+      config_file = (0 == Process.uid ? ROOT_DEFAULT_CONFIG_FILE : File.expand_path(USER_DEFAULT_CONFIG_FILE))
+      result = Ralf::Config.load_file(config_file) if File.exist?(config_file)
+    end
+    result || Ralf::Config.new
+  end
+  
+  def self.update_rlimit_nofile(number_of_files)
+    new_rlimit_nofile = number_of_files + RLIMIT_NOFILE_HEADROOM
+
+    # getrlimit returns array with soft and hard limit [soft, hard]
+    rlimit_nofile = Process::getrlimit(Process::RLIMIT_NOFILE)
+    if new_rlimit_nofile > rlimit_nofile.first
+      Process.setrlimit(Process::RLIMIT_NOFILE, new_rlimit_nofile) rescue nil
+    end 
+  end
+  
+end

data/spec/fixtures/apache.log

@@ -0,0 +1,7 @@
+10.34.211.35 - F7CECCD7A21B4B96 [10/Feb/2010:07:16:19 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 456 - "-" "Jakarta Commons-HttpClient/3.0" -
+10.34.211.35 - E415BCEA05AFB84F [10/Feb/2010:07:16:19 +0000] "POST /soap/ HTTP/1.1" 200 - 797 1003 74 41 "-" "Axis/1.3" -
+10.36.142.18 - BE89D3B6953D719E [10/Feb/2010:07:16:28 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 433 - "-" "Jakarta Commons-HttpClient/3.0" -
+10.36.142.18 - E15F489437E852EA [10/Feb/2010:07:16:28 +0000] "POST /soap/ HTTP/1.1" 200 - 797 954 36 14 "-" "Axis/1.3" -
+10.32.219.38 - 784FD457838EFF42 [10/Feb/2010:07:17:01 +0000] "GET /?acl HTTP/1.1" 200 - 1384 - 399 - "-" "Jakarta Commons-HttpClient/3.0" -
+10.32.219.38 - 6E239BC5A4AC757C [10/Feb/2010:07:17:02 +0000] "POST /soap/ HTTP/1.1" 200 - 797 686 63 31 "-" "Axis/1.3" -
+10.217.37.15 - 0B76C90B3634290B [10/Feb/2010:07:24:40 +0000] "GET /?acl HTTP/1.1" 307 TemporaryRedirect 488 - 7 - "-" "Jakarta Commons-HttpClient/3.0" -

data/spec/fixtures/example_buckets.yaml

@@ -0,0 +1,48 @@
+--- 
+-
+  :name: 'test1' 
+  :logging_info: 
+    :enabled: true
+    :targetbucket: test1
+    :targetprefix: logs/
+  :keys: 
+  -
+    :name: 'logs/2010-02-10-00-05-32-ZDRFGTCKUYVJCT'
+  -
+    :name: 'logs/2010-02-11-00-05-32-ZDRFGTCKUYVJCT'
+-
+  :name: 'test2' 
+  :logging_info: 
+    :enabled: true
+    :targetbucket: target_test2
+    :targetprefix: logs/access_log-
+  :keys: 
+  -
+    :name: 'logs/access_log-2010-02-12-00-05-32-ZDRFGTCKUYVJCT'
+  -
+    :name: 'logs/access_log-2010-02-13-00-05-32-ZDRFGTCKUYVJCT'
+-
+  :name: 'test3' 
+  :logging_info: 
+    :enabled: false
+    :targetbucket: ''
+    :targetprefix: ''
+  :keys: []
+-
+  :name: 'test4' 
+  :logging_info: 
+    :enabled: true
+    :targetbucket: test4
+    :targetprefix: logs/
+  :keys:
+  - 
+    :name: 'logs/2010-02-14-00-05-32-ZDRFGTCKUYVJCT'
+  -
+    :name: 'logs/2010-02-15-00-05-32-ZDRFGTCKUYVJCT'
+-
+  :name: 'target_test2' 
+  :logging_info: 
+    :enabled: false
+    :targetbucket: ''
+    :targetprefix: ''
+  :keys: []

data/spec/ralf/bucket_spec.rb

@@ -0,0 +1,119 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+require 'ralf'
+require 'ralf/bucket'
+
+describe Ralf::Bucket do
+  
+  before(:all) do
+    @valid_logging_info = {
+      :enabled      => true,
+      :targetbucket => 'targetbucket',
+      :targetprefix => 'logs/',
+    }
+  end
+  
+  it "should initialize properly" do
+    mock_S3
+    
+    name = 's3_bucket'
+    logging_info = {
+      :enabled      => true,
+      :targetbucket => 's3_bucket',
+      :targetprefix => 'logs/',
+    }
+    bucket = mock(name)
+    bucket.should_receive(:logging_info).and_return(logging_info)
+    bucket.should_receive(:name).and_return(name)
+    
+    lambda {
+      Ralf::Bucket.new(bucket)
+    }.should_not raise_error
+  end
+  
+  it "should set targetbucket to bucket returned by logging_info" do
+    mock_S3
+    
+    name = 's3_bucket'
+    targetbucket_name = 's3_targetbucket'
+    logging_info = {
+      :enabled      => true,
+      :targetbucket => targetbucket_name,
+      :targetprefix => 'logs/',
+    }
+    bucket = mock(name)
+    bucket.should_receive(:logging_info).and_return(logging_info)
+    bucket.should_receive(:name).and_return(name)
+    
+    Ralf::Bucket.new(bucket)
+  end
+  
+  it "should support iteration over all buckets" do
+    mock_S3
+    
+    yielded_buckets = []
+    Ralf::Bucket.each do |bucket|
+      yielded_buckets << bucket
+    end
+    yielded_buckets.should have(@enabled_buckets_count).items
+    yielded_buckets.each { |bucket| bucket.name.should_not be_nil }
+  end
+
+  it "should support iteration over specific buckets" do
+    mock_S3
+    
+    yielded_buckets = []
+    Ralf::Bucket.each(@example_buckets.keys) do |bucket|
+      yielded_buckets << bucket
+    end
+    yielded_buckets.should have(@enabled_buckets_count).items
+    yielded_buckets.each do |bucket|
+      bucket.logging_enabled?.should eql(@example_buckets[bucket.name].logging_info[:enabled])
+      bucket.targetbucket.should eql(@example_buckets[bucket.name].logging_info[:targetbucket])
+      bucket.targetprefix.should eql(@example_buckets[bucket.name].logging_info[:targetprefix])
+    end
+  end
+  
+  it "should support iterating over all logs in a bucket" do
+    mock_S3
+    
+    bucket_mock = @example_buckets['test1']
+    key1, key2 = mock('key1'), mock('key2')
+    keys = [key1, key2]
+    bucket_mock.should_receive(:keys).with(:prefix => 'logs/2010-05-17').and_return(keys)
+
+    expected_logs = []
+    keys.each do |key|
+      log = mock('Log')
+      Ralf::Log.should_receive(:new).with(key, @example_buckets['test1'].logging_info[:targetprefix]).and_return(log)
+      expected_logs << log
+    end
+
+    bucket = Ralf::Bucket.new(bucket_mock)
+    
+    yielded_logs = []
+    bucket.each_log(Date.new(2010,05,17)) do |log|
+      yielded_logs << log
+    end
+    yielded_logs.should have(keys.size).items
+    yielded_logs.should eql(expected_logs)
+  end
+  
+  def mock_S3
+    @s3_mock = mock('S3')
+    Ralf::Bucket.s3 = @s3_mock
+    
+    # load example buckets (2 disabled)
+    @example_buckets = load_example_bucket_mocks
+    @enabled_buckets_count = @example_buckets.size - 2
+    
+    # make s3_mock return individual buckets
+    @s3_mock.should_receive(:bucket).any_number_of_times do |name|
+      @example_buckets[name]
+    end
+    
+    # make s3_mock return all buckets
+    @s3_mock.should_receive(:buckets).any_number_of_times.and_return(@example_buckets.values)
+  end
+  
+end

data/spec/ralf/config_spec.rb

@@ -0,0 +1,131 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+require 'ralf'
+require 'ralf/config'
+
+describe Ralf::Config do
+  
+  before(:all) do
+    ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'] = nil, nil
+    
+    @aws_credentials = {
+      :aws_access_key_id     => 'access_key_id',
+      :aws_secret_access_key => 'secret_key',
+    }
+    
+    @valid_options = {
+      :output_file           => 'my_log.log',
+    }.merge(@aws_credentials)
+
+    @date = Date.strptime('2010-02-09')
+    @bucket = 'my.bucket.org'
+  end
+  
+  it "should initialize properly" do
+    config = Ralf::Config.new
+  end
+  
+  it "should raise error for missing credentials" do
+    lambda {
+      config = Ralf::Config.new
+      config.validate!
+    }.should raise_error(Ralf::Config::ConfigurationError, 'aws_access_key_id missing, aws_secret_access_key missing')
+  end
+  
+  it "should handle range assignment" do
+    now = Time.now
+    yesterday = Date.today - 1
+    Time.should_receive(:now).any_number_of_times.and_return(now)
+    config = Ralf::Config.new(@valid_options)
+    config.range = 'yesterday'
+    config.range.should eql(Range.new(yesterday, yesterday))
+  end
+  
+  it "should interpolate date (:year, :month, :day) variables in output_file" do
+    config = Ralf::Config.new(@valid_options.merge(:output_file => ':year/:month/access.log'))
+    config.output_file(:date => @date).should eql('2010/02/access.log')
+  end
+
+  it "should interpolate :bucket variable in output_file" do
+    config = Ralf::Config.new(@valid_options.merge(:output_file => ':bucket.log'))
+    config.output_file(:date => @date, :bucket => @bucket).should eql('my.bucket.org.log')
+  end
+  
+  it "should interpolate variables in cache_dir" do
+    config = Ralf::Config.new(@valid_options.merge(:cache_dir => ':year/:month/:bucket'))
+    config.cache_dir(:date => @date, :bucket => @bucket).should eql('2010/02/my.bucket.org')
+  end
+  
+  it "should allow 'this month' with base 'yesterday'" do
+    Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
+    config = Ralf::Config.new(:range => 'this month', :now => 'yesterday')
+    config.range.to_s.should eql('2010-04-01..2010-04-30')
+  end
+  
+  it "should merge options" do
+    Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
+    config = Ralf::Config.new
+    config.merge!(:range => 'this month', :now => 'yesterday')
+    config.range.to_s.should eql('2010-04-01..2010-04-30')
+  end
+  
+  it "should support setting now after range and recompute range" do
+    Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
+    config = Ralf::Config.new
+    config.merge!(:range => 'this month')
+    config.merge!(:now => 'yesterday')
+    config.range.to_s.should eql('2010-04-01..2010-04-30')
+  end
+  
+  it "should support setting range first then change now (1st day of month)" do
+    Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 01 16:31:00 +0100 2010'))
+    config = Ralf::Config.new
+    config.merge!(:range => 'this month')
+    config.range.to_s.should eql('2010-05-01..2010-05-01')
+    config.merge!(:now => 'yesterday')
+    config.range.to_s.should eql('2010-04-01..2010-04-30')
+  end
+  
+  it "should support setting range first then change now" do
+    Time.should_receive(:now).any_number_of_times.and_return(Time.parse('Sat May 08 16:31:00 +0100 2010'))
+    config = Ralf::Config.new
+    config.merge!(:range => 'this month')
+    config.range.to_s.should eql('2010-05-01..2010-05-07')
+    config.merge!(:now => '2010-05-06')
+    config.range.to_s.should eql('2010-05-01..2010-05-06')
+  end
+  
+  it "should set default cache_dir to '/var/log/ralf/:bucket' when running as root" do
+    Process.should_receive(:uid).and_return(0)
+    config = Ralf::Config.new
+    config.cache_dir_format.should eql('/var/log/ralf/:bucket')
+  end
+  
+  it "should set default cache_dir to '~/.ralf/:bucket' when running as regular user" do
+    Process.should_receive(:uid).and_return(1)
+    File.should_receive(:expand_path).with('~/.ralf/:bucket').and_return('/Users/me/.ralf/:bucket')
+    config = Ralf::Config.new
+    config.cache_dir_format.should eql('/Users/me/.ralf/:bucket')
+  end
+  
+  it "should allow overriding bucket with merge!" do
+    config = Ralf::Config.new(:buckets => ['bucket1', 'bucket2'])
+    config.buckets = ['bucket3']
+    config.buckets.should eql(['bucket3'])
+  end
+  
+  it "should load configuration from file" do
+    YAML.should_receive(:load_file).with('my_config.yaml').and_return(@valid_options)
+    config = Ralf::Config.load_file('my_config.yaml')
+    config.output_file_format.should eql(@valid_options[:output_file])
+  end
+  
+  it "should warn and continue (not raise exception) when unknown configuration variables are set" do
+    $stdout = StringIO.new
+    lambda {
+      config = Ralf::Config.new(:unknown_variable => 100)
+    }.should_not raise_error
+    $stdout.string.should eql("Warning: invalid configuration variable: unknown_variable\n")
+  end
+
+end

data/spec/ralf/interpolation_spec.rb

@@ -0,0 +1,45 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+require 'ralf/interpolation'
+
+describe Ralf::Interpolation do
+  before(:all) do
+    @bucket = 'bucket.mybuckets.org'
+    @date = Date.strptime('2010-02-09')
+  end
+
+  [
+    [':year',   '2010'],
+    [':month',  '02'],
+    [':day',    '09'],
+    [':week',   '06']
+  ].each do |match, result, bucket|
+    it "should replace '#{match}' with '#{result}'" do
+      Ralf::Interpolation.interpolate(match, :date => @date).should  eql(result)
+    end
+  end
+
+  it "should raise an error when not all symbols could be interpolated" do
+    lambda {
+      Ralf::Interpolation.interpolate(':unknown', :date => @date)
+    }.should raise_error(Ralf::Interpolation::NotAllInterpolationsSatisfied)
+  end
+  
+  it "should interpolate :bucket" do
+    Ralf::Interpolation.interpolate(':year/:month/:day/:bucket.log',
+      :date => @date, :bucket => @bucket).should eql('2010/02/09/bucket.mybuckets.org.log')
+  end
+
+  it "should require :bucket interpolation if bucket specified" do
+    lambda {
+      Ralf::Interpolation.interpolate(':year/:month/:day.log', {:date => @date, :bucket => @bucket}, [:bucket])
+    }.should raise_error(Ralf::Interpolation::VariableMissing, ':bucket variable missing')
+  end
+  
+  it "should raise an error when :bucket can not be interpolated" do
+    lambda {
+      Ralf::Interpolation.interpolate(':bucket', :date => @date)
+    }.should raise_error(Ralf::Interpolation::NotAllInterpolationsSatisfied)
+  end
+
+end

data/spec/ralf/log_spec.rb

@@ -0,0 +1,63 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+require 'ralf'
+require 'ralf/log'
+
+describe Ralf::Log do
+
+  it "should initialize properly" do
+    key    = mock('key')
+    prefix = mock('prefix')
+    lambda {
+      Ralf::Log.new(key, prefix)
+    }.should_not raise_error
+  end
+  
+  it "should remove prefix when returing name" do
+    key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
+    log = Ralf::Log.new(key, 'log/access_log-')
+    log.name.should eql('2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
+  end
+  
+  it "should save specified file to dir" do
+    key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
+    log = Ralf::Log.new(key, 'log/access_log-')
+
+    dir      = '/var/log/s3'
+    filename = File.join(dir, log.name)
+    fileio   = mock('File')
+    key.should_receive(:data).and_return('testdata')
+    File.should_receive(:open).with(filename, 'w').and_yield(fileio)
+    File.should_receive(:exist?).with(filename).and_return(false)
+    fileio.should_receive(:write).with('testdata')
+    log.save_to_dir(dir)
+  end
+  
+  it "should not save file if it exists and caching enabled (default)" do
+    key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
+    log = Ralf::Log.new(key, 'log/access_log-')
+
+    dir      = '/var/log/s3'
+    filename = File.join(dir, log.name)
+    fileio   = mock('File')
+    File.should_receive(:exist?).with(filename).and_return(true)
+    File.should_not_receive(:open).with(filename, 'w')
+    log.save_to_dir(dir)
+  end
+  
+  it "should not check file if use_cache = false" do
+    key = mock('key', :name => 'log/access_log-2010-02-10-00-05-32-ZDRFGTCKUYVJCT')
+    log = Ralf::Log.new(key, 'log/access_log-')
+
+    dir      = '/var/log/s3'
+    filename = File.join(dir, log.name)
+    fileio   = mock('File')
+
+    key.should_receive(:data).and_return('testdata')
+    File.should_receive(:open).with(filename, 'w').and_yield(fileio)
+    File.should_not_receive(:exist?).with(filename)
+    fileio.should_receive(:write).with('testdata')
+    log.save_to_dir(dir, false)
+  end
+  
+end

data/spec/ralf/option_parser_spec.rb

@@ -0,0 +1,97 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+require 'ralf'
+require 'ralf/option_parser'
+
+describe Ralf::OptionParser do
+
+  before(:all) do
+    @valid_arguments = [
+      [ :buckets, '--buckets', [ 'bucket1.mydomain.net', 'bucket2.mydomain.net' ] ],
+      [ :buckets, '-b',        [ 'bucket1.mydomain.net', 'bucket2.mydomain.net' ] ],
+      
+      [ :range, '--range', ['today'] ],
+      [ :range, '-r',      ['today'] ],
+
+      [ :now, '--now', 'yesterday' ],
+      [ :now, '-t',    'yesterday' ],
+      
+      [ :output_file, '--output-file', '/var/log/s3/:year/:month/:bucket.log' ],
+      [ :output_file, '-o',            '/var/log/s3/:year/:month/:bucket.log' ],
+
+      [ :cache_dir, '--cache-dir', '/var/run/s3_cache/:bucket/:year/:month/:day' ],
+      [ :cache_dir, '-x',          '/var/run/s3_cache/:bucket/:year/:month/:day' ],
+      
+      [ :list, '--list', true ],
+      [ :list, '-l',     true ],
+      
+      [ :debug, '--debug', true ],
+      [ :debug, '-d',      true ],
+      [ :debug, '--debug', 'aws' ],
+      [ :debug, '-d',      'aws' ],
+      
+      [ :config_file, '--config-file', '/my/config/file.conf' ],
+      [ :config_file, '-c',            '/my/config/file.conf' ],
+      [ :config_file, '--config-file', '' ],
+      [ :config_file, '-c',            '' ],
+      
+      # :rename_bucket_keys    => [ '-m', '--rename-bucket-keys',    nil ],
+    ]
+  end
+  
+  it "should show help message" do
+    output = StringIO.new
+    options = Ralf::OptionParser.parse('-h'.split, output)
+    options.should be_nil
+    output.string.should_not be_empty
+    output.string.should include("Show this message")
+  end
+  
+  it "should show version" do
+    output = StringIO.new
+    File.should_receive(:read).and_return('1.2.3')
+    options = Ralf::OptionParser.parse('-v'.split, output)
+    options.should be_nil
+    output.string.should_not be_empty
+    output.string.should include("1.2.3")
+  end
+
+  it "should parse all options short or long" do
+    output = StringIO.new
+    
+    @valid_arguments.to_a.each do |argument_spec|
+      options = to_argv_array(argument_spec)
+      config = Ralf::OptionParser.parse(options, output)
+    
+      config.should have_key(argument_spec.first)
+      config[argument_spec.first].should eql(argument_spec.last)
+
+      output.string.should be_empty
+    end
+  
+  end
+
+  it "should allow two dates for range" do
+    output = StringIO.new
+    
+    options = Ralf::OptionParser.parse("--range yesterday,today".split, output)
+    
+    options.should have_key(:range)
+    options[:range].should eql(['yesterday', 'today'])
+  end
+  
+  it "should produce error for missing argument" do
+    output = StringIO.new
+    
+    lambda {
+      options = Ralf::OptionParser.parse("--range".split, output)
+    }.should raise_error(OptionParser::MissingArgument)
+  end
+  
+  private
+  
+  def to_argv_array(spec)
+    spec[1..-1].delete_if{ |v| true == v }.map{ |v| v.is_a?(Array) ? v.join(',') : v }
+  end
+  
+end