ralf 0.1.5 → 1.0.0

data/.gitignore

@@ -0,0 +1,4 @@
+**.gem
+tmp/**
+ralf.gemspec
+rdoc

data/README.rdoc

@@ -0,0 +1,126 @@
+= Synopsis
+
+Download, merge and convert Amazon S3 bucket log files for a specified date or date range.
+
+* Download S3 bucket log files produced by Amazon S3. Log files are downloaded once and cached locally.
+* Merge those log files together into a single logfile per bucket (sorting on ascending timestamp)
+* Convert the log file from Amazon Server Access Log Format to Apache Common Log Format
+
+Ralf is an acronym for Retrieve Amazon Log Files. Ralf does the following things:
+
+= Usage
+
+  Usage: ./bin/ralf [options]
+
+  Download and merge Amazon S3 bucket log files for a specified date range and
+  output a Common Log File. Ralf is an acronym for Retrieve Amazon Log Files.
+
+  Ralf downloads bucket log files to local cache directories, merges the Amazon Log
+  Files and converts them to Common Log Format.
+
+  Example: ./bin/ralf --range month --now yesterday --output-file '/var/log/amazon/:year/:month/:bucket.log'
+
+  AWS credentials (Access Key Id and Secret Access Key) are required to access
+  S3 buckets. For security reasons these credentials can only be specified in a
+  configuration file (see --config-file) or through the environment using the
+  AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+
+  Log selection options:
+      -l, --[no-]list                  List buckets that have logging enabled. Does not process log files.
+      -b, --buckets x,y,z              Buckets for which to process log files. Defaults to all log-enabled buckets.
+      -r, --range BEGIN[,END]          Date or date range to process. Defaults to 'today'.
+      -t, --now TIME                   Date to use as base for range. Defaults to 'today'.
+
+      You can use Chronic expressions for '--range' and '--now'. See http://chronic.rubyforge.org.
+    
+      Example: --range 'last week'
+        All days of previous week.
+      Example: --range 'this week'
+        Beginning of this week (sunday) upto and including today.
+      Example: --range '2010-01-01','2010-04-30'
+        First four months of this year.
+      Example: --range 'this month' --now yesterday
+        This will select log files from the beginning of yesterday's month upto and including yesterday.
+
+      The --buckets, --range and --now options are optional. If unspecified, (incomplete)
+      logging for today will be processed for all buckets (that have logging enabled).
+      This is equivalent to specifying "--range 'today'" and "--now 'today'".
+
+  Output options:
+      -o, --output-file FORMAT         Output file, e.g. '/var/log/s3/:year/:month/:bucket.log'. Required.
+
+      The --output-file format uses the last day of the range specified by (--range)
+      to determine the filename. E.g. when the format contains ':year/:month/:day' and
+      the range is 2010-01-15..2010-02-14, then the output file will be '2010/02/14'.
+
+      -x, --cache-dir FORMAT           Directory name(s) in which to cache downloaded log files. Optional.
+
+      The --cache-dir format expands to as many directory names as needed for the 
+      range specified by --range. E.g. "/var/run/s3_cache/:year/:month/:day/:bucket"
+      expands to 31 directories for range 2010-01-01..2010-01-31.
+    
+      Defaults to '~/.ralf/:bucket' or '/var/log/ralf/:bucket' (when running as root).
+
+  Config file options:
+      -c, --config-file [FILE]         Path to file with configuration settings (in YAML format).
+
+      Configuration settings are read from the (-c) specified configuration file
+      or from ~/.ralf.conf or from /etc/ralf.conf (when running as root).
+      Command-line options override settings read from the configuration file.
+
+      The configuration file must be in YAML format. Each command-line options has an
+      equivalent setting in a configuration file replacing dash (-) by underscore(_).
+
+      The Amazon Access Key Id and Secret Access Key can only be specified in the 
+
+      Example:
+        output_file:           /var/log/amazon_s3/:year:month/:bucket.log
+        aws_access_key_id:     my_access_key_id
+        aws_secret_access_key: my_secret_access_key
+
+      To only use command-line options simply specify -c or --config-file without
+      an argument.
+
+  Debug options:
+      -d, --[no-]debug [aws]           Show debug messages.
+
+  Common options:
+      -h, --help                       Show this message.
+      -v, --version                    Show version.
+
+= Library
+
+You can also use Ralf from within your own ruby code. Each command-line option
+has a corresponding option in the options has passed to Ralf.new and Ralf.run.
+Replace a dash (-) by an underscore (_) in the names:
+
+  options = { :output_file => '/var/log/s3/:bucket.log' }
+
+  require 'rubygems'
+  require 'ralf'
+  r = Ralf.new({ :config_file => '/Users/me/ralf.yaml' }.merge(options))
+  r.run
+
+Or run it in one go:
+  
+  Ralf.run({ :config_file => '/Users/me/ralf.yaml' }.merge(options))
+
+= Requirements
+
+* Credentials for an Amazon S3 account
+* Enable logging on S3
+  You can use Cyberduck[http://cyberduck.ch/] for example.
+
+= Gem dependencies
+
+Ralf depends on the following gems which will automatically installed when you
+install the ralf gem.
+
+* chronic
+* right_aws
+* logmerge
+
+= Authors
+Authors: {Leon Berenschot}[http://github.com/LeipeLeon] and {K.J. Wierenga}[http://github.com/kjwierenga]
+
+This program is used for {kerkdienstgemist.nl}[http://kerkdienstgemist.nl] {Amazon S3}[http://aws.amazon.com/s3/] log file processing.

data/Rakefile

@@ -0,0 +1,57 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+
+    gem.name = "ralf"
+    gem.summary = "Retrieve Amazon Log Files"
+    gem.description = <<-EOF
+      Download logfiles from Amazon S3 buckets to local disk and combine them in one Apache CLF per bucket
+    EOF
+    gem.email = [ "k.j.wierenga@gmail.com", "leonb@beriedata.nl" ]
+    gem.homepage = "http://github.com/kjwierenga/ralf"
+    gem.authors = ["Klaas Jan Wierenga", "Leon Berenschot"]
+
+    gem.add_development_dependency 'rspec',   '>= 1.3.0'
+    gem.add_development_dependency 'fakeweb', '>= 1.2.8'
+
+    gem.add_dependency 'right_aws', '>= 1.10.0'
+    gem.add_dependency 'logmerge',  '>= 1.0.2'
+    gem.add_dependency 'chronic',   '>= 0.2.3'
+
+    gem.rdoc_options << '--exclude' << '.'
+    gem.has_rdoc = false
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "ralf #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/bin/ralf

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'ralf'
+require 'ralf/option_parser'
+
+# Class to suppress writing of a particular string to an IO instance
+class IOSuppress < IO
+
+  @expect_newline = false
+  @suppress = nil
+  
+  def initialize(suppress, *args)
+    @suppress = suppress
+    super(*args)
+  end
+
+  # override write method to suppress string
+  def write(str)
+    case str
+    when "\n"
+      if @expect_newline
+        @expect_newline = false
+        return 1
+      end
+    when @suppress
+      @expect_newline = true
+      return @suppress.size
+    end
+    super(str)
+  end
+end
+$stderr = IOSuppress.new("warning: peer certificate won't be verified in this SSL session", $stderr.fileno, "w")
+
+begin
+  if options = Ralf::OptionParser.parse(ARGV)
+    Ralf.run(options)
+  end
+rescue => e
+  $stderr.puts "Error: #{e}"
+end

data/lib/ralf/bucket.rb

@@ -0,0 +1,66 @@
+require 'ralf/log'
+
+class Ralf
+
+  class Bucket
+    
+    def initialize(bucket)
+      raise ArgumentError.new("Bucket.s3 not assigned yet") if @@s3.nil?
+      
+      @bucket = bucket
+      @logging_info = @bucket.logging_info
+      if @logging_info[:enabled] and @bucket.name != @logging_info[:targetbucket]
+        @targetbucket = @@s3.bucket(@logging_info[:targetbucket])
+      else
+        @targetbucket = @bucket
+      end
+    end
+
+    def self.s3=(s3)
+      @@s3 = s3
+    end
+
+    def self.each(names = nil, with_logging = true)
+      # find specified buckets
+     if names
+       names.map do |name|
+         if s3_bucket = @@s3.bucket(name)
+           bucket = Bucket.new(s3_bucket)
+           yield bucket if !with_logging or bucket.logging_enabled?
+         else
+           puts("Warning: bucket '#{name}' not found.") if bucket.nil?
+         end
+       end
+     else
+       @@s3.buckets.each do |s3_bucket|
+         bucket = Bucket.new(s3_bucket)
+         yield bucket if !with_logging or bucket.logging_enabled?
+       end
+     end
+    end
+    
+    def name
+      @bucket.name
+    end
+    
+    def logging_enabled?
+      !!@logging_info[:enabled]
+    end
+    
+    def targetbucket
+      @logging_info[:targetbucket]
+    end
+
+    def targetprefix
+      @logging_info[:targetprefix]
+    end
+    
+    def each_log(date)
+      search_string = "%s%s" % [@logging_info[:targetprefix], date]
+      @targetbucket.keys(:prefix => search_string).each do |key|
+        yield Log.new(key, @logging_info[:targetprefix])
+      end
+    end
+  end
+
+end

data/lib/ralf/config.rb

@@ -0,0 +1,176 @@
+require 'ralf/interpolation'
+
+class Ralf::Config
+  
+  USER_DEFAULT_CACHE_DIR = '~/.ralf/:bucket'
+  ROOT_DEFAULT_CACHE_DIR = '/var/log/ralf/:bucket'
+  
+  class ConfigurationError < StandardError ; end
+  class RangeError         < StandardError ; end
+
+  attr_accessor \
+    :buckets,
+    :now,
+    # :range,
+    :aws_access_key_id,
+    :aws_secret_access_key
+    
+  attr_writer \
+    :debug,       # reader is debug?
+    :output_file, # reader interpolates format
+    :cache_dir    # reader interpolates format
+    
+  attr_reader :errors
+  
+  protected
+  
+  attr_accessor :options
+  
+  public
+
+  def self.load_file(filepath)
+    self.new(YAML.load_file(filepath))
+  end
+
+  def initialize(options = {})
+    @options = options.dup
+    
+    # assign defaults
+    @options[:now]       ||= nil
+    @options[:range]     ||= 'today'
+    @options[:cache_dir] ||= (0 == Process.uid ? ROOT_DEFAULT_CACHE_DIR : File.expand_path(USER_DEFAULT_CACHE_DIR))
+
+    assign_options(@options)
+  end
+  
+  def merge!(options)
+    @options.merge!(options)
+
+    assign_options(options)
+  end
+  
+  def debug?
+    @debug || false
+  end
+  
+  # compare two configurations
+  def ==(other)
+    @options == other.options
+  end
+
+  # return the range
+  def range
+    raise ArgumentError unless 2 == @range.size
+    Range.new(time_to_date(@range.first), time_to_date(@range.last)) # inclusive
+  end
+  
+  # set a range by a single Chronic expression or an array of 1 or 2 Chronic expressions
+  def range=(args)
+    args ||= []
+    args = [args] unless args.is_a?(Array)
+    
+    @range_value = args
+    
+    raise ArgumentError.new("too many range items") if args.size > 2
+
+    range = []
+    args.each_with_index do |expr, i|
+      raise RangeError if i > 1 # this should have been caught by ArgumentError before the loop
+      
+      chronic_options = { :context => :past, :guess => false }
+      if self.now
+        chronic_options.merge!(:now => Chronic.parse(self.now, :context => :past))
+      end
+      
+      if span = Chronic.parse(expr, chronic_options)
+        if span.width <= 24 * 3600 # on same date
+          range << span.begin
+        else
+          raise RangeError, "range end '#{expr}' is not a single date" if i > 0
+          range << span.begin
+          range << span.end + (self.now ? 0 : -1)
+        end
+      else
+        raise RangeError, "invalid expression '#{expr}'"
+      end
+    end
+    
+    range = [ Date.today ] if range.empty? # empty range means today
+    range = range*2 if 1 == range.size     # single day has begin == end
+
+    @range = range
+  end
+  
+  def output_file(variables)
+    Ralf::Interpolation.interpolate(@output_file, variables)
+  end
+  
+  def output_file_format
+    @output_file
+  end
+  
+  def cache_dir(variables)
+    Ralf::Interpolation.interpolate(@cache_dir, variables, [:bucket])
+  end
+  
+  def cache_dir_format
+    @cache_dir
+  end
+  
+  def empty?
+    @options.empty?
+  end
+  
+  def valid?
+    @errors = []
+    unless (@aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'])
+      @errors << 'aws_access_key_id missing'
+    end
+      
+    unless (@aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'])
+      @errors << 'aws_secret_access_key missing'
+    end
+  end
+  
+  def validate!
+    valid?
+    unless @errors.empty?
+      raise ConfigurationError.new(@errors.join(', '))
+    end
+  end
+  
+  def output_file_missing?
+    !@output_file
+  end
+  
+  private
+  
+  def time_to_date(time)
+    Date.new(time.year, time.month, time.day)
+  end
+  
+  def assign_options(new_options)
+    options = new_options.dup
+
+    # always re-assign range in case now has changed
+    if options.has_key?(:now)
+      self.now   = options.delete(:now)   
+      self.range = options.delete(:range) || @range_value
+    end
+    options.each do |attr, val|
+      begin
+        self.send("#{attr.to_s}=", val)
+      rescue NoMethodError => e
+        puts "Warning: invalid configuration variable: #{method_name(e)}"
+      end
+    end
+  end
+  
+  # Take NoMethodException string and extract the method name,
+  # e.g. "undefined method `out_path=' for #<Ralf::Config:0x17931b8>"
+  # should return 'out_path'
+  def method_name(e)
+    e.to_s.split('`')[1].split('=')[0]
+  end
+  
+end

data/lib/ralf/interpolation.rb

@@ -0,0 +1,48 @@
+class Ralf
+
+  class Interpolation
+    class NotAllInterpolationsSatisfied < StandardError ; end
+    class VariableMissing               < StandardError ; end
+
+    def self.interpolate(string, variables, required_variables = [])
+      required_variables.each do |name|
+        raise VariableMissing, ":#{name.to_s} variable missing" unless string.match(/:#{name.to_s}/)
+      end
+      processor = Ralf::Interpolation.new(string, variables)
+      raise NotAllInterpolationsSatisfied, "Not all keys are interpolated: '#{string}'" if processor.result.match(/:/)
+      processor.result
+    end
+    
+    attr :result
+
+    def initialize(string, variables)
+      @variables = variables
+      @result = string.dup
+      (Ralf::Interpolation.public_instance_methods(false) - ['result']).each do |tag|
+        @result.gsub!(/:#{tag}/, self.send( tag )) unless self.send(tag).nil?
+      end
+    end
+
+    def bucket
+      @variables[:bucket]
+    end
+
+    def week
+      "%02d" % @variables[:date].cweek if @variables[:date]
+    end
+
+    def day
+      "%02d" % @variables[:date].day if @variables[:date]
+    end
+
+    def month
+      "%02d" % @variables[:date].month if @variables[:date]
+    end
+
+    def year
+      "%04d" % @variables[:date].year if @variables[:date]
+    end
+
+  end
+
+end

data/lib/ralf/log.rb

@@ -0,0 +1,18 @@
+class Ralf
+  class Log
+    def initialize(key, targetprefix)
+      @key          = key
+      @targetprefix = targetprefix
+    end
+  
+    def name
+      @key.name.gsub(@targetprefix, '')
+    end
+  
+    def save_to_dir(dir, use_cache = true)
+      file = File.join(dir, name)
+      File.open(file, 'w') { |f| f.write(@key.data) } unless use_cache and File.exist?(file)
+      file # return saved filename
+    end
+  end
+end

data/lib/ralf/option_parser.rb

@@ -0,0 +1,149 @@
+require 'optparse'
+
+class Ralf::OptionParser
+  
+  def self.parse(args, output = $stdout)
+    options = {}
+
+    opts = ::OptionParser.new do |opts|
+      opts.banner = <<USAGE_END
+Usage: #{$0} [options]
+
+Download and merge Amazon S3 bucket log files for a specified date range and
+output a Common Log File. Ralf is an acronym for Retrieve Amazon Log Files.
+
+Ralf downloads bucket log files to local cache directories, merges the Amazon Log
+Files and converts them to Common Log Format.
+
+Example: #{$0} --range month --now yesterday --output-file '/var/log/amazon/:year/:month/:bucket.log'
+
+AWS credentials (Access Key Id and Secret Access Key) are required to access
+S3 buckets. For security reasons these credentials can only be specified in a
+configuration file (see --config-file) or through the environment using the
+AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+USAGE_END
+
+      opts.separator ""
+      opts.separator "Log selection options:"
+      opts.on("-l", "--[no-]list", "List buckets that have logging enabled. Does not process log files.") do |value|
+        options[:list] = value
+      end
+      opts.on("-b", "--buckets x,y,z", Array, "Buckets for which to process log files. Defaults to all log-enabled buckets.") do |buckets|
+        options[:buckets] = buckets.compact
+      end
+      opts.on("-r", "--range BEGIN[,END]", Array, "Date or date range to process. Defaults to 'today'.") do |range|
+        options[:range] = range.compact
+      end
+      log_selection_help =<<LOG_SELECTION_HELP
+Date to use as base for range. Defaults to 'today'.
+
+    You can use Chronic expressions for '--range' and '--now'. See http://chronic.rubyforge.org.
+    
+    Example: --range 'last week'
+      All days of previous week.
+    Example: --range 'this week'
+      Beginning of this week (sunday) upto and including today.
+    Example: --range '2010-01-01','2010-04-30'
+      First four months of this year.
+    Example: --range 'this month' --now yesterday
+      This will select log files from the beginning of yesterday's month upto and including yesterday.
+
+    The --buckets, --range and --now options are optional. If unspecified, (incomplete)
+    logging for today will be processed for all buckets (that have logging enabled).
+    This is equivalent to specifying "--range 'today'" and "--now 'today'".
+LOG_SELECTION_HELP
+      opts.on("-t", "--now TIME", log_selection_help) do |now|
+        options[:now] = now
+      end
+      
+      # opts.on("-m", "--[no-]rename-bucket-keys", "Rename original log files on Amazon using format from '--cache-dir' option.") do |value|
+      #   options[:rename_bucket_keys] = value
+      # end
+
+      # opts.separator ""
+      opts.separator "Output options:"
+
+      output_file_help =<<OUTPUT_FILE_HELP
+Output file, e.g. '/var/log/s3/:year/:month/:bucket.log'. Required.
+
+    The --output-file format uses the last day of the range specified by (--range)
+    to determine the filename. E.g. when the format contains ':year/:month/:day' and
+    the range is 2010-01-15..2010-02-14, then the output file will be '2010/02/14'.
+OUTPUT_FILE_HELP
+      opts.on("-o", "--output-file FORMAT", output_file_help) do |format|
+        options[:output_file] = format
+      end
+      
+      cache_dir_help =<<CACHE_DIR_HELP
+Directory name(s) in which to cache downloaded log files. Optional.
+
+    The --cache-dir format expands to as many directory names as needed for the 
+    range specified by --range. E.g. "/var/run/s3_cache/:year/:month/:day/:bucket"
+    expands to 31 directories for range 2010-01-01..2010-01-31.
+    
+    Defaults to '~/.ralf/:bucket' or '/var/log/ralf/:bucket' (when running as root).
+CACHE_DIR_HELP
+      opts.on("-x", "--cache-dir FORMAT", cache_dir_help) do |format|
+        options[:cache_dir] = format
+      end
+
+      # opts.on("-f", "--output-dir-format FORMAT", "Output directory format, e.g. ':year/:month/:day'") do |format|
+      #   options[:output_dir_format] = format
+      # end
+
+      # opts.on("-o", "--output-basedir DIR", "Base directory for output files.") do |dir|
+      #   options[:output_basedir] = dir
+      # end
+
+      # opts.on("-p", "--output-prefix STRING", "Prefix string for output files.") do |string|
+      #   options[:output_prefix] = string
+      # end
+      
+      # opts.separator ""
+      opts.separator "Config file options:"
+      config_file_help =<<CONFIG_FILE_HELP
+Path to file with configuration settings (in YAML format).
+
+    Configuration settings are read from the (-c) specified configuration file
+    or from ~/.ralf.conf or from /etc/ralf.conf (when running as root).
+    Command-line options override settings read from the configuration file.
+
+    The configuration file must be in YAML format. Each command-line options has an
+    equivalent setting in a configuration file replacing dash (-) by underscore(_).
+
+    The Amazon Access Key Id and Secret Access Key can only be specified in the 
+
+    Example:
+      output_file:           /var/log/amazon_s3/:year:month/:bucket.log
+      aws_access_key_id:     my_access_key_id
+      aws_secret_access_key: my_secret_access_key
+
+    To only use command-line options simply specify -c or --config-file without
+    an argument.
+CONFIG_FILE_HELP
+      opts.on("-c", "--config-file [FILE]", config_file_help) do |file|
+        options[:config_file] = file
+      end
+      
+      opts.separator "Debug options:"
+      opts.on("-d", "--[no-]debug [aws]", "Show debug messages.") do |aws|
+        options[:debug] = aws || true
+      end
+
+      opts.separator ""
+      opts.separator "Common options:"
+      opts.on_tail("-h", "--help", "Show this message.") do
+        output.puts opts
+        return nil
+      end
+      opts.on_tail("-v", "--version", "Show version.") do
+        output.print File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION'))
+        return nil
+      end
+    end
+    remaining = opts.parse!(args)
+    opts.warn "Warning: unused arguments: #{remaining.join(' ')}" unless remaining.empty?
+    options
+  end
+
+end