logbox 0.2.10

data/lib/logbox/log_parser.rb

@@ -0,0 +1,79 @@
+require 'date'
+require 'cgi'
+require 'stringio'
+
+# Parses a standard web server log file stream and returns a hash with
+# key/values for each line. Includes the Enumerable interface.
+class LogParser
+  include Enumerable
+
+  # Support both strings and streams as input.
+  def initialize(input)
+    input = StringIO.new(input) if input.class == String
+    @stream = input
+  end
+
+  # Enumerable interface.
+  def each
+    while(observation = get_next_observation)
+      yield observation
+    end
+  end
+
+  def get_next_observation
+    line = @stream.gets
+    line && LogParser.parse_line(line)
+  end
+
+  LOG_FORMAT = /([^ ]*) [^ ]* [^ ]* \[([^\]]*)\] "([^"]*)" ([^ ]*)/
+  LOG_DATE_FORMAT = "%d/%b/%Y:%H:%M:%S %z"
+  LOG_KEY_VALUE_FORMAT = /[?&]([^=]+)=([^&]+)/
+  SERVER_ATTRIBUTES = [:ip, :timestamp, :request, :status]
+
+  # Parse one log line and return a hash with all attributes.
+  def self.parse_line(line)
+    return nil if line.strip.empty?
+    
+    line =~ LOG_FORMAT
+    result = {}
+
+    # Save ip, timestamp and request.
+    result[:ip] = $1
+    begin
+      result[:timestamp] = DateTime.strptime($2, LOG_DATE_FORMAT)
+    rescue ArgumentError
+      raise ParseError.new("Error while parsing timestamp")
+    end
+    result[:request] = $3
+    result[:status] = $4
+
+    # Extract key/values pairs from the query part of the request.
+    $3.scan(LOG_KEY_VALUE_FORMAT) do |key, value|
+      begin
+        key = CGI.unescape(key).to_sym
+        value = CGI.unescape(value)
+      rescue Encoding::CompatibilityError => e
+        raise ParseError.new("Error while parsing query parameters")
+      end
+
+      if result.has_key? key
+        if result[key].is_a? Array
+          result[key] << value
+        else
+          result[key] = [result[key], value]
+        end
+      else
+        result[key] = value
+      end
+    end
+
+    return result
+  rescue ParseError
+    raise
+  rescue
+    raise ParseError.new("Unknown parsing error")
+  end
+  class ParseError < StandardError ; end
+end
+
+

data/lib/logbox/mockup_log.rb

@@ -0,0 +1,44 @@
+require 'time'
+require 'uri'
+
+module MockupLog
+  
+  # Create a custom log file string.
+  def mockup_log lines_data
+    template_lines = {
+      :visit_page => %Q'72.211.248.18 - - [02/Mar/2009:06:58:12 +0100] "GET /log.gif?a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010_5_5%3B%20en-us)%20AppleWebKit%2F525.27.1%20(KHTML%2C%20like%20Gecko)%20Version%2F3.2.1%20Safari%2F525.27.1&aid=xroads&l=en-us&n=netscape&o=visit_page&p=macintel&r=http%3A%2F%2Fdesignhousestockholm.xroads.se%2FPortal%2FProducts%2FProductGroup.asp%3Fclick%3D1%26pageIndex%3D14%26type%3Dgeneric&s=1920x1200&sid=DHS_Shop&t=Product&u=http%3A%2F%2Fdesignhousestockholm.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D6005&uid=1235973461091911420&x=35412&z=480& HTTP/1.1" 200 35 "http://designhousestockholm.xroads.se/Portal/Products/Product.asp?itemId=6005" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1" "-" "-"\n',
+      :view_item => %Q'72.211.248.18 - - [02/Mar/2009:06:59:32 +0100] "GET /log.gif?_current_price=US%2471&_image=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FDesignhouseStockholm%2F1132-1000_stor.jpg&_item_id=6005&_sku=HHAW09%20Shelter%20Pants&_thumbnail=..%2F..%2FArchive%2FImages%2FWebshop%2FProducts%2FDesignhouseStockholm%2F1132-1000_liten.jpg&_title=%C2%A0PLEECE%20-%20HOOD&_url=http%3A%2F%2Fdesignhousestockholm.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D6005&a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010_5_5%3B%20en-us)%20AppleWebKit%2F525.27.1%20(KHTML%2C%20like%20Gecko)%20Version%2F3.2.1%20Safari%2F525.27.1&aid=xroads&l=en-us&n=netscape&o=view_item&p=macintel&r=http%3A%2F%2Fdesignhousestockholm.xroads.se%2FPortal%2FProducts%2FProductGroup.asp%3Fclick%3D1%26pageIndex%3D14%26type%3Dgeneric&s=1920x1200&sid=DHS_Shop&t=Product&u=http%3A%2F%2Fdesignhousestockholm.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D6005&uid=1235973461091911420&x=35412&z=480& HTTP/1.1" 200 35 "http://designhousestockholm.xroads.se/Portal/Products/Product.asp?itemId=6005" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1" "-" "-"\n',
+      :pick_item => %Q'72.211.248.18 - - [02/Mar/2009:07:00:28 +0100] "GET /log.gif?_item_id=6004&a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010_5_5%3B%20en-us)%20AppleWebKit%2F525.27.1%20(KHTML%2C%20like%20Gecko)%20Version%2F3.2.1%20Safari%2F525.27.1&aid=xroads&e=click&l=en-us&n=netscape&o=pick_item&p=macintel&r=http%3A%2F%2Fdesignhousestockholm.xroads.se%2FPortal%2FProducts%2FProductGroup.asp%3Fclick%3D1%26pageIndex%3D14%26type%3Dgeneric&s=1920x1200&sid=DHS_Shop&t=Product&u=http%3A%2F%2Fdesignhousestockholm.xroads.se%2FPortal%2FProducts%2FProduct.asp%3FitemId%3D6004&uid=1235973461091911420&x=98033&z=480& HTTP/1.1" 200 35 "http://designhousestockholm.xroads.se/Portal/Products/Product.asp?itemId=6004" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1" "-" "-"\n',
+      :buy_item => %Q'72.211.248.18 - - [02/Mar/2009:07:01:28 +0100] "GET /log.gif?_current_price=1250&_order_id=13945&_quantity=1&_sku=FERTILITETSMONITOR&_title=Fertilitetsmonitor+fr%C3%A5n+Clearblue&a=Mozilla%2F4.0%20(compatible%3B%20MSIE%207.0%3B%20Windows%20NT%205.1%3B%20.NET%20CLR%201.1.4322%3B%20.NET%20CLR%202.0.50727%3B%20.NET%20CLR%203.0.4506.2152%3B%20.NET%20CLR%203.5.30729)&aid=jetshop&l=sv&n=microsoft%20internet%20explorer&o=buy_item&p=win32&s=1280x960&sid=www.medistore.se&t=H%C3%A4r%20hittar%20du%20sjukv%C3%A5rdsprodukter%2C%20blodtrycksm%C3%A4tare%2C%20f%C3%B6rsta%20hj%C3%A4lpen%20och%20pulsklockor&u=https%3A%2F%2Fwww.medistore.se%2FOrderDetailsConfirmed.aspx&uid=1268173983791130948&x=68410&z=-60& HTTP/1.1" 200 35 "http://designhousestockholm.xroads.se/Portal/Products/Product.asp?itemId=6004" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1" "-" "-"\n',
+      :heed_recommendation => %Q'72.211.248.18 - - [02/Mar/2009:07:01:28 +0100] "GET /log.gif?_current_price=189%2C00%20kr&_item_id=1441&_rec_for=2407&_rec_type=viewed_this_also_viewed&_recommended_ids%5B0%5D=1441&_recommended_ids%5B1%5D=960&_recommended_ids%5B2%5D=535&_url=http%3A%2F%2Fwww.shirtstore.se%2Fits-on-like-donkey-kong-p-1441.aspx&a=Mozilla%2F4.0%20(compatible%3B%20MSIE%208.0%3B%20Windows%20NT%206.1%3B%20Trident%2F4.0%3B%20GTB6.4%3B%20SLCC2%3B%20.NET%20CLR%202.0.50727%3B%20.NET%20CLR%203.5.30729%3B%20.NET%20CLR%203.0.30729%3B%20Media%20Center%20PC%206.0)&aid=jetshop&e=click&l=sv&n=microsoft%20internet%20explorer&o=heed_recommendation&p=win32&r=http%3A%2F%2Fwww.shirtstore.se%2Ftshirts-c-491-1.aspx&s=1280x800&sid=www.shirtstore.se&t=The%20Hulk%20Distressed&u=http%3A%2F%2Fwww.shirtstore.se%2Fthe-hulk-distressed-p-2407-c-491.aspx&uid=1266087502030697666&x=55724&z=-60& HTTP/1.1" 200 35 "http://designhousestockholm.xroads.se/Portal/Products/Product.asp?itemId=6004" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; en-us) AppleWebKit/525.27.1 (KHTML, like Gecko) Version/3.2.1 Safari/525.27.1" "-" "-"\n',
+    }
+    log = ""
+    lines_data.each do |line_data|
+      line_data = {:aid => "xroads", :sid => "Fortum", :uid => "12345", :timestamp => Time.parse("2009-03-02 07:01:28")}.merge(line_data)
+      observation_type = line_data.delete(:o) || raise("Missing observation type attribute (:o => 'xx') in line data.")
+      raise "Has no template for this observation type" unless template_lines[observation_type.to_sym]
+      # Construct the log line.
+      line = template_lines[observation_type.to_sym].dup
+      line_data.each do |key, value|
+        case key
+        when :timestamp then
+          line.sub!(/\[[^\]]+?\]/, value.nil? ? '' : value.strftime('[%d/%b/%Y:%H:%M:%S +0000]'))
+        when :ip then
+          line.sub!(/^\d+\.\d+\.\d+\.\d+/, value)
+        else
+          value = URI.escape(value, /[^-_.!~*'()a-zA-Z\d]/n)
+          line.gsub!(Regexp.new("#{key.to_s}=[^&]+&", true), "#{key.to_s}=#{value}&")
+        end
+      end
+      log << line
+    end
+    log    
+  end
+  
+  def mockup_log_file(path, lines_data)
+    File.open(path, "w") do |file|
+      file.write(mockup_log(lines_data))
+    end
+  end
+  
+end

data/lib/logbox/observation.rb

@@ -0,0 +1,162 @@
+require 'uri'
+require 'log_parser'
+
+class Observation
+
+  BASIC_ATTRIBUTES_NAMES = {
+    :aid => :account_id,
+    :sid => :shop_id,
+    :uid => :user_id,
+    :u   => :document_url,
+    :r   => :referrer,
+    :t   => :document_title,
+    :s   => :screen_resolution,
+    :l   => :browser_language,
+    :p   => :platform,
+    :a   => :user_agent,
+    :h   => :history_count,
+    :n   => :navigator_name,
+    :z   => :time_zone_offset,
+    :x   => :seed,
+    :o   => :observation_type
+  }
+ 
+  OBSERVATION_ATTRIBUTES_NAMES = {
+    :_item_id       => :item_id,
+    :_rec_for       => :rec_for,
+    :_title         => :title,
+    :_url           => :url,
+    :_basket_url    => :basket_url,
+    :_description   => :description,
+    :_normal_price  => :normal_price,
+    :_current_price => :current_price,
+    :_thumbnail     => :thumbnail,
+    :_image         => :image,
+    :_stock         => :stock,
+    :_expires       => :expires,
+    :_order_blob    => :order_blob,
+    :_order_id      => :order_id,
+    :_sku           => :sku,
+    :_quantity      => :quantity,
+    :_tags          => :tags,
+    :_user1         => :user1,
+    :_user2         => :user2,
+    :_user3         => :user3,
+    :_user4         => :user4,
+    :_user5         => :user5,
+    :_culture       => :culture,
+    :_currency      => :currency,
+    :_group_ref     => :group_ref,
+    :_with_vat      => :with_vat
+  }
+  BASIC_ATTRIBUTES = BASIC_ATTRIBUTES_NAMES.values
+  OBSERVATION_ATTRIBUTES = OBSERVATION_ATTRIBUTES_NAMES.values
+  ITEM_ATTRIBUTES = OBSERVATION_ATTRIBUTES
+  OBSERVATION_TYPE_STRINGS = %w[visit_page view_item pick_item buy_basket buy_item view_recommendation heed_recommendation]
+  OBSERVATION_TYPES = OBSERVATION_TYPE_STRINGS.map { |e| e.to_sym }
+  # Konverterade med ett stort perl-uttryck:
+  # perl -pi -e 's/([&?])_id=/\1_item_id=/g; s/currentprice/current_price/g; s/normalprice/normal_price/g; s/basketurl/basket_url/g; s/([&?])_type=/\1o=/g; s/buyitem/pick_item/g; s/viewitem/view_item/g; s/searchclick/search_pick_item/g; s/sid=bwintest/aid=bwin/g; s/sid=/aid=crossroads&sid=/g; s/aid=bwin/aid=bwin&sid=bwintest/g;' observer.access.log.*
+  
+  attr_reader :attributes, :unknown_attributes, :type, :errors
+
+  def initialize(logline_attributes)
+    transfer_attributes(logline_attributes.clone)
+    validate_type
+    ensure_correct_urls
+  end
+
+  def valid?
+    @errors.empty?
+  end
+
+  def [](key)
+    @attributes[key]
+  end
+
+  def view_item?
+    @type == :view_item
+  end
+
+  def pick_item?
+    @type == :pick_item
+  end
+  
+  def buy_item?
+    @type == :buy_item
+  end
+  
+private
+
+  # Takes the input hash and moves all known attributes to the @attributes
+  # hash, renaming some of them. Unknown attributes are also saved.
+  def transfer_attributes(logline_attributes)
+    @attributes = {}
+
+    # Transfer basic attributes.
+    (BASIC_ATTRIBUTES_NAMES.to_a + OBSERVATION_ATTRIBUTES_NAMES.to_a).each do |key, new_key|
+      value = logline_attributes.delete(key)
+      @attributes[new_key] = value if value
+    end
+    
+    # Transfer server attributes.
+    LogParser::SERVER_ATTRIBUTES.each do |key|
+      value = logline_attributes.delete(key)
+      @attributes[key] = value if value
+    end
+    
+    @unknown_attributes = logline_attributes
+  end
+  
+  # Ensure that URLs are absolute and don't contain the anchor part.
+  def ensure_correct_urls
+    if @attributes.has_key? :document_url
+      @errors ||= []
+      [:url, :thumbnail, :image, :basket_url].each do |key|
+        begin
+          if @attributes.has_key? key
+            # Ensure that the url is not encoded (it is ok to unencode a url that is not encoded).
+            url = URI.unescape(@attributes[key])
+            # But we store and handle all urls encoded (according to the RFC). URI.join also requires it.
+            url = URI.escape(url)
+            base_url = URI.escape(@attributes[:document_url])
+            # Ensure absoulte.
+            full_url = URI.join(base_url, url).to_s
+            # Ensure no debug anchor. Just to make sure that #debug urls do not reach Dumbo.
+            full_url.gsub!(/%23debug(on|off)?/, '')
+            @attributes[key] = full_url
+          end
+        rescue
+          @errors << "url_error_on_#{key}".to_sym
+        end
+        
+      end
+    end
+  end
+
+  REQUIRED_ATTRIBUTES = [:user_id, :account_id, :shop_id, :item_id]
+  REQUIRED_ATTRIBUTES_FOR_TYPE = {
+    :buy_basket => [:user_id, :account_id, :shop_id],
+    :buy_item => [:user_id, :account_id, :shop_id, :sku],
+    :visit_page => [:user_id, :account_id, :shop_id],
+    :view_recommendation => [:user_id, :account_id, :shop_id, :rec_for],
+    :heed_recommendation => [:user_id, :account_id, :shop_id, :item_id]
+  }
+  
+  def validate_type
+    @type = :unknown
+    @errors ||= []
+
+    # Only handle successfull requests.
+    @errors << :status_not_ok and return unless @attributes[:status] == "200"
+    
+    # Only handle types we know of.
+    @errors << :unknown_type and return unless OBSERVATION_TYPE_STRINGS.include? @attributes[:observation_type]
+    @type = @attributes[:observation_type].to_sym
+    
+    # For all valid types we require some attributes.
+    (REQUIRED_ATTRIBUTES_FOR_TYPE[@type] || REQUIRED_ATTRIBUTES).each do |a|
+      @errors << "missing_#{a}".to_sym unless @attributes[a] =~ /.+/
+    end
+  end
+  
+end

data/lib/logbox/observation_compiler.rb

@@ -0,0 +1,311 @@
+# -*- encoding: utf-8 -*-
+require 'rubygems'
+require 'time'
+require 'date'
+require 'fileutils'
+require 'right_aws'
+require 'rake'
+require 'yaml'
+require 'logbox'
+
+module ObservationCompiler
+  
+  class Job
+
+    def initialize(options = {})
+      @raw_logs_bucket     = options[:raw_logs_bucket] || "rwdata-logs"
+      @raw_logs_prefix     = options[:raw_logs_prefix] || "observer-log-"
+      @processed_logs_path = options[:processed_logs_path] || "local_files"
+      temp_dir = File.exist?("/apps/smartass") ? "/apps/smartass/tmp/" : "/tmp/"
+      @working_path        = options[:working_path] ||= "#{temp_dir}observation_compiler/#{Process.pid}"
+    end
+
+    def fetch_and_merge(raw_date_range)
+      # A raw log-file for a date may contain observations from the day before. 
+      processed_date_range = (raw_date_range.first-1)..(raw_date_range.last)
+
+      create_working_folders
+      copy_processed_logs_to_working_folder(processed_date_range)
+      unzip_processed(processed_date_range)
+
+      raw_date_range.each do |date|
+        download_raw_logs_to_working_folder(date)
+        unzip_raw(date)
+        merge_raw_into_processed(date)
+        remove_raw_logs(date)
+      end
+
+    ensure
+      sort_processed(processed_date_range)
+      zip_processed(processed_date_range)
+      move_processed_back(processed_date_range)
+      remove_working_path
+    end
+    
+    def create_working_folders
+      FileUtils.mkdir_p(raw_working_path)
+      FileUtils.mkdir_p(processed_working_path)
+    end
+    
+    def raw_working_path
+      File.join(@working_path, "r")
+    end
+    
+    def processed_working_path
+      File.join(@working_path, "p")
+    end
+
+    def processed_log_name(date)
+      "observer-log-#{date.strftime('%Y-%m-%d')}"
+    end
+    
+    def raw_logs_prefix(date)
+      @raw_logs_prefix + date.strftime('%Y-%m-%d')
+    end
+    
+    def raw_log_paths(date)
+      Dir.glob(File.join(raw_working_path, "#{raw_logs_prefix(date)}*")).sort
+    end
+
+    def remove_working_path
+      FileUtils.rm_r @working_path
+    end
+
+    def copy_processed_logs_to_working_folder(date_range)
+      date_range.each do |date|
+        name = processed_log_name(date) + ".gz"
+        source = File.join(@processed_logs_path, name)
+        destination = File.join(processed_working_path, name)
+        if File.exist? source
+          log "Copying #{name} to working folder"
+          FileUtils.copy(source, destination)
+        end
+      end
+    end
+    
+    def download_raw_logs_to_working_folder(date)
+      s3 = RightAws::S3.new(*aws_keys)
+      bucket = s3.bucket(@raw_logs_bucket)
+      raise "Unknown bucket: #{@raw_logs_bucket}" if bucket.nil?
+
+      raw_logs = bucket.keys(:prefix => raw_logs_prefix(date))
+      raw_logs.each do |raw_log|
+        log "Getting #{raw_log.name}"
+        File.open(File.join(raw_working_path, raw_log.name), "w") do |file|
+          s3.interface.get(@raw_logs_bucket, raw_log.name) do |chunk|
+            file.write(chunk)
+          end
+        end
+      end
+    end
+    
+    def unzip_raw(date)
+      log "Unzipping raw logs for #{date}"
+      raw_log_paths(date).each do |raw_log|
+        system "gunzip #{raw_log}" if raw_log.end_with?(".gz")
+      end
+    end
+
+    def merge_raw_into_processed(date)
+      start_time = Time.now
+      count = 0
+      out_files = {}
+      raw_log_paths(date).each do |raw_log|
+        if raw_log_already_processed?(raw_log)
+          log "Skipping #{raw_log}"
+          next
+        else
+          log "Processing #{raw_log}"
+        end
+        File.foreach raw_log do |line|
+          log_line = LogLine.new(line)
+          next unless log_line.valid?
+          date = log_line.date
+          name = File.join(processed_working_path, processed_log_name(date))
+          out_files[name] ||= File.open(name, "a")
+          out_files[name] << log_line.normalize
+          count += 1
+        end
+      end
+    ensure
+      out_files.each_value { |file| file.close }
+      log "#{count} rader på #{(Time.now - start_time).to_f}s (#{count/(Time.now - start_time).to_f} rader/s)"
+    end
+    
+    def raw_log_already_processed?(log_file_name)
+      # Look for the last observation to see if it is already processed.
+      last_observation = `tail -n 1 #{log_file_name}`
+      log_line = LogLine.new(last_observation)
+      return false unless log_line.valid?
+      date = log_line.date
+      processed_file_name = File.join(processed_working_path, processed_log_name(date))
+      File.exist?(processed_file_name) && system("grep", "-qF", last_observation, processed_file_name.chomp)
+    end
+    
+    def remove_raw_logs(date)
+      log "Removing raw logs for #{date}"
+      raw_log_paths(date).each do |raw_log|
+        FileUtils.rm(raw_log)
+      end
+    end
+    
+    def sort_processed(date_range) 
+      date_range.each do |date|
+        name = processed_log_name(date)
+        Dir.chdir processed_working_path do
+          next unless File.exist?(name)
+          log "Sorting #{name}"
+          ENV['LC_ALL'] = 'C'
+          ok = system "sort -t: -k2,4 #{name} | uniq > #{name}.sorted"
+          raise "Sort error!" unless ok
+          File.rename("#{name}.sorted", name)
+        end
+      end
+    end
+  
+    def zip_processed(date_range)
+      log "Zipping processed files"
+      date_range.each do |date|
+        name = processed_log_name(date)
+        file = File.join(processed_working_path, name)
+        system "gzip #{file}" if File.exist? file
+      end
+    end
+    
+    def unzip_processed(date_range)
+      log "Unzipping processed files"
+      date_range.each do |date|
+        name = processed_log_name(date) + ".gz"
+        file = File.join(processed_working_path, name)
+        system "gunzip #{file}" if File.exist? file
+      end
+    end
+    
+    def move_processed_back(date_range)
+      date_range.each do |date|
+        name = processed_log_name(date) + ".gz"
+        source = File.join(processed_working_path, name)
+        destination = File.join(@processed_logs_path, name)
+        if File.exist? source
+          log "Moving #{name} back"
+          FileUtils.move(source, destination)
+        end
+      end
+    end
+    
+    def log msg
+      unless defined?(TEST_RUN)
+        puts msg
+      end
+    end
+    
+    DEFAULT_KEY_FILE = '/etc/s3_key.yml'
+    def aws_keys
+      if File.exists? DEFAULT_KEY_FILE
+        hash = YAML.load_file(DEFAULT_KEY_FILE)
+        [hash[:access_key_id], hash[:secret_access_key]]
+      else
+        access_key_id = ENV['OBSENTER_S3_KEY']
+        secret_access_key = secret_access_key_from_keychain!(access_key_id)
+        [access_key_id, secret_access_key]
+      end
+    end
+
+    # These two methods are borrowed from Awsborn
+    def secret_access_key_from_keychain! (key_id)
+      secret = secret_access_key_from_keychain key_id
+      raise "Could not find secret access key for #{key_id}" if secret.to_s == ''
+      secret
+    end
+
+    def secret_access_key_from_keychain (key_id)
+      @credentials ||= {}
+      unless @credentials[key_id]
+        dump = `security -q find-generic-password -a "#{key_id}" -g 2>&1`
+        secret_key = dump[/password: "(.*)"/, 1]
+        @credentials[key_id] = secret_key
+      end
+      @credentials[key_id]
+    end
+  end
+
+  class LogLine
+    
+    def initialize(line)
+      @line = Logbox::StringEncoder.iconv(line)
+    end
+
+    def valid?
+      normalize
+      true
+    rescue
+      false
+    end
+    
+    def normalize
+      normalize_s3_format
+      normalize_apache_format
+      normalize_timestamp
+      @line
+    end
+
+    TIMESTAMP_MATCHER = /(\d+)\/(\w+)\/(\d+):(\d+):(\d+):(\d+)\s([-+]?\d{2})/
+
+    def timestamp
+      unless @timestamp
+        match = @line.match(TIMESTAMP_MATCHER)
+        @timestamp = Time.utc(match[3], match[2], match[1], match[4], match[5], match[6])
+        @timestamp -= match[7].to_i * 3600  # Correct the zone. Works only on whole hours timezones.
+      end
+      @timestamp
+    end
+
+    def date
+      timestamp.send :to_date
+    end
+
+    def to_s
+      @line
+    end
+    
+  private
+  
+    S = '(\\S+)'
+    # Equivalent to /("(?:\\\\|\\"|[^\\"])*")/
+    Q = '("(?:\\\\\\\\|\\\\"|[^\\\\"])*")'
+    TIMESTAMP = '(\\[[^\\]]+\\])'
+
+    S3_FORMAT = Regexp.new('^' + [S,S,TIMESTAMP,S,S,S,S,S,Q,S,S,S,S,S,S,Q,Q,S].join(' '), 'm')
+    def normalize_s3_format
+      # %Q{owner bucket [16/Mar/2010:16:00:00 +0000] 85.225.221.221 requester requestID operation key "GET /log.gif?_item_id=987&_title=V%C3%A4skor%2FFodral&_url=http%3A%2F%2Fwww.24.se%2Fvaskorfodral-c-987-1.aspx%3Fsortorder%3D1%26direction%3D0%26defps%3D10%26pagesize%3D30%26pagenum%3D3%26useparams%3D0&a=Mozilla%2F5.0%20(Macintosh%3B%20U%3B%20Intel%20Mac%20OS%20X%2010.6%3B%20sv-SE%3B%20rv%3A1.9.2)%20Gecko%2F20100115%20Firefox%2F3.6&aid=jetshop&l=sv-se&n=netscape&o=view_tag&p=macintel&r=http%3A%2F%2Fwww.24.se%2Fvaskorfodral-c-987-1.aspx%3Fsortorder%3D1%26direction%3D0%26defps%3D10%26pagesize%3D30%26pagenum%3D2%26useparams%3D0&s=1280x800&sid=www.24.se&t=V%C3%A4skor%2FFodral&u=http%3A%2F%2Fwww.24.se%2Fvaskorfodral-c-987-1.aspx%3Fsortorder%3D1%26direction%3D0%26defps%3D10%26pagesize%3D30%26pagenum%3D3%26useparams%3D0&uid=1256057859704610385&x=32058&z=-60& HTTP/1.1" 200 - 35 35 6 5 "http://www.24.se/vaskorfodral-c-987-1.aspx?sortorder=1&direction=0&defps=10&pagesize=30&pagenum=3&useparams=0" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; sv-SE; rv:1.9.2) Gecko/20100115 Firefox/3.6" -}
+      match = @line.match(S3_FORMAT)
+      if match
+        @line = %Q(#{match[4]} - - #{match[3]} #{match[9]} #{match[10]} #{match[12]} #{match[16]} #{match[17]} "-" "-"\n)
+      end
+      @line
+    end
+
+    APACHE_WITHOUT_COOKIES = Regexp.new('^' + [S,S,S,TIMESTAMP,Q,S,S,Q,Q].join(' ') + '$')
+    def normalize_apache_format
+      # Add third party cookies at end if they are not there.
+      # %Q{124.191.88.9 - - [26/May/2009:23:59:50 +0000] "GET /log.gif" "Mozilla/5.0"}
+      match = @line.match(APACHE_WITHOUT_COOKIES)
+      if match
+        @line = %Q(#{match[0]} "-" "-"\n)
+      end
+      @line
+    end
+
+    def normalize_timestamp
+      # 12/Apr/2010:09:07:23 +0200 => 12/Apr/2010:07:07:23 +0000
+      match = @line.match(/^(.*?\[)([^\]]+)(\].+)$/m)
+      unless match[2].end_with?('0000')
+        @line = "#{match[1]}#{timestamp.strftime('%d/%b/%Y:%H:%M:%S +0000')}#{match[3]}"
+      end
+      @line
+    end
+
+  end
+  
+end
+