logbox 0.2.10

data/lib/logbox/observation_mover.rb

@@ -0,0 +1,142 @@
+require 'digest/sha1'
+require 'digest/md5'
+require 'yaml'
+
+require 'rubygems'
+require 'single_instance'
+require 'right_aws'
+
+class ObservationMover
+
+  DEFAULT_KEY_FILE = '/etc/s3_key.yml'
+  DEFAULT_BUCKET = 'rwdata-logs'
+
+  def initialize (path, pid_file = '/var/run/nginx.pid')
+    @dir, @file_name = File.split(path)
+    @basename = File.basename(@file_name, File.extname(@file_name))
+    @pid_file = pid_file
+  end
+
+  def file_path
+    File.join(@dir, @file_name)
+  end
+
+  def run
+    SingleInstance.exclusive_non_blocking(:move_observations) do
+      prepare_rotated_for_archiving
+      move_prepared_to_archive
+      rotate_current && notify_logger
+    end
+  end
+
+  def move_prepared_to_archive
+    prepared_for_archiving.each do |file|
+      store_file_on_s3(file) && delete_file(file)
+    end
+  end
+
+  # TODO: report transmission errors:
+  # W, [2010-04-23T09:10:49.396732 #30275]  WARN -- : ##### RightAws::S3Interface returned an error: 400 Bad Request
+  # <?xml version="1.0" encoding="UTF-8"?>
+  # <Error><Code>BadDigest</Code><Message>The Content-MD5 you specified did not match what we received.</Message><ExpectedDigest>5f7d35353204db919b646cd3eeeedca2</ExpectedDigest><CalculatedDigest>X301NTIE25GbZGzT7u7cog==</CalculatedDigest><RequestId>CA99C7EA27C5563D</RequestId><HostId>n125lcdYZDbNTV1Wy8OSfL9W5itYbB7wJtLHCo0Uzq/gKfXbu6hzUzwOP9dgkKXy</HostId></Error> #####
+  # W, [2010-04-23T09:10:49.396788 #30275]  WARN -- : ##### RightAws::S3Interface request: rwdata-logs.s3.amazonaws.com:443/observer-log-2010-04-17-23-20-01-4ef6cbdef683b156 ####
+  # /usr/lib/ruby/gems/1.8/gems/right_aws-1.10.0/lib/awsbase/right_awsbase.rb:359:in `request_info_impl': BadDigest: The Content-MD5 you specified did not match what we received. (RightAws::AwsError)
+  
+  def store_file_on_s3 (file)
+    File.open file do |f|
+      s3.put(bucket, File.basename(file), f, "Content-MD5" => md5(file), 'Content-Type' => 'text/plain')
+    end
+    true
+  end
+
+  def delete_file (file)
+    File.unlink(file)
+  end
+
+  def prepare_rotated_for_archiving
+    Dir[rotated_glob].each do |file|
+      signed_name = "#{file}-#{short_sha(file)}"
+      File.rename file, signed_name
+      system "gzip", signed_name
+    end
+  end
+
+  def rotate_current
+    if File.size?(file_path)
+      rename_current
+      create_new
+      true
+    else
+      false
+    end
+  end
+
+  def notify_logger
+    Process.kill 'USR1', File.read(@pid_file).to_i if File.exists?(@pid_file)
+  end
+
+  ##
+
+  def s3
+    @s3 ||= RightAws::S3.new(*aws_keys).interface
+  end
+
+  def aws_keys (file = key_file)
+    hash = YAML.load_file(file)
+    [hash[:access_key_id], hash[:secret_access_key]]
+  end
+
+  def md5 (path)
+    [Digest::MD5.file(path).digest].pack('m').chomp
+  end
+
+  def key_file
+    @key_file ||= DEFAULT_KEY_FILE
+  end
+  attr_writer :key_file
+
+  def bucket
+    @bucket ||= DEFAULT_BUCKET
+  end
+  attr_writer :bucket
+
+  def short_sha (file)
+    Digest::SHA1.file(file).hexdigest[0,16]
+  end
+
+  def rename_current
+    File.rename file_path, rotated_path if File.exists?(file_path)
+  end
+
+  def create_new
+    File.new(file_path, "a", 0644).close
+  end
+
+  TIME_FORMAT = '-%Y-%m-%d-%H-%M-%S'
+
+  def rotated_path
+    File.join(@dir, @file_name.gsub('.','-') + Time.now.strftime(TIME_FORMAT))
+  end
+
+  def rotated_glob
+    file_part = @file_name.gsub('.','-')
+    time_part = Time.now.strftime(TIME_FORMAT).gsub(/\d/,'[0-9]')
+    File.join(@dir, file_part + time_part)
+  end
+
+  def prepared_for_archiving
+    glob = rotated_glob + '-' + ('[0-9a-f]' * 16) + ".gz"
+    Dir[glob]
+  end
+
+  def self.act_as_fake_nginx
+    Dir.mkdir 'testlogs' unless File.exists?('testlogs')
+    File.open('testlogs/test.log', 'a') { |f| f.puts "loggy" }
+    File.open('test.pid', 'w') { |f| f.print $$ }
+    Signal.trap("USR1") { puts "USR1 @ #{Time.now}" }
+    puts "Listening for USR1s"
+    10.times { sleep 10}
+    puts "Done listening for USR1s"
+  end
+
+end

data/lib/logbox/stream_wrapper.rb

@@ -0,0 +1,20 @@
+$: << File.dirname(__FILE__)
+require "stream_wrapper/gzip_multi_file"
+require "stream_wrapper/observation_filter"
+require "stream_wrapper/order_blob_splitter"
+
+module StreamWrapper
+
+  class << self
+
+    def open(paths = [], filters = nil)
+      if filters
+        ObservationFilter.new(OrderBlobSplitter.new(GzipMultiFile.new(paths, filters)), filters)
+      else
+        OrderBlobSplitter.new(GzipMultiFile.new(paths))
+      end
+    end
+    
+  end  
+  
+end

data/lib/logbox/stream_wrapper/gzip_multi_file.rb

@@ -0,0 +1,90 @@
+module StreamWrapper
+  class GzipMultiFile
+
+    def initialize (paths = [], filters = {})
+      @paths = paths.is_a?(Array) ? paths.dup : [paths]
+      @filters = filters
+      if @paths.empty?
+        @current_line = 0
+        @current_file = $stdin
+        @current_path = '<stdin>'
+      end
+    end
+
+    def gets
+      if file = current_stream
+        @current_line += 1
+        file.gets
+      else
+        nil
+      end
+    end
+
+    def eof?
+      ! current_stream
+    end
+
+    def close
+      @current_file.close unless @current_file.nil?
+    end
+
+    def current_position
+      "#{@current_path}:#{@current_line}"
+    end
+
+    def debug= (bool)
+      @debug = bool
+    end
+
+    def debug?
+      @debug
+    end
+
+    protected
+
+    def log (message)
+      $stderr.puts message if debug?
+    end
+    
+    def current_stream
+      if @current_file.nil? || @current_file.eof?
+        close
+        @current_file = next_available_file
+      end
+      @current_file
+    end
+
+    def next_available_file
+      if @current_path
+        log "Closed #{@current_path} after reading #{@current_line} lines"
+      end
+      @current_line = 0
+      @current_path = @paths.shift or return nil
+      log "Switching to #{@current_path}"
+      file = IO.popen(shell_command(@current_path))
+      file.eof? ? next_available_file : file
+    end
+
+    # Use whitelist to be sure. Observation filters is applied after anyways.
+    SHOP_AND_ACCOUNT_ID_FILTER = /^[a-zA-Z0-9._]+$/
+
+    def shell_command (path)
+      if path =~ /\.gz/
+        command = "gunzip -c #{path.gsub(' ','\\ ')}"
+      else
+        command = "cat #{path.gsub(' ','\\ ')}"
+      end
+      if @filters[:shop_id] && @filters[:shop_id] =~ SHOP_AND_ACCOUNT_ID_FILTER  
+        command += " | grep -F sid=#{@filters[:shop_id]}"
+      end
+      if @filters[:account_id] && @filters[:account_id] =~ SHOP_AND_ACCOUNT_ID_FILTER
+        command += " | grep -F aid=#{@filters[:account_id]}"
+      end
+      if @filters[:observation_type]
+        command += " | grep -F o=#{@filters[:observation_type]}"
+      end
+      command
+    end
+    
+  end
+end

data/lib/logbox/stream_wrapper/observation_filter.rb

@@ -0,0 +1,113 @@
+# -*- encoding: utf-8 -*-
+require 'uri'
+require 'observation'
+require 'logbox'
+
+module StreamWrapper
+  class ObservationFilter
+    include Enumerable
+    attr_reader :observation_count
+
+    # Support both strings and streams as input.
+    def initialize(input, filter_options)
+      input = StringIO.new(input) if input.class == String
+      @stream = input
+      # Quick check of options. It would be easy to set :valid => false but it is not allowed.
+      if filter_options.detect { |key, value| value == false }
+        raise "A filter option can not have false as value. Use skip instead." 
+      end
+      @filter_options = filter_options
+      build_filters @filter_options
+    end
+
+    # Get next row and remove invalid utf-8 byte sequences.
+    def gets
+      while row = @stream.gets
+        row = Logbox::StringEncoder.iconv(row)
+        break if keep? row
+      end
+      row
+    end
+
+    def eof?
+      @stream.eof?
+    end
+  
+    # Enumerate over Observations
+    def each
+      @observation_count = 0
+      while(row = gets)
+        begin
+          attrs = LogParser.parse_line(row) || next
+        rescue LogParser::ParseError => e
+          $stderr.puts("#{e} for row:#{row}")
+          next
+        end
+        observation = Observation.new(attrs)
+        if (@filter_options[:valid] && observation.valid?) ||
+           (@filter_options[:skip_valid] && !observation.valid?) ||
+           (@filter_options[:valid].nil? && @filter_options[:skip_valid].nil?)
+          yield observation
+          @observation_count += 1
+        end
+      end
+    end
+  
+  protected
+
+    def build_filters options
+      @filters = []
+      options.keys.each do |full_key|
+        # Filter keys can be of format :account_id or :skip_account_id.
+        key_fragments = /^(skip_)?(.*)$/.match(full_key.to_s)
+        key = key_fragments[2].to_sym
+        skip = (key_fragments[1] == "skip_")
+
+        if options[full_key] != true
+          # Expressions are url escaped in log file. Encode same characters as
+          # JavaScript (as far as we can know).
+          # See https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Global_Functions/encodeURIComponent
+          expression = URI.escape(options[full_key].to_s, /[^-_.!~*'()a-zA-Z\d]/n)
+                
+          if key == :account_id
+            @filters << [skip, Regexp.new(Regexp.escape("aid=#{expression}"), true)]
+          end
+
+          if key == :shop_id
+            @filters << [skip, Regexp.new(Regexp.escape("sid=#{expression}"), true)]
+          end
+
+          if key == :observation_type
+            @filters << [skip, Regexp.new(Regexp.escape("o=#{expression}"), true)]
+          end
+        end
+      
+        if key == :debug_observations && options[full_key] == true
+          if skip
+            # When SKIPPING tests, we can have SEVERAL regular expressions.
+            # Watch out for debug flag, but also known ips.
+            @filters << [skip, /^81\.231\.246\.117/] # ICU Intellegence
+            @filters << [skip, /^84\.218\.38\.122/] # Nils Svangård
+            @filters << [skip, /d=debug/]
+          else
+            @filters << [skip, /d=debug/]
+          end
+        end
+      end
+    end
+
+    def keep? row
+      # All filters must match to accept row.
+      @filters.each do |filter|
+        skip, regexp = filter
+        if skip
+          return false if row =~ regexp
+        else
+          return false unless row =~ regexp
+        end
+      end
+      true
+    end
+  end
+end
+

data/lib/logbox/stream_wrapper/order_blob_splitter.rb

@@ -0,0 +1,96 @@
+# -*- encoding: utf-8 -*-
+module StreamWrapper
+  
+  # Modfies and injects buy_basket blob information into stream.
+  # 1. Inserts attributes from buy_basket blob into the buy_basket observation.
+  # 2. Adds a buy_item observation for each item in the basket.
+  class OrderBlobSplitter
+
+    def initialize (input)
+      input = StringIO.new(input) if input.class == String  # Used for testing.
+      @stream = input
+      @buy_item_lines = []
+    end
+
+    def gets
+      if @buy_item_lines.size > 0
+        return @buy_item_lines.shift
+      end
+      
+      line = @stream.gets
+      if line && line.include?("&o=buy_basket&")
+        extract_buy_items(line)
+        line = reformat_buy_basket(line)
+      end
+      line
+    end
+
+    def eof?
+      @stream.eof? 
+    end
+
+  private
+  
+    # Blob format:
+    # RW:T|2142||1079.00|215.80|0.00|strömstad||Sweden|
+    # RW:I|2142|MATAW09 Wanni L|Overall (98)||1079.00|1|
+    #
+    # See also http://www.google.com/support/googleanalytics/bin/answer.py?hl=en&answer=55528
+
+    def reformat_buy_basket(line)
+      blob = extract_order_blob(line)
+      order_blob = blob.match(/^RW:T\|(([^\|]*\|[^\|]*){7})\|/)[1]
+      values = order_blob.split("|")
+      attributes = {}
+      [:order_id, :affiliation, :total, :tax, :shipping, :city, :state, :country].each_with_index do |name, index|
+        attributes[name] = values[index]
+      end
+      insert_user_attributes(line, attributes)
+    rescue
+      line
+    end
+    
+    def extract_buy_items(line)
+      blob = extract_order_blob(line)
+      blob.scan(/RW:I\|(([^\|]*\|[^\|]*){5})\|/) do |item_blob|
+        values = item_blob[0].split("|")
+        attributes = {}
+        [:order_id, :sku, :title, :category, :current_price, :quantity].each_with_index do |name, index|
+          attributes[name] = values[index]
+        end
+        @buy_item_lines << insert_user_attributes(line, attributes, :replace).gsub("&o=buy_basket", "&o=buy_item")
+      end
+    end
+    
+    def extract_order_blob(line)
+      CGI::unescape(line.match(/_order_blob=([^&]+)&/)[1]) rescue ""
+    end
+    
+    # 81.225.100.141 - - [23/Nov/2009:06:45:23 +0100] "GET /log.gif?_order_blob=RW%3AT%7C2142%7C%7C1079.00%7C215.80%7C0.00%7Cstr%C3%B6mstad%7C%7CSweden%7C%20%0ARW%3AI%7C2142%7CMATAW09%20Wanni%20L%7COverall%20(98)%7C%7C1079.00%7C1%7C&_order_nr=002142&a=Mozilla%2F4.0%20(compatible%3B%20MSIE%208.0%3B%20Windows%20NT%205.1%3B%20Trident%2F4.0%3B%20Mozilla%2F4.0%20(compatible%3B%20MSIE%206.0%3B%20Windows%20NT%205.1%3B%20SV1)%20%3B%20.NET%20CLR%202.0.50727%3B%20.NET%20CLR%203.0.4506.2152%3B%20.NET%20CLR%203.5.30729)&aid=jetshop&l=sv&n=microsoft%20internet%20explorer&o=buy_basket&p=win32&s=1024x600&sid=www.pixiekids.se&t=Unika%20barnkl%C3%A4der%20%26%20babykl%C3%A4der%20hos%20Pixiekids%20%E2%8E%AE%200-10%20%C3%A5r&u=https%3A%2F%2Fwww.pixiekids.se%2FOrderDetailsConfirmed.aspx&uid=1258954551578656003&x=87544&z=-60& HTTP/1.1" 200 35 "https://www.pixiekids.se/OrderDetailsConfirmed.aspx" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1) ; .NET CLR 2.0.50727; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)"
+    def insert_user_attributes(line, attributes, method = :merge)
+      encoded_attributes = []
+      attributes_regexp = /\?(.*?)&(?!_)/
+      # Add existing attributes from line.
+      unless method == :replace
+        encoded_attributes = line.match(attributes_regexp)[1].split("&").map { |pair| pair.split("=") }
+      end
+      # Add in attributes from method argument.
+      attributes.each do |key, value|
+        next if value.nil? || value == ""
+        encoded_key = CGI.escape("_#{key}")
+        encoded_value = CGI.escape(value)
+        encoded_attributes.reject!{ |k, v| k == encoded_key }
+        encoded_attributes << [encoded_key, encoded_value]
+      end
+      # Reconstruct the user attributes part of the line.
+      attributes_string = encoded_attributes.sort.map do |key, value|
+        "#{key}=#{value}&"
+      end.join
+      # Replace the old user attributes in the line with the new ones.
+      line.sub(attributes_regexp, "?#{attributes_string}")
+    end
+    
+  end
+end
+
+