logbox 0.2.10

data/.bundle/config

@@ -0,0 +1,3 @@
+--- 
+BUNDLE_DISABLE_SHARED_GEMS: "1"
+BUNDLE_WITHOUT: ""

data/.rvmrc

@@ -0,0 +1,2 @@
+rvm use 1.8.7-p302@logbox --create
+rvm use 1.9.2-p0@logbox --create

data/Gemfile

@@ -0,0 +1,17 @@
+source "http://rubygems.org"
+
+gem 'right_aws', '~>2.0'
+gem 'single_instance'
+gem 'rake'
+
+group :test do
+  gem 'shoulda', '2.10.3'
+  gem 'mocha', '0.9.8'
+end
+
+# required for jeweler
+group :development do
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.1"
+  gem "rcov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,30 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.5.1)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    mocha (0.9.8)
+      rake
+    rake (0.8.7)
+    rcov (0.9.9)
+    right_aws (2.0.0)
+      right_http_connection (>= 1.2.1)
+    right_http_connection (1.2.4)
+    shoulda (2.10.3)
+    single_instance (0.2.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.5.1)
+  mocha (= 0.9.8)
+  rake
+  rcov
+  right_aws (~> 2.0)
+  shoulda (= 2.10.3)
+  single_instance

data/README

@@ -0,0 +1,14 @@
+== A Toolbox for Logs and Observations
+
+Two main uses:
+
+1. Use the tools in bin to download and view logs etc.
+2. Include logbox.rb and use the code directly.
+
+StreamWrapper, LogParser and Observation are the main entry points to the code.
+
+== Development and testing
+
+Running `rake` will run the test suite for ruby 1.8.7 and 1.9.2.
+
+Run `rake setup` to generate shell commands that will enable you to run all the tests.

data/Rakefile

@@ -0,0 +1,74 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "logbox"
+  gem.executables = %w(download_logs obsstats rotate viewobs)
+  gem.homepage = "https://github.com/icehouse/logbox"
+  gem.license = "MIT"
+  gem.summary = %Q{A Toolbox for Logs and Observations}
+  gem.description = %Q{Log-related code and tools that are used cross different applications}
+  gem.email = "dev@icehouse.se"
+  gem.authors = ["dvrensk", "enarsson", "Jell"]
+  gem.files.include(["lib/*"])
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "logbox #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+RUBIES = %w[1.8.7-p302 1.9.2-p0]
+desc "Run tests with ruby 1.8.7 and 1.9.2"
+task :default do
+  RUBIES.each do |ruby|
+    sh "rvm #{ruby}@logbox rake test"
+  end
+end
+
+desc "Install bundler and gems for each environment in #{RUBIES.join','}"
+task :setup do
+  cmd = []
+  RUBIES.each do |ruby|
+    cmd << "rvm --create #{ruby}@logbox"
+    cmd << "gem list | grep -q ^rake || gem install rake"
+    cmd << "gem list | grep -q ^bundler || gem install bundler"
+    cmd << "bundle install"
+  end
+  puts "# RUN THE FOLLOWING LINES IN YOUR SHELL"
+  puts cmd
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.10

data/bin/download_logs

@@ -0,0 +1,20 @@
+#! /usr/bin/env ruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'setup_environment'
+require 'optparse'
+require 'observation_compiler'
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: download_logs number_of_days\n" +
+                "Downloads logs from all log servers and merge them to one log file per day. Observations are ordered."
+end
+
+ENV['OBSENTER_S3_KEY'] ||= "AKIAI6RLB45ZDVINPZ3Q"
+
+if ARGV.size == 1 && ARGV[0] =~ /^\d+$/
+  start_date = Date.today - ARGV[0].to_i + 1
+  job = ObservationCompiler::Job.new(:processed_logs_path => File.exist?("/apps/observation_logs") ? "/apps/observation_logs" : "local_files")
+  job.fetch_and_merge(start_date..Date.today)
+else
+  puts option_parser
+end

data/bin/obsstats

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'setup_environment'
+require 'observation'
+require 'stream_wrapper'
+require 'ansi_colors'
+require 'set'
+require 'yaml'
+
+filters = {
+  :valid => true,
+  :skip_debug_observations => true
+}
+observations = StreamWrapper.open(ARGV, filters)
+
+shops = Hash.new do |h,k|
+  h2 = Hash.new(0)
+  h2[:u_items] = Set.new
+  h2[:u_users] = Set.new
+  h[k] = h2
+end
+
+observations.each do |obs|
+  stats = shops["#{obs[:account_id]}/#{obs[:shop_id]}"]
+  stats[obs.type]  += 1
+  stats[:u_items] << obs[:item_id] if obs[:item_id]
+  stats[:u_users] << obs[:user_id] if obs[:user_id]
+end
+
+shops.each_pair do |k,stats|
+  [:u_items, :u_users].each do |u|
+    stats[u] = stats[u].size
+  end
+end
+
+shops.keys.sort.each do |shop_id|
+  stats = shops[shop_id]
+  puts stats.to_yaml.sub('---', "*** #{shop_id}")
+end

data/bin/rotate

@@ -0,0 +1,17 @@
+#! /usr/bin/env ruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'logbox/observation_mover'
+
+if ARGV.first == '--act-as-fake-nginx'
+  ObservationMover.act_as_fake_nginx
+else
+  ObservationMover.new(*ARGV).run
+end
+
+# To test, run with
+#   rm -rf testlogs
+#   bin/rotate --act-as-fake-nginx &
+# and then
+#   bin/rotate testlogs/test.log $(pwd)/test.pid
+#   ls -l testlogs
+# Repeat.

data/bin/viewobs

@@ -0,0 +1,198 @@
+#!/usr/bin/env ruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+require 'setup_environment'
+require 'log_parser'
+require 'observation'
+require 'ansi_colors'
+require 'stream_wrapper'
+
+
+# We don't need a stack dump when interrupted with Ctrl-C
+trap("INT", "EXIT")
+
+# Make symbols sortable.
+class Symbol
+  def <=>(other)
+    self.to_s <=> other.to_s
+  end
+end
+
+class Viewobs
+  attr_accessor :options
+  
+  def parse_options
+    require 'optparse'
+    options = { :delimiter => "\t" }
+    option_parser = OptionParser.new do |opts|
+      opts.banner = "Usage: viewobs.rb [options] files"
+      opts.separator ""
+      opts.separator "Observation filters:"
+      opts.on("-i IP", "--ip IP", "Filter on IP.") { |v| options[:filter_ip] = v }
+      opts.on("-s SHOP_ID", "--shop SHOP_ID", "Filter on shop ID.") { |v| options[:filter_shop_id] = v }
+      opts.on("-a ACCOUNT_ID", "--account ACCOUNT_ID", "Filter on account ID.") { |v| options[:filter_account_id] = v }
+      opts.on("-t TYPE", "--type TYPE", "Filter on observation type.") { |v| options[:filter_type] = v }
+      opts.on("-v", "--valid", "Display only valid observations.") { |v| options[:filter_valid] = v }
+      opts.separator ""
+      opts.separator "Display filters:"
+      opts.on("-A", "--all", "Display all attributes.") { |v| options[:all_attributes] = v }
+      opts.on("-u", "--user", "Display user set attributes.") { |v| options[:user_attributes] = v }
+      opts.on("-o", "--observation", "Display observation attributes.") { |v| options[:observation_attributes] = v }
+      opts.on("-e", "--errors", "Display errors (if any).") { |v| options[:errors] = v }
+      opts.on("-r", "--request", "Display raw request.") { |v| options[:request] = v }
+      opts.separator ""
+      opts.separator "Other options:"
+      opts.on("-f", "--follow", "Waits at the end of the log for updates.") { |v| options[:follow] = v }
+      opts.on("-c", "--colorize", "Colorize output.") { |v| String.colorize }
+      opts.on("-T", "--table", "Output as table or csv.", "Does not work with data from stdin (reads twice).") { options[:table] = true }
+      opts.on("-d CHAR", "--delimiter CHAR", "Delimiter for table (default is tab).") { |v| options[:delimiter] = v[0,1] }
+      opts.separator ""
+    end
+
+    begin
+      # Parse will leave any filename argument intact in ARGV but it will remove all options.
+      option_parser.parse!(ARGV)
+    rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
+      puts e
+      puts option_parser
+      exit 1
+    end
+    @options = options
+  end
+
+
+  def view_line(attributes, observation)
+    # Print basic information.
+    out = ""
+    out << "#{observation.attributes[:account_id].to_s.red}" if observation.attributes[:account_id]
+    out << " - #{observation.attributes[:shop_id].to_s.red}" if observation.attributes[:shop_id]
+    out << " - #{attributes[:o].blue}" if attributes[:o] # Observation type.
+    out << " - #{attributes[:timestamp].strftime('%Y-%m-%d %H.%M.%S')}"
+    out << " - #{attributes[:ip]}"
+    out << " - #{attributes[:status]}"
+    out << " - #{'valid'.green}" if observation.valid?
+    puts out
+
+    # Print extended information depending on options.
+    if options[:all_attributes]
+      attributes.keys.sort.each do |key|
+        puts "  #{key.to_s.blue}: #{attributes[key]}" unless [:ip, :request, :timestamp, :status].include?(key)
+      end
+      puts
+    end
+    if options[:user_attributes]
+      attributes.keys.sort.each do |key|
+        puts "  #{key.to_s[1..-1].blue}: #{attributes[key]}" if key.to_s[0..0] == '_'
+      end
+      puts
+    end
+    if options[:observation_attributes]
+      observation.attributes.keys.sort.each do |key|
+        puts "  #{key.to_s.blue}: #{observation.attributes[key]}" if observation.attributes.has_key?(key) && key != :request
+      end
+      puts
+    end  
+    if options[:errors]
+      puts observation.errors.inspect unless observation.valid?
+      puts
+    end  
+    if options[:request]
+      puts attributes[:request]
+      puts
+    end  
+  end
+
+  def view_table_line(attributes, observation)
+    # Print basic information.
+    out = []
+    out << observation.attributes[:account_id].to_s.red
+    out << observation.attributes[:shop_id].to_s.red
+    out << attributes[:o].blue # Observation type.
+    out << attributes[:timestamp].strftime('%Y-%m-%d %H.%M.%S')
+    out << attributes[:ip]
+    out << attributes[:status]
+    out << (observation.valid? ? 'valid' : 'invalid')
+
+    @all_keys.each do |key|
+      out << attributes[key] unless [:o, :ip, :request, :timestamp, :status].include?(key)
+    end
+
+    puts out.map { |s| s.to_s.tr("#{delimiter}\0- "," ") }.join(delimiter)
+  end
+
+  def view_until_end
+    # gets will take lines from any filename given in ARGV or from stdin.
+    pingdom_re = /pingdom/
+    filters = {
+      :account_id => options[:filter_account_id],
+      :shop_id => options[:filter_shop_id],
+      :observation_type => options[:filter_type]
+    }
+    stream = StreamWrapper.open(ARGV, filters)
+    while line = stream.gets
+      attributes = LogParser.parse_line(line)
+      observation = Observation.new(attributes)
+
+      # Skip lines from pingdom.
+      next if attributes[:request].match(pingdom_re)
+
+      # Filter depending on options.
+      if options[:filter_ip]
+        next unless attributes[:ip].match(options[:filter_ip])
+      end
+      if options[:filter_valid]
+        next unless observation.valid?
+      end
+
+      begin
+        if options[:gathering_keys]
+          gather_keys(attributes, observation)
+        elsif options[:table]
+          view_table_line(attributes, observation)
+        else
+          view_line(attributes, observation)
+        end
+      rescue Exception => e
+        $stderr.puts "Could not render line #{$.} (#{e.message})"
+      end
+    end
+    if options[:gathering_keys]
+      @all_keys = @all_keys.sort
+    end
+  end
+
+  def gather_keys (attributes, observation)
+    @all_keys ||= []
+    @all_keys = @all_keys | attributes.keys
+  end
+
+  def print_header
+    print %w[account_id shop_id o timestamp ip status valid].join(delimiter)
+    print delimiter
+    puts (@all_keys - [:o, :timestamp, :ip, :status, :request]).map { |sym| sym.to_s }.join(delimiter)
+  end
+  
+  def delimiter
+    options[:delimiter]
+  end
+  
+end
+
+if File.basename($0) == File.basename(__FILE__)
+  viewobs = Viewobs.new
+  viewobs.parse_options
+
+  if viewobs.options[:follow]
+    while true
+      viewobs.view_until_end
+      sleep(0.1)
+    end
+  else
+    if viewobs.options[:table]
+      viewobs.options[:gathering_keys] = true
+      viewobs.view_until_end
+      viewobs.print_header
+      viewobs.options[:gathering_keys] = false
+    end
+    viewobs.view_until_end
+  end
+end

data/lib/logbox.rb

@@ -0,0 +1,9 @@
+$: << File.join(File.dirname(__FILE__), 'logbox')
+require 'stream_wrapper'
+require 'log_parser'
+require 'observation'
+require 'iconv'
+
+module Logbox
+  StringEncoder = Iconv.new('UTF-8//IGNORE', 'UTF-8')
+end

data/lib/logbox/ansi_colors.rb

@@ -0,0 +1,28 @@
+class String
+
+  types = {
+    :bold => "\e[1m",
+    :underline => "\e[4m",
+    :black => "\e[30m", 
+    :red => "\e[31m",
+    :green => "\e[32m",
+    :yellow => "\e[33m",
+    :blue => "\e[34m",
+    :magenta => "\e[35m",
+    :cyan => "\e[36m", 
+    :white => "\e[37m", 
+  }
+
+  types.each do |name, color_code|
+    define_method(name) do
+      @@colorize ? "#{color_code}#{self}\e[0m" : self
+    end
+  end
+
+  class << self
+    @@colorize = false
+    def colorize
+      @@colorize = true
+    end
+  end
+end