logbox 0.2.10

data/test/observation_mover_test.rb

@@ -0,0 +1,135 @@
+require 'test_helper'
+require 'observation_mover'
+
+class ObservationMoverTest < Test::Unit::TestCase
+  context "ObservationMover" do
+    setup do
+      setup_log_dir
+      @mover = ObservationMover.new(log_file_path)
+    end
+
+    context "#rotate" do
+      context "with a non-empty log file" do
+        should "move the log file" do
+          @mover.rotate_current
+          rotated = rotated_path!
+          assert_not_nil rotated
+          assert_equal "loggy\n", File.read(rotated)
+        end
+        should "create a new, empty log file" do
+          @mover.rotate_current
+          assert File.exists?(log_file_path)
+          assert ! File.size?(log_file_path)
+        end
+        should "return true" do
+          assert @mover.rotate_current
+        end
+      end
+
+      context "with an empty log file" do
+        setup do
+          File.open(log_file_path, "w").close
+        end
+        should "not rename the log file" do
+          @mover.rotate_current
+          assert_equal [log_file_path], Dir[log_file_path('*')]
+        end
+        should "return false" do
+          assert ! @mover.rotate_current
+        end
+      end
+    end
+
+    context "#prepare_rotated_for_archiving" do
+      setup do
+        @mover.rotate_current
+      end
+      should "rename a rotated file" do
+        rotated = rotated_path!
+        @mover.prepare_rotated_for_archiving
+        assert ! File.exists?(rotated), "#{rotated} exists!"
+      end
+      should "compress a rotated file" do
+        rotated = rotated_path!
+        @mover.prepare_rotated_for_archiving
+        assert ! Dir[rotated + '*.gz'].empty?
+      end
+    end
+
+    context '#move_prepared_to_archive' do
+      context "with one file with a sha" do
+        setup do
+          @shad_name = log_file_path('observer-log-2010-01-05-21-13-11-1234567890abcdef.gz')
+          File.open(@shad_name, 'w') { |f| f << "logline\n" }
+        end
+        should "move that file to s3" do
+          @mover.expects(:store_file_on_s3).with(@shad_name).returns(true)
+          @mover.move_prepared_to_archive
+        end
+        should "delete the file if the move succeeds" do
+          @mover.expects(:store_file_on_s3).with(@shad_name).returns(true)
+          @mover.move_prepared_to_archive
+          assert ! File.exists?(@shad_name)
+        end
+        should "not delete the file if the move fails" do
+          @mover.expects(:store_file_on_s3).with(@shad_name).returns(false)
+          @mover.move_prepared_to_archive
+          assert File.exists?(@shad_name)
+        end
+        # should "talk to s3" do
+        #   #WebMock.disable_net_connect!
+        #   #WebMock.allow_net_connect!
+        #   # stub_request(:put, %r'https://rwdata-logs.s3.amazonaws.com/.*').to_return do |request|
+        #   #   raise request.inspect
+        #   # end
+        #   # PUT https://rwdata-logs.s3.amazonaws.com/observer-log-2010-01-05-21-13-11-1234567890abcdef with body 'logline\n' with headers {'Accept'=>'*/*', 'Authorization'=>'AWS AAAAAAAAAA:4aYODn+8Yx7nIGnGf1coqWbEjWY=', 'Content-Length'=>'8', 'Content-Md5'=>'2ee0c171dab3e30627525ff535355eba', 'Content-Type'=>'', 'Date'=>'Thu, 'User-Agent'=>'', 22 Apr 2010 12:11:09 GMT'}
+        #   @mover.move_prepared_to_archive
+        # end
+        # should "talk to s3" do
+        #   WebMock.disable_net_connect!
+        #   #WebMock.allow_net_connect!
+        #   stub_request(:any, 'https://rwdata-logs.s3.amazonaws.com/?location').to_return(:status => [500, 'Internal Server knas'])
+        #   @mover.s3.bucket_location('rwdata-logs')
+        # end
+      end
+    end
+
+    context '#aws_keys' do
+      should "read from a yaml file and return a list" do
+        file = fixture_file('aws_keys_yaml.txt')
+        key, secret = @mover.aws_keys(file)
+        assert_equal 'PUBLICKEY', key
+        assert_equal 'SECRET/KEY', secret
+      end
+    end
+
+    context '#md5' do
+      should "return a Base64-coded string" do
+        file = fixture_file('../fixtures/log-for-md5.log')
+        assert_equal 'X301NTIE25GbZGzT7u7cog==', @mover.md5(file)
+      end
+    end
+  end
+
+  def log_file_path (file_name = 'observer.log')
+    File.join(log_dir, file_name)
+  end
+
+  def log_dir
+    '/tmp/test/logs'
+  end
+
+  def setup_log_dir
+    FileUtils.mkdir_p log_dir
+    FileUtils.rm_r log_dir
+    FileUtils.mkdir log_dir
+    File.open(log_file_path, "w") { |f| f.puts "loggy" }
+  end
+
+  def rotated_path!
+    paths = Dir[log_file_path('*')] - [log_file_path]
+    assert_equal 1, paths.size, "Not exactly one path in #{paths.inspect}"
+    paths.first
+  end
+
+end

data/test/observation_test.rb

@@ -0,0 +1,114 @@
+require 'test_helper'
+require 'observation'
+
+class ObservationTest < Test::Unit::TestCase 
+  
+  context "The Observation when initialized" do
+
+    should "copy and rename attributes" do
+      log_attr = {:r => "referrer", :_item_id => "id", :_user1 => "custom", :ip => "ip"}
+      o = Observation.new(log_attr)
+      assert_equal 0, o.unknown_attributes.size
+      assert_equal 4, o.attributes.size
+      assert_equal "referrer", o.attributes[:referrer]
+      assert_equal "id", o.attributes[:item_id]
+      assert_equal "custom", o.attributes[:user1]
+      assert_equal "ip", o.attributes[:ip]
+    end
+
+    should "convert local urls to absolute urls" do
+      log_attr = {
+        :_url => "../index.html", 
+        :_thumbnail => 'http://www.test.com/te st.html#debug',
+        :_image => 'http://www.test.com/te%st.html#debugon',
+        :_basket_url => 'http://www.test.com/te%20st.html#debugoff',
+        :u => 'http://www.test.com/f1/f2/index.html#debugon'
+      }
+      o = Observation.new(log_attr)
+      assert_equal 0, o.unknown_attributes.size
+      assert_equal 5, o.attributes.size
+      assert_equal "http://www.test.com/f1/index.html", o.attributes[:url]
+      assert_equal "http://www.test.com/te%20st.html", o.attributes[:thumbnail]
+      assert_equal "http://www.test.com/te%25st.html", o.attributes[:image]
+      assert_equal "http://www.test.com/te%20st.html", o.attributes[:basket_url]
+    end
+
+    should "convert not tamper with javascript urls" do
+      log_attr = {
+        :_basket_url => "javascript:alert('basket')", 
+        :u => 'http://www.test.com/'
+      }
+      o = Observation.new(log_attr)
+      assert_equal "javascript:alert('basket')", o.attributes[:basket_url]
+    end
+
+    should "report error for invalid urls" do
+      log_attr = {
+        :_url => "../index.html", 
+        :u => 'index.html#anchor2'
+      }
+      o = Observation.new(log_attr)
+      assert o.errors.include?(:url_error_on_url)
+    end
+
+    should "validate that status is ok (200)" do
+      log_attr = { :status => '200' }
+      o = Observation.new(log_attr)
+      assert !o.errors.include?(:status_not_ok)
+
+      log_attr = { :status => '404' }
+      o = Observation.new(log_attr)
+      assert o.errors.include?(:status_not_ok)
+    end
+    
+    should "validate type attribute" do
+      log_attr = { :o => "view_item", :status => "200" }
+      o = Observation.new(log_attr)
+      assert !o.errors.include?(:unknown_type)
+      assert_equal :view_item, o.type
+
+      log_attr = { :o => "pick_item", :status => "200" }
+      o = Observation.new(log_attr)
+      assert !o.errors.include?(:unknown_type)
+      assert_equal :pick_item, o.type
+
+      log_attr = { :o => "visit_page", :status => "200" }
+      o = Observation.new(log_attr)
+      assert !o.errors.include?(:unknown_type)
+      assert_equal :visit_page, o.type
+
+      log_attr = { :o => "apa", :status => "200" }
+      o = Observation.new(log_attr)
+      assert o.errors.include?(:unknown_type)
+      assert_equal :unknown, o.type
+    end
+    
+    should "validate required attributes" do
+      other_names = Observation::BASIC_ATTRIBUTES_NAMES.merge(Observation::OBSERVATION_ATTRIBUTES_NAMES)
+      [ :uid, :sid, :aid, :_item_id ].each do |name|
+        error_message = "missing_#{other_names[name]}".to_sym
+        
+        log_attr = { :o => "view_item", :status => "200", name => "1000" }
+        o = Observation.new(log_attr)
+        assert !o.errors.include?(error_message)
+
+        log_attr = { :o => "view_item", :status => "200" }
+        o = Observation.new(log_attr)
+        assert o.errors.include?(error_message), "'#{error_message}' not found in #{o.errors.inspect}"
+      end
+    end
+    
+    should "validate visit_page with shop, account and user id" do
+      assert Observation.new({ :o => "visit_page", :aid => "jshop", :sid => "shirtstore", :uid => "1234", :status => "200" }).valid?
+      assert !Observation.new({ :o => "visit_page", :sid => "shirtstore", :uid => "1234", :status => "200" }).valid?
+      assert !Observation.new({ :o => "visit_page", :aid => "jshop", :uid => "1234", :status => "200" }).valid?
+      assert !Observation.new({ :o => "visit_page", :aid => "jshop", :sid => "shirtstore", :status => "200" }).valid?
+    end
+    should "accept visit_page without item_id as a valid observation" do
+      assert Observation.new({ :o => "visit_page", :status => "200", :uid => '1' * 16, :sid => 'shoop', :aid => 'act'}).valid?
+    end
+    
+  end
+
+end
+

data/test/stream_wrapper/gzip_multi_file_test.rb

@@ -0,0 +1,147 @@
+require 'test_helper'
+require 'stream_wrapper'
+include StreamWrapper
+
+class StreamWrapper::GzipMultiFileTest < Test::Unit::TestCase 
+  context "A GzipMultiFile with one file" do
+    setup { @sw = GzipMultiFile.new fixture_file("log1.log") }
+    
+    should "read from that file with #gets" do
+      assert_equal("Log 1 line 1\n", @sw.gets)
+    end
+    
+    should "return nil from #gets after the last line" do
+      @sw.gets
+      assert_equal(nil, @sw.gets)
+    end
+  end
+
+  context "A GzipMultiFile with data to read" do
+    setup { @sw = GzipMultiFile.new fixture_file("log1.log") }
+    
+    should "not be #eof?" do
+      assert ! @sw.eof?
+    end
+  end
+
+  context "A GzipMultiFile with no data left to read" do
+    setup do
+      @sw = GzipMultiFile.new fixture_file("log1.log")
+      @sw.gets
+    end
+    
+    should "be #eof?" do
+      assert @sw.eof?
+    end
+  end
+
+  context "A GzipMultiFile with two files" do
+    setup do
+      @sw = GzipMultiFile.new [fixture_file("log1.log"), fixture_file("log2.log")]
+    end
+    
+    should "read the line from file 1 first" do
+      assert_equal("Log 1 line 1\n", @sw.gets)
+    end
+    
+    should "read the lines from file 2 after file 1" do
+      @sw.gets
+      assert_equal("Log 2 line 1\n", @sw.gets)
+      assert_equal("Log 2 line 2\n", @sw.gets)
+      assert_equal(nil, @sw.gets)
+    end
+    
+    should "return nil when file 2 is empty" do
+      3.times { @sw.gets }
+      assert_equal(nil, @sw.gets)
+    end
+  end
+
+  context "A GzipMultiFile without files" do
+    setup do
+      @stdin = $stdin
+      $stdin = StringIO.new "Line 1\nLine 2\n"
+      @sw = GzipMultiFile.new
+    end
+    teardown do
+      $stdin = @stdin
+    end
+    
+    should "read from STDIN" do
+      assert_equal("Line 1\n", @sw.gets)
+      assert_equal("Line 2\n", @sw.gets)
+      assert_equal(nil, @sw.gets)
+    end
+  end
+
+  context "A GzipMultiFile with one gzipped file" do
+    setup do
+      @sw = GzipMultiFile.new fixture_file("log1.log.gz")
+    end
+    
+    should "read the unzipped data with #gets" do
+      assert_equal("Log 1 line 1\n", @sw.gets)
+    end
+    
+    should "return nil from #gets after the last line" do
+      @sw.gets
+      assert_equal(nil, @sw.gets)
+    end
+  end
+  
+  context "A GzipMultiFile with two gzipped files" do
+    setup do
+      @sw = GzipMultiFile.new [fixture_file("log1.log.gz"), fixture_file("log2.log.gz")]
+    end
+    
+    should "read the line from file 1 first" do
+      assert_equal("Log 1 line 1\n", @sw.gets)
+    end
+    
+    should "read the lines from file 2 after file 1" do
+      @sw.gets
+      assert_equal("Log 2 line 1\n", @sw.gets)
+      assert_equal("Log 2 line 2\n", @sw.gets)
+      assert_equal(nil, @sw.gets)
+    end
+    
+    should "return nil when file 2 is empty" do
+      3.times { @sw.gets }
+      assert_equal(nil, @sw.gets)
+    end
+  end
+
+  context "If one of the files is empty, the GzipMultiFile" do
+    setup do
+      @sw = GzipMultiFile.new [fixture_file("log0.log"), fixture_file("log2.log")]
+    end
+    
+    should "ignore that file without a glitch" do
+      assert_equal("Log 2 line 1\n", @sw.gets)
+    end
+  end
+
+  context "#shell_command" do
+    should "cat regular files" do
+      command = GzipMultiFile.new.send :shell_command, "file.log"
+      assert_match /^cat file.log/, command
+    end
+    should "unzip zipped files" do
+      command = GzipMultiFile.new.send :shell_command, "file.gz"
+      assert_match /^gunzip -c file.gz/, command
+    end
+    should "add filter for shop" do
+      command = GzipMultiFile.new("", {:shop_id => "www.24.se"}).send :shell_command, "file.gz"
+      assert_match /grep -F sid=www\.24\.se/, command
+    end
+    should "add filter for account" do
+      command = GzipMultiFile.new("", {:account_id => "jetshop"}).send :shell_command, "file.gz"
+      assert_match /grep -F aid=jetshop/, command
+    end
+    should "add filter for type" do
+      command = GzipMultiFile.new("", {:observation_type => "view_item"}).send :shell_command, "file.gz"
+      assert_match /grep -F o=view_item/, command
+    end
+  end
+
+end

data/test/stream_wrapper/observation_filter_test.rb

@@ -0,0 +1,171 @@
+require 'test_helper'
+require 'stream_wrapper'
+include StreamWrapper
+
+class StreamWrapper::ObservationFilterTest < Test::Unit::TestCase 
+
+  context "An ObservationFilter (regardless of filter options)" do
+    setup do
+      @of = ObservationFilter.new "line 1\nline 2", {}
+    end
+    
+    should "read from from a stream with #gets" do
+      assert_equal("line 1\n", @of.gets)
+      assert_equal("line 2", @of.gets)
+    end
+    
+    should "return nil from #gets after the last line" do
+      @of.gets
+      @of.gets
+      assert_equal(nil, @of.gets)
+    end
+    
+    should "know if the stream is eof" do
+      assert_equal(false, @of.eof?)
+      @of.gets
+      assert_equal(false, @of.eof?)
+      @of.gets
+      assert_equal(true, @of.eof?)
+    end
+  end
+
+  context "An ObservationFilter" do
+    should "be able to handle empty lines" do
+      assert_nothing_raised{ObservationFilter.new("\n", {}).to_a}
+    end
+
+    should "filter case insensitive" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new(file, {:account_id => "xRoAdS"})
+        assert_not_nil(of.gets);
+      end
+    end
+
+    should "match escaped strings" do
+      file = "aid=Trygg%20Hansa\n"
+      of = ObservationFilter.new(file, {:account_id => "Trygg Hansa"})
+      assert_not_nil(of.gets);
+    end
+
+    should "only return rows with matching account id" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new(file, {:account_id => "xroads"})
+        assert_not_nil(of.gets);
+      end
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new(file, {:account_id => "apa"})
+        assert_nil(of.gets);
+        assert(of.eof?)
+      end
+    end
+
+    should "only return rows with matching shop id" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new(file, {:shop_id => "Fortum"})
+        assert_not_nil(of.gets);
+      end
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new(file, {:account_id => "apa"})
+        assert_nil(of.gets);
+        assert(of.eof?)
+      end
+    end
+    
+    should "only return rows with matching account id AND shop id" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:account_id => "xroads", :shop_id => "fortum"}
+        assert(!(of.gets =~ /fortum/));
+      end
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:account_id => "apa", :shop_id => "fortum"}
+        assert_nil(of.gets);
+      end
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:account_id => "xroads", :shop_id => "apa"}
+        assert_nil(of.gets);
+      end
+    end
+    
+    should "skip shops" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:skip_shop_id => "fortum"}
+        assert(!(of.gets =~ /fortum/i));
+        assert(!(of.gets =~ /fortum/i));
+        assert_nil(of.gets);
+      end
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:skip_shop_id => "trygg\shansa"}
+        assert(!(of.gets =~ /trygg\shansa/i));
+        assert(!(of.gets =~ /trygg\shansa/i));
+        assert(!(of.gets =~ /trygg\shansa/i));
+        assert(!(of.gets =~ /trygg\shansa/i));
+        assert(!(of.gets =~ /trygg\shansa/i));
+        assert_nil(of.gets);
+      end
+    end
+
+    should "only return rows with matching observation type" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:observation_type => "view_item"}
+        assert(of.gets =~ /o=view_item/);
+      end
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:observation_type => "pick_item"}
+        assert(of.gets =~ /o=pick_item/);
+      end
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:observation_type => "apa"}
+        assert_nil(of.gets);
+      end
+    end
+    
+    should "ignore inhouse and debug rows" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:skip_debug_observations => true}
+        assert_not_nil(of.gets);
+        assert_not_nil(of.gets);
+        assert_nil(of.gets);
+      end
+    end
+
+    if RUBY_VERSION > "1.9"
+      should "ignore invalide byte sequence" do
+        File.open(fixture_file('log_invalid_mixed_encoding.log'), :encoding => "utf-8") do |file|
+          of = ObservationFilter.new file, {:skip_debug_observations => true}
+          assert_nothing_raised{of.gets}
+        end
+      end
+    end
+    
+    should "iterate over observations" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        of = ObservationFilter.new file, {:observation_type => "view_item"}
+        of.each do |o|
+          assert(o.kind_of?(Observation))
+        end
+        assert_equal(4, of.observation_count)
+      end
+    end
+
+    should "filter valid observations" do
+      file = File.open(fixture_file("observation_filter.log")).readlines.join
+      of = ObservationFilter.new file, {:valid => true}
+      assert_equal(5, of.count)
+      file += '81.233.91.34 - - [16/Feb/2009:08:36:36 +0100] "GET /log.gif?Intesahimlavaliddirekt& HTTP/1.1" 200 35 "http://www.xroads.se/trygghansa/Products/Product.asp?itemId=6971" "Mozilla/5.0 (Windows; U; Windows NT 5.1; sv-SE; rv:1.9.0.6) Gecko/2009011913 Firefox/3.0.6"\n'
+      of = ObservationFilter.new file, {:valid => true}
+      assert_equal(5, of.count)
+      of = ObservationFilter.new file, {:skip_valid => true}
+      assert_equal(1, of.count)
+    end
+    
+    should "not accept false values for any options" do
+      File.open(fixture_file("observation_filter.log")) do |file|
+        assert_raise RuntimeError do
+          of = ObservationFilter.new file, {:valid => false}
+        end
+      end
+    end    
+
+  end
+
+end