appstats 0.0.13 → 0.0.14

data/spec/context_spec.rb

@@ -0,0 +1,123 @@
+require 'spec_helper'
+
+module Appstats
+  describe Context do
+
+    before(:each) do
+      @context = Appstats::Context.new
+      @time = Time.parse('2010-01-02 10:20:30')
+    end
+
+    describe "#initialize" do
+
+      it "should set context_key to nil" do
+        @context.context_key.should == nil
+      end
+
+      it "should set context_value to nil" do
+        @context.context_value.should == nil
+      end
+      
+      it "should set context_int to nil" do
+        @context.context_int.should == nil
+      end
+
+      it "should set context_float to nil" do
+        @context.context_float.should == nil
+      end
+
+      it "should set on constructor" do
+        context = Appstats::Context.new(:context_key => 'a', :context_value => "b", :context_int => 1, :context_float => 1.3)
+        context.context_key.should == 'a'
+        context.context_value.should == 'b'
+        context.context_int.should == nil
+        context.context_float.should == nil
+      end
+    
+    end
+
+    describe "#context_int" do
+    
+      it "should be nil if not an int" do
+        @context.context_int.should == nil
+        @context.context_value = "1"
+        @context.context_int.should == 1
+
+        @context.context_value = "2"
+        @context.context_int.should == 2
+       
+        @context.context_value = "c"
+        @context.context_int.should == nil
+
+        @context.context_value = nil
+        @context.context_int.should == nil
+
+      end
+    
+    end
+
+    describe "#context_float" do
+
+      it "should be nil if not an int" do
+        @context.context_float.should == nil
+        @context.context_value = "1"
+        @context.context_float.should == 1.0
+
+        @context.context_value = "2.1"
+        @context.context_float.should == 2.1
+       
+        @context.context_value = "c"
+        @context.context_float.should == nil
+
+        @context.context_value = nil
+        @context.context_float.should == nil
+
+      end      
+    end
+    
+    describe "#entry" do
+      
+      before(:each) do
+        @entry = Appstats::Entry.new(:action => "a")
+        @entry.save.should == true
+      end
+      
+      it "should have an entry" do
+        @context.entry.should == nil
+        @context.entry = @entry
+        @context.save.should == true
+        @context.reload
+        @context.entry.should == @entry
+        
+        @context = Context.last
+        @context.entry.should == @entry
+      end
+      
+    end
+
+    describe "#to_s" do
+    
+      before(:each) do
+        @context = Appstats::Context.new
+      end
+    
+      it "should return no context if no key" do
+        @context.to_s.should == 'No Context'
+        @context.context_key = ''
+        @context.to_s.should == 'No Context'
+      end
+      
+      it "should return the context_key name if no date" do
+        @context.context_key = "Blah"
+        @context.to_s.should == 'Blah[]'
+      end
+      
+      it "should return context_key and context_value if available" do
+        @context.context_key = "More Blah"
+        @context.context_value = "true"
+        @context.to_s.should == "More Blah[true]"
+      end
+      
+    end
+  end
+end

data/spec/entry_spec.rb

@@ -3,32 +3,62 @@ require 'spec_helper'
 module Appstats
   describe Entry do
 
+    before(:each) do
+      @time = Time.parse('2010-01-02 10:20:30')
+      @entry = Appstats::Entry.new
+      Time.stub!(:now).and_return(@time)
+    end
+
     describe "#initialize" do
 
-      before(:each) do
-        @entry = Appstats::Entry.new
-      end
-    
-      it "should set entry_type to nil" do
-        @entry.entry_type.should == nil
+      it "should set action to nil" do
+        @entry.action.should == nil
       end
 
-      it "should set name to nil" do
-        @entry.name.should == nil
+      it "should set occurred_at to nil" do
+        @entry.occurred_at.should == nil
       end
-    
-      it "should set description to nil" do
-        @entry.description.should == nil
+      
+      it "should set raw_entry to nil" do
+        @entry.raw_entry.should == nil
       end
     
       it "should set on constructor" do
-        entry = Appstats::Entry.new(:entry_type => 'a', :name => 'b', :description => 'c')
-        entry.entry_type.should == 'a'
-        entry.name.should == 'b'
-
+        entry = Appstats::Entry.new(:action => 'a', :occurred_at => @time, :raw_entry => 'b')
+        entry.action.should == 'a'
+        entry.occurred_at.should == @time
+        entry.raw_entry.should == 'b'
       end
     
     end
+    
+    describe "#contexts" do
+      
+      it "should have none by default" do
+        @entry.contexts.size.should == 0
+      end
+      
+      it "should be able add contexts" do
+        context = Appstats::Context.new(:context_key => 'a', :context_value => 'one')
+        context.save.should == true
+        @entry.contexts<< context
+        @entry.save.should == true
+        @entry.reload
+        @entry.contexts.size.should == 1
+        @entry.contexts[0].should == context
+      end
+      
+      it "should alphabetize them" do
+        zzz = Appstats::Context.create(:context_key => 'zzz', :context_value => 'one')
+        aaa = Appstats::Context.create(:context_key => 'aaa', :context_value => 'one')
+        @entry.contexts<< zzz
+        @entry.contexts<< aaa
+        @entry.save.should == true
+        @entry.reload
+        @entry.contexts.should == [aaa,zzz]
+      end
+      
+    end
 
     describe "#to_s" do
     
@@ -36,9 +66,124 @@ module Appstats
         @entry = Appstats::Entry.new
       end
     
-      it "should run the test" do
-        @entry.to_s.should == 'Entry [type],[name],[description]'
+      it "should return no entry if no action" do
+        @entry.to_s.should == 'No Entry'
+        @entry.action = ''
+        @entry.to_s.should == 'No Entry'
+      end
+      
+      it "should return the action name if no date" do
+        @entry.action = "Blah"
+        @entry.to_s.should == 'Blah'
+      end
+      
+      it "should return action and date if available" do
+        @entry.action = "More Blah"
+        @entry.occurred_at = @time
+        @entry.to_s.should == "More Blah at 2010-01-02 10:20:30"
       end
+      
     end
+    
+    describe "#load_from_logger_file" do
+      
+      before(:each) do
+        @before_count = Entry.count
+        Appstats::Logger.reset
+        Time.stub!(:now).and_return(Time.parse('2010-09-21 23:15:20'))
+      end
+
+      after(:each) do
+        File.delete(Appstats::Logger.filename) if File.exists?(Appstats::Logger.filename)
+      end
+
+      it "should handle nil" do
+        Entry.load_from_logger_file(nil).should == false
+        Entry.count.should == @before_count
+        Entry.count.should == @before_count
+      end
+      
+      it "should handle unknown files" do
+        File.exists?("should_not_exist.txt").should == false
+        Entry.load_from_logger_file("should_not_exist.txt").should == false
+        Entry.count.should == @before_count
+      end
+      
+      it "should handle appstat files" do
+        Appstats::Logger.entry("test_action")
+        Appstats::Logger.entry("another_test_action")
+        Entry.load_from_logger_file(Appstats::Logger.filename).should == true
+        Entry.count.should == @before_count + 2
+        Entry.last.action.should == "another_test_action"
+      end
+      
+    end
+    
+    
+    describe "#load_from_logger_entry" do
+      
+      before(:each) do
+        @before_count = Entry.count
+      end
+      
+      it "should handle nil" do
+        Entry.load_from_logger_entry(nil).should == false
+        Entry.count.should == @before_count
+
+        Entry.load_from_logger_entry("").should == false
+        Entry.count.should == @before_count
+      end
+      
+      it "should create an unknown for unknown entries" do
+        entry = Entry.load_from_logger_entry("blah")
+        Entry.count.should == @before_count + 1
+        entry.action.should == "UNKNOWN_ACTION"
+        entry.raw_entry.should == "blah"
+        entry.occurred_at.should == nil
+      end
+      
+      it "should understand an entry without contexts" do
+        entry = Entry.load_from_logger_entry("0.0.14 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
+        Entry.count.should == @before_count + 1
+        entry.action.should == "address_search"
+        entry.raw_entry.should == "0.0.14 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
+        entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
+      end
+      
+      it "should understand contexts" do
+        entry = Entry.load_from_logger_entry("0.0.14 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
+        Entry.count.should == @before_count + 1
+        entry.action.should == "address_filter"
+        entry.raw_entry.should == "0.0.14 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
+        entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
+        entry.contexts.size.should == 2
+        entry.contexts[0].context_key = "app_name"
+        entry.contexts[0].context_value = "Market"
+        entry.contexts[1].context_key = "server"
+        entry.contexts[1].context_value = "Live"
+      end
+      
+    end
+   
+    describe "#log_collector" do
+      
+      before(:each) do
+        @log_collector = Appstats::LogCollector.new(:host => "a")
+        @log_collector.save.should == true
+      end
+      
+      it "should have a log_collector" do
+        @entry.log_collector.should == nil
+        @entry.log_collector = @log_collector
+        @entry.save.should == true
+        @entry.reload
+        @entry.log_collector.should == @log_collector
+        
+        @entry = Entry.last
+        @entry.log_collector.should == @log_collector
+      end
+      
+    end    
+
   end
 end

data/spec/log_collector_spec.rb

@@ -0,0 +1,284 @@
+require 'spec_helper'
+
+module Appstats
+  describe LogCollector do
+
+    before(:each) do
+      LogCollector.delete_all
+      @log_collector = Appstats::LogCollector.new
+      @login = { :host => "myhost.localnet", :user => "deployer", :password => "pass" }
+      @login2 = { :host => "yourhost.localnet", :user => "deployer", :password => "ssap" }
+      @logins = [@login2, @login]
+    end
+    
+    after(:each) do
+      LogCollector.all.each do |log_collector|
+        File.delete(log_collector.local_filename) if File.exists?(log_collector.local_filename)
+      end
+    end
+    
+    def simple_path(local_path_to_filename)
+      File.expand_path("#{File.dirname(__FILE__)}/#{local_path_to_filename}")
+    end
+
+    describe "#initialize" do
+
+      it "should set host to nil" do
+        @log_collector.host.should == nil
+      end
+
+      it "should set filename to nil" do
+        @log_collector.filename.should == nil
+      end
+      
+      it "should set status to unprocessed" do
+        @log_collector.status.should == nil
+      end
+    
+      it "should set on constructor" do
+        log_collector = Appstats::LogCollector.new(:host => 'a', :filename => 'b', :status => 'c')
+        log_collector.host.should == 'a'
+        log_collector.filename.should == 'b'
+        log_collector.status.should == 'c'
+      end
+    
+    end
+    
+    describe "#find_remote_files" do
+      
+      before(:each) do
+        @before_count = LogCollector.count
+      end
+      
+      it "should log all transactions" do
+        ssh = mock(Net::SSH)
+        Net::SSH.should_receive(:start).with("myhost.localnet","deployer",{ :password => "pass"}).and_yield(ssh)
+        ssh.should_receive(:exec!).with("cd /my/path/log && ls | grep mystats").and_return("mystats1\nmystats2")
+        
+        Appstats.should_receive(:log).with(:info, "Looking for logs in [deployer@myhost.localnet:/my/path/log] labelled [mystats]")
+        Appstats.should_receive(:log).with(:info, "About to analyze 2 file(s).")
+        Appstats.should_receive(:log).with(:info, "  - deployer@myhost.localnet:/my/path/log/mystats1")
+        Appstats.should_receive(:log).with(:info, "  - deployer@myhost.localnet:/my/path/log/mystats2")
+        Appstats.should_receive(:log).with(:info, "Loaded 2 file(s).")
+        LogCollector.find_remote_files(@login,"/my/path/log","mystats").should == 2
+      end
+      
+      it "should talk to remote server" do
+        ssh = mock(Net::SSH)
+        Net::SSH.should_receive(:start).with("myhost.localnet","deployer",{ :password => "pass"}).and_yield(ssh)
+        ssh.should_receive(:exec!).with("cd /my/path/log && ls | grep mystats").and_return("mystats1\nmystats2")
+        
+        LogCollector.find_remote_files(@login,"/my/path/log","mystats").should == 2
+        LogCollector.count.should == @before_count + 2
+
+        log_collector = LogCollector.last
+        log_collector.host.should == "myhost.localnet"
+        log_collector.filename.should == "/my/path/log/mystats2"
+        log_collector.status.should == "unprocessed"
+      end
+      
+      it "should fail silently for bad connections" do
+        Net::SSH.should_receive(:start).with("myhost.localnet","deployer",{ :password => "pass"}).and_raise("Some bad message")
+        Appstats.should_receive(:log).with(:info, "Looking for logs in [deployer@myhost.localnet:/my/path/log] labelled [mystats]")
+        Appstats.should_receive(:log).with(:error,"Something bad occurred during Appstats::LogCollector#find_remote_files")
+        Appstats.should_receive(:log).with(:error,"Some bad message")
+        LogCollector.find_remote_files(@login,"/my/path/log","mystats").should == 0
+      end
+      
+    end
+    
+    describe "#load_remote_files" do
+      
+      before(:each) do
+        @before_count = LogCollector.count
+      end
+
+      it "should log if nothing to load" do
+        log = LogCollector.create(:status => "not_unprocessed")
+        Appstats.should_receive(:log).with(:info,"No remote logs to load.")
+        LogCollector.load_remote_files(@login,"/my/path/log",[]).should == 0
+      end
+
+      
+      it "should log the files loaded" do
+        LogCollector.load_remote_files(@login,"/my/path/log",["app2"]).should == 1
+        
+        Appstats.should_receive(:log).with(:info, "About to analyze 3 file(s).")
+        Appstats.should_receive(:log).with(:info, "  - deployer@myhost.localnet:/my/path/log/app1")
+        Appstats.should_receive(:log).with(:info, "  - ALREADY LOADED deployer@myhost.localnet:/my/path/log/app2")
+        Appstats.should_receive(:log).with(:info, "  - deployer@myhost.localnet:/my/path/log/app3")
+        Appstats.should_receive(:log).with(:info, "Loaded 2 file(s).")
+        LogCollector.load_remote_files(@login,"/my/path/log",["app1","app2","app3"]).should == 2
+      end
+      
+      it "should create an unprocessed record per file" do
+        LogCollector.load_remote_files(@login,"/my/path/log",["app1","app2","app3"]).should == 3
+        LogCollector.count.should == @before_count + 3
+        log_collector = LogCollector.last
+        
+        log_collector.host.should == "myhost.localnet"
+        log_collector.filename.should == "/my/path/log/app3"
+        log_collector.status.should == "unprocessed"
+      end
+      
+      it "should ignore the same file" do
+        LogCollector.load_remote_files(@login,"/my/path/log",["app1","app3"]).should == 2
+        LogCollector.load_remote_files(@login,"/my/path/log",["app1","app2","app3"]).should == 1
+        LogCollector.count.should == @before_count + 3
+        log_collector = LogCollector.last
+        
+        log_collector.host.should == "myhost.localnet"
+        log_collector.filename.should == "/my/path/log/app2"
+        log_collector.status.should == "unprocessed"
+      end
+      
+    end
+    
+    describe "#local_filename" do
+      
+      it "should return a standardized name with the log collector id" do
+        log = LogCollector.create
+        log.local_filename.should == simple_path("../log/appstats_remote_log_#{log.id}.log")
+      end
+      
+    end
+    
+    describe "#download_remote_files" do
+
+      before(:each) do
+        @before_count = LogCollector.count
+      end
+      
+      it "should only process unprocessed files" do
+        log = LogCollector.create(:status => "not_unprocessed")
+        Appstats.should_receive(:log).with(:info,"No remote logs to download.")
+        LogCollector.download_remote_files(@logins).should == 0
+      end
+      
+      it "should log exceptions" do
+        LogCollector.load_remote_files(@login,"/my/path/log",["app1"]).should == 1
+        log1 = LogCollector.find_by_filename("/my/path/log/app1")
+
+        scp = mock(Net::SCP)
+        Net::SCP.should_receive(:start).with("myhost.localnet","deployer",{ :password => "pass"}).and_yield(scp)
+        scp.should_receive(:download!).with("/my/path/log/app1",simple_path("../log/appstats_remote_log_#{log1.id}.log")).and_raise("Something bad happened again")
+
+        Appstats.should_receive(:log).with(:info,"About to download 1 file(s).")
+        Appstats.should_receive(:log).with(:error,"Something bad occurred during Appstats::LogCollector#download_remote_files")
+        Appstats.should_receive(:log).with(:error,"Something bad happened again")
+        Appstats.should_receive(:log).with(:error, "File #{simple_path("../log/appstats_remote_log_#{log1.id}.log")} did not download.")
+        Appstats.should_receive(:log).with(:info,"Downloaded 0 file(s).")
+        LogCollector.download_remote_files(@logins).should == 0
+      end
+      
+      it "should ignore if file not downloaded" do
+        LogCollector.load_remote_files(@login,"/my/path/log",["app1"]).should == 1
+        log1 = LogCollector.find_by_filename("/my/path/log/app1")
+
+        scp = mock(Net::SCP)
+        Net::SCP.should_receive(:start).with("myhost.localnet","deployer",{ :password => "pass"}).and_yield(scp)
+        scp.should_receive(:download!).with("/my/path/log/app1",simple_path("../log/appstats_remote_log_#{log1.id}.log"))
+        
+        Appstats.should_receive(:log).with(:info,"About to download 1 file(s).")        
+        Appstats.should_receive(:log).with(:error, "File #{simple_path("../log/appstats_remote_log_#{log1.id}.log")} did not download.")
+        Appstats.should_receive(:log).with(:info,"Downloaded 0 file(s).")
+        LogCollector.download_remote_files(@logins).should == 0
+        log1.reload
+        log1.status.should == "failed_download"
+      end
+      
+      it "should copy the file based on the log_collector id" do
+        LogCollector.load_remote_files(@login,"/my/path/log",["app1","app2"]).should == 2
+        
+        log1 = LogCollector.find_by_filename("/my/path/log/app1")
+        log2 = LogCollector.find_by_filename("/my/path/log/app2")
+        
+        File.open(log1.local_filename, 'w') {|f| f.write("testfile - delete") }
+        File.open(log2.local_filename, 'w') {|f| f.write("testfile - delete") }
+
+
+        scp = mock(Net::SCP)
+        1.upto(3) do
+          Net::SCP.should_receive(:start).with("myhost.localnet","deployer",{ :password => "pass"}).and_yield(scp)
+        end
+        scp.should_receive(:download!).with("/my/path/log/app1",simple_path("../log/appstats_remote_log_#{log1.id}.log"))
+        scp.should_receive(:download!).with("/my/path/log/app2",simple_path("../log/appstats_remote_log_#{log2.id}.log"))
+        
+        LogCollector.download_remote_files(@logins).should == 2
+        
+        log1.reload and log2.reload
+        log1.status.should == "downloaded"
+        log2.status.should == "downloaded"
+        
+        LogCollector.load_remote_files(@login,"/my/path/log",["app3"]).should == 1
+        log3 = LogCollector.find_by_filename("/my/path/log/app3")
+        File.open(log3.local_filename, 'w') {|f| f.write("testfile - delete") }
+        
+        localfile = simple_path("../log/appstats_remote_log_#{log3.id}.log")
+        scp.should_receive(:download!).with("/my/path/log/app3",localfile)
+
+        Appstats.should_receive(:log).with(:info,"About to download 1 file(s).")
+        Appstats.should_receive(:log).with(:info, "  - deployer@myhost.localnet:/my/path/log/app3 > #{localfile}")
+        Appstats.should_receive(:log).with(:info,"Downloaded 1 file(s).")
+        LogCollector.download_remote_files(@logins).should == 1
+        log3.reload and log3.reload
+        log3.status.should == "downloaded"
+      end
+      
+    end
+    
+    describe "#process_local_files" do
+      
+      before(:each) do
+        @entry_count = Entry.count
+      end
+      
+      it "should only process downloaded files" do
+        log = LogCollector.create(:status => "not_downloaded")
+        Appstats.should_receive(:log).with(:info,"No local logs to process.")
+        LogCollector.process_local_files.should == 0
+      end
+
+      it "should process downloaded files into entries" do
+        LogCollector.load_remote_files(@login,"/my/path/log",["appstats1"]).should == 1
+        log3 = LogCollector.find_by_filename("/my/path/log/appstats1")
+        log3.status = "downloaded" and log3.save.should == true
+        File.open(log3.local_filename, 'w') {|f| f.write(Appstats::Logger.entry_to_s("test_action1") + "\n" + Appstats::Logger.entry_to_s("test_action2")) }
+
+        Appstats.should_receive(:log).with(:info,"About to process 1 file(s).")
+        Appstats.should_receive(:log).with(:info,"  - 2 entr(ies) in #{log3.local_filename}.")
+        Appstats.should_receive(:log).with(:info,"Processed 1 file(s) with 2 entr(ies).")
+        LogCollector.process_local_files.should == 1
+        
+        log3.reload
+        log3.status.should == "processed"
+        Entry.count.should == @entry_count + 2
+        entry = Entry.last
+        entry.log_collector.should == log3
+        entry.action.should == "test_action2"
+      end
+      
+      it "should deal with exceptions" do
+        LogCollector.load_remote_files(@login,"/my/path/log",["appstats1"]).should == 1
+        log3 = LogCollector.find_by_filename("/my/path/log/appstats1")
+        log3.status = "downloaded" and log3.save.should == true
+        File.open(log3.local_filename, 'w') {|f| f.write(Appstats::Logger.entry_to_s("test_action1") + "\n" + Appstats::Logger.entry_to_s("test_action2")) }
+
+        File.stub!(:open).and_raise("bad error")
+        
+        Appstats.should_receive(:log).with(:info,"About to process 1 file(s).")
+        Appstats.should_receive(:log).with(:error,"Something bad occurred during Appstats::LogCollector#process_local_files")
+        Appstats.should_receive(:log).with(:error,"bad error")
+        Appstats.should_receive(:log).with(:info,"Processed 0 file(s) with 0 entr(ies).")
+        LogCollector.process_local_files.should == 0
+        
+        log3.reload
+        log3.status.should == "downloaded"
+        Entry.count.should == @entry_count
+      end      
+      
+    end
+    
+    
+  end
+end