appstats 0.7.0 → 0.8.0

data/lib/appstats/version.rb

@@ -1,3 +1,3 @@
 module Appstats
-  VERSION = "0.7.0"
+  VERSION = "0.8.0"
 end

data/spec/entry_spec.rb

@@ -188,18 +188,18 @@ module Appstats
       end
       
       it "should understand an entry without contexts" do
-        entry = Entry.load_from_logger_entry("0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
+        entry = Entry.load_from_logger_entry("0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
         Entry.count.should == @before_count + 1
         entry.action.should == "address_search"
-        entry.raw_entry.should == "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
+        entry.raw_entry.should == "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
         entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
       end
       
       it "should understand contexts" do
-        entry = Entry.load_from_logger_entry("0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
+        entry = Entry.load_from_logger_entry("0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
         Entry.count.should == @before_count + 1
         entry.action.should == "address_filter"
-        entry.raw_entry.should == "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
+        entry.raw_entry.should == "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
         entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
         entry.contexts.size.should == 2
         entry.contexts[0].context_key = "app_name"

data/spec/host_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+module Appstats
+  describe Host do
+
+    before(:each) do
+      Appstats::LogCollector.delete_all
+      Appstats::Host.delete_all
+      @host = Appstats::Host.new
+    end
+    
+    describe "#initialize" do
+
+      it "should set name to nil" do
+        @host.name.should == nil
+      end
+
+      it "should set status to nil" do
+        @host.status.should == nil
+      end
+      
+      it "should set on constructor" do
+        host = Appstats::Host.new(:name => 'a', :status => 'c')
+        host.name.should == 'a'
+        host.status.should == 'c'
+      end
+    
+    end
+    
+    describe "#update_hosts" do
+      
+      it "should do nothing if no events" do
+        Appstats::Host.update_hosts.should == 0
+        Appstats::Host.count.should == 0
+      end
+      
+      it "should add entry host names" do
+        Appstats::LogCollector.create(:host => 'a')
+        Appstats::Host.update_hosts.should == 1
+        Appstats::Host.count.should == 1
+        
+        host = Appstats::Host.last
+        host.name = 'a'
+        host.status = 'derived'
+      end
+      
+    end
+    
+
+  end
+end

data/spec/logger_spec.rb

@@ -115,7 +115,7 @@ module Appstats
       
       it "should accept numbers" do
         Appstats::Logger.entry(5, :blah => 6)   
-        Appstats::Logger.raw_read.should == ["0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=5 : blah=6"]
+        Appstats::Logger.raw_read.should == ["0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=5 : blah=6"]
       end
 
     end
@@ -124,7 +124,7 @@ module Appstats
 
       it "should look similar to regular entry" do
         Appstats::Logger.exception_entry(RuntimeError.new("blah"),:on => "login")   
-        Appstats::Logger.raw_read.should == ["0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=appstats-exception : error=blah : on=login"]
+        Appstats::Logger.raw_read.should == ["0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=appstats-exception : error=blah : on=login"]
       end
       
     end
@@ -141,29 +141,29 @@ module Appstats
     
        it "should handle a statistics entry" do
          expected = { :action => "address_search", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
+         actual = Appstats::Logger.entry_to_hash("0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
          actual.should == expected
        end
        
        it "should handle contexts" do
          expected = { :action => "address_filter", :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => 'Market' }
-         actual = Appstats::Logger.entry_to_hash("0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
          actual.should == expected
        end
     
        it "should handle actions with the delimiter (and change the delimiter)" do
          expected = { :action => "address:=search-n", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.7.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n")
+         actual = Appstats::Logger.entry_to_hash("0.8.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n")
          actual.should == expected
     
          expected = { :action => "address::search==--n", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.7.0 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n")
+         actual = Appstats::Logger.entry_to_hash("0.8.0 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n")
          actual.should == expected
        end
     
        it "should handle contexts with the delimiter (and change the delimiter)" do
          expected = { :action => "address", :timestamp => "2010-09-21 23:15:20", :server => "market:eval=-n" }
-         actual = Appstats::Logger.entry_to_hash("0.7.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n")
+         actual = Appstats::Logger.entry_to_hash("0.8.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n")
          actual.should == expected
        end
        
@@ -172,66 +172,66 @@ module Appstats
     describe "#entry_to_s" do
       
       it "should handle a statistics entry" do
-        expected = "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
+        expected = "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
         actual = Appstats::Logger.entry_to_s("address_search")
         actual.should == expected
       end
       
       it "should handle numbers" do
-        expected = "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=1 : note=2.2"
+        expected = "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=1 : note=2.2"
         actual = Appstats::Logger.entry_to_s(1,:note => 2.2)
         actual.should == expected
       end
       
       it "should handle default contexts" do
         Appstats::Logger.default_contexts[:app_name] = "market"
-        expected = "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : app_name=market"
+        expected = "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : app_name=market"
         actual = Appstats::Logger.entry_to_s("address_search")
         actual.should == expected
       end
       
       it "should handle contexts (and sort them by symbol)" do
-        expected = "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
+        expected = "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
         actual = Appstats::Logger.entry_to_s("address_filter", { :server => "Live", :app_name => 'Market' })
         actual.should == expected
       end
 
       it "should handle actions with the delimiter (and change the delimiter)" do
-        expected = "0.7.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n"
+        expected = "0.8.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n"
         actual = Appstats::Logger.entry_to_s("address:=search-n")
         actual.should == expected
 
-        expected = "0.7.0 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n"
+        expected = "0.8.0 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n"
         actual = Appstats::Logger.entry_to_s("address::search==--n")
         actual.should == expected
       end
 
       it "should handle contexts with the delimiter (and change the delimiter)" do
-        expected = "0.7.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n"
+        expected = "0.8.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n"
         actual = Appstats::Logger.entry_to_s("address", :server => 'market:eval=-n')
         actual.should == expected
       end
 
       it "should ignore spaces" do
-        expected = "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address search"
+        expected = "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address search"
         actual = Appstats::Logger.entry_to_s("address search")
         actual.should == expected
       end
       
       it "should convert newlines in action" do
-        expected = "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_-nsearch"
+        expected = "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_-nsearch"
         actual = Appstats::Logger.entry_to_s("address_\nsearch")
         actual.should == expected
       end
 
       it "should convert newlines in context" do
-        expected = "0.7.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : blah=some-nlong-nstatement"
+        expected = "0.8.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : blah=some-nlong-nstatement"
         actual = Appstats::Logger.entry_to_s("address_search",:blah => "some\nlong\nstatement")
         actual.should == expected
       end
       
       it "should convert newlines based on the delimiter" do
-        expected = "0.7.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=--nsearch-n"
+        expected = "0.8.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=--nsearch-n"
         actual = Appstats::Logger.entry_to_s("address:=\nsearch-n")
         actual.should == expected
       end

data/spec/parser_spec.rb

@@ -0,0 +1,329 @@
+require 'spec_helper'
+
+module Appstats
+  describe Parser do
+
+    before(:each) do
+      @parser = Parser.new
+    end
+     
+    describe("#initialize") do
+      
+      it "should set rules to nil" do
+        @parser.raw_rules.should == nil
+        @parser.raw_tokenize.should == nil
+        @parser.repeating.should == false
+        @parser.tokenize_regex == nil
+        @parser.rules.should == []
+        @parser.tokenize.should == []
+        @parser.constants.should == []
+      end
+      
+      it "should set rules from constructor" do
+        parser = Parser.new(:rules => ":name or :bust", :tokenize => "a bb c", :repeating => true)
+        parser.raw_rules.should == ":name or :bust"
+        parser.raw_tokenize.should == "a bb c"
+        parser.repeating.should == true
+        parser.rules.should == [ { :rule => :name, :stop => :constant }, "OR", { :rule => :bust, :stop => :end} ]
+        parser.tokenize.should == ["\\s+A","\\s+BB","\\s+C"]
+        parser.tokenize_no_spaces.should == ["A","BB","C"]
+        parser.constants.should == ["OR"]
+      end
+    
+      it "should espace tokens as required" do
+        parser = Parser.new(:tokenize => "( ) abc |")
+        parser.tokenize.should == ['\(','\)','\s+ABC','\|']
+        parser.tokenize_regex.should == '\(|\)|\s+ABC|\|'
+        parser.tokenize_no_spaces.should == ['\(','\)','ABC','\|']
+        parser.tokenize_regex_no_spaces.should == '\(|\)|ABC|\|'
+      end
+    
+    end
+     
+     
+    describe "#rules" do
+    
+      it "should end on constant if tokens present" do
+        Parser.new(:rules => ":name", :tokenize => ")").rules.should == [ { :rule => :name, :stop => :end } ]
+      end
+    
+      
+      it "should handle one variable" do
+        Parser.new(:rules => ":name").rules.should == [ { :rule => :name, :stop => :end } ]
+      end
+    
+      it "should handle many variables" do
+        Parser.new(:rules => ":name :date").rules.should == [ { :rule => :name, :stop => :space }, { :rule => :date, :stop => :end } ]
+      end
+    
+      it "should deal with colons" do
+        Parser.new(:rules => ":name : :date").rules.should == [ { :rule => :name, :stop => :constant }, ":", { :rule => :date, :stop => :end } ]
+      end
+    
+      it "should deal with constant" do
+        Parser.new(:rules => "blah").rules.should == ["BLAH"]
+      end
+    
+      it "should deal with constant and variables" do
+        Parser.new(:rules => ":name blah :date").rules.should == [ { :rule => :name, :stop => :constant }, "BLAH", { :rule => :date, :stop => :end } ]
+      end
+    
+      it "should deal with multiple constants and variables" do
+        Parser.new(:rules => ":name blah more blah :date").rules.should == [ { :rule => :name, :stop => :constant }, "BLAH", "MORE", "BLAH", { :rule => :date, :stop => :end } ]
+      end
+      
+    end
+     
+    describe "#constants" do
+      
+      it "should be empty if only variables" do
+        Parser.new(:rules => ":name :blah").constants.should == [ ]
+      end
+    
+      it "should track all constants" do
+        Parser.new(:rules => ":name : :date").constants.should == [ ":" ]
+      end
+    
+      it "should upper case constants" do
+        Parser.new(:rules => "blah").constants.should == ["BLAH"]
+      end
+    
+      it "should deal with multiple constants" do
+        Parser.new(:rules => ":name = :blah and :moreblah").constants.should == ["=","AND"]
+      end
+    
+    end
+     
+    describe "#parse_constant" do
+    
+      it "should handle nil" do
+        Parser.parse_constant(nil,nil).should == [nil,nil]
+        Parser.parse_constant("",nil).should == [nil,nil]
+      end
+    
+      it "should find the constant" do
+        Parser.parse_constant("= blah blah more blah ","=").should == ["=","blah blah more blah"]
+      end
+    
+      it "should find the constants with multiple characters" do
+        Parser.parse_constant("hey blah blah more blah ","hey").should == ["hey","blah blah more blah"]
+      end
+    
+      it "should return nil if not found" do
+        Parser.parse_constant("blah blah more blah ","=").should == [nil,"blah blah more blah"]
+      end
+    
+      it "should be case insensitive" do
+        Parser.parse_constant(" blah stuff on more blah stuff ","blah").should == ["blah","stuff on more blah stuff"]
+        Parser.parse_constant(" BLAH stuff on more blah stuff ","blah").should == ["BLAH","stuff on more blah stuff"]
+        Parser.parse_constant(" blah stuff on more blah stuff ","BLAH").should == ["blah","stuff on more blah stuff"]
+      end
+    
+      it "should only find the first instance" do
+        Parser.parse_constant("one == two","==").should == [nil,"one == two"]
+      end
+      
+    end
+     
+    describe "#parse_word" do
+      
+      it "should handle nil" do
+        @parser.parse_word(nil,nil).should == [nil,nil]
+        @parser.parse_word("",nil).should == [nil,nil]
+      end
+    
+      it "should look for global tokens" do
+        parser = Parser.new(:tokenize => "xx")
+        parser.parse_word("blah xx",:end).should == ["blah","xx"]
+        parser.parse_word("blah xx stop","stop").should == ["blah","xx stop"]
+      end
+    
+      it "should handle :end" do
+        @parser.parse_word("blah",:end).should == ["blah",nil]
+        @parser.parse_word("more blah blah blah",:end).should == ["more blah blah blah",nil]
+      end
+    
+      it "should handle :space" do
+        @parser.parse_word("more blah blah blah",:space).should == ["more","blah blah blah"]
+        @parser.parse_word("who dat",:space).should == ["who","dat"]
+      end
+    
+      it "should handle :space when no more spaces" do
+        @parser.parse_word("blah",:space).should == ["blah",nil]
+      end
+    
+      it "should strip front space" do
+        @parser.parse_word(" blah ",:space).should == ["blah",nil]
+        @parser.parse_word(" more blah blah blah ",:space).should == ["more","blah blah blah"]
+        @parser.parse_word(" who dat ",:space).should == ["who","dat"]
+      end
+    
+      it "should on constant" do
+        @parser.parse_word(" blah stuff = more blah stuff ","=").should == ["blah stuff","= more blah stuff"]
+      end
+    
+      it "should be case insensitive" do
+        @parser.parse_word(" blah stuff on more blah stuff ","on").should == ["blah stuff","on more blah stuff"]
+        @parser.parse_word(" blah stuff ON more blah stuff ","on").should == ["blah stuff","ON more blah stuff"]
+        @parser.parse_word(" blah stuff on more blah stuff ","ON").should == ["blah stuff","on more blah stuff"]
+      end
+    
+      it "should grab whole string if not constant not present" do
+        @parser.parse_word(" blah ","x").should == ["blah", nil]
+      end
+    
+      it "should be able to explicitly set if you want an exact match or not" do
+        @parser.parse_word(" aa bbb ","x",false).should == ["aa bbb", nil]
+        @parser.parse_word(" aa bbb ","x",true).should == [nil, "aa bbb"]
+    
+        @parser.parse_word(" aa bbb ","bbb",false).should == ["aa", "bbb"]
+        @parser.parse_word(" aa bbb ","bbb",true).should == ["aa", "bbb"]
+      end
+      
+      it "should remove tokens from the beginning" do
+        parser = Appstats::Parser.new(:tokenize => "xx yy zz")
+        parser.parse_word(" xx yy zz zzaa bbb ",:space).should == ["aa", "bbb"]
+        parser.raw_results.should == [ "xx", "yy", "zz", "zz" ]
+      end
+    
+      it "should remove tokens from the end (:space)" do
+        parser = Parser.new(:tokenize => "xx yy zz")
+        parser.parse_word("  aa xx y z bbb ",:space).should == ["aa", "xx y z bbb"]
+        parser.raw_results.should == [ ]
+      end
+    
+      it "should remove tokens from the end (:end)" do
+        parser = Parser.new(:tokenize => "xx yy zz")
+        parser.parse_word("  aa xx y z bbb ",:end).should == ["aa", "xx y z bbb"]
+        parser.raw_results.should == [ ]
+      end
+    
+      it "should remove tokens from the end (constant)" do
+        parser = Parser.new(:tokenize => "xx yy zz")
+        parser.parse_word("  aa xx y on z bbb ","on").should == ["aa", "xx y on z bbb"]
+        parser.raw_results.should == [ ]
+      end
+    
+      it "should handle constants" do
+        parser = Parser.new(:rules => ":operation :action :date on :server where :contexts")
+        parser.parse_word("today on my.local where stuff","on",true).should == ["today","on my.local where stuff"]  
+      end
+    
+    end
+    
+    describe "#merge_regex_filter" do
+      
+      it "should handle nil" do
+        Parser.merge_regex_filter(nil,nil).should == ""
+        Parser.merge_regex_filter('','').should == ""
+      end
+    
+      it "should handle nil on one side" do
+        Parser.merge_regex_filter('\s',nil).should == '(\s)'
+        Parser.merge_regex_filter(nil,'\s').should == '(\s)'
+        Parser.merge_regex_filter('\s','').should == '(\s)'
+        Parser.merge_regex_filter('','\s').should == '(\s)'
+      end
+    
+      it "should handle both sides" do
+        Parser.merge_regex_filter('\s','a|b').should == '(\s|a|b)'
+      end
+      
+      
+    end
+     
+    describe "#parse" do
+    
+    
+      it "fails if no rules" do
+        @parser.parse("blah").should == false
+      end
+      
+      it "fails on nil" do
+        parser = Parser.new(:rules => ":name")
+        parser.parse(nil).should == false
+      end
+      
+      describe "non repeating" do
+      
+        it "passes on a simple rule" do
+          parser = Parser.new(:rules => ":name")
+          parser.parse("blah").should == true
+          parser.raw_results.should == [ {:name => "blah"} ]
+        end
+          
+        it "passes on several rules" do
+          parser = Parser.new(:rules => ":one :two :three")
+          parser.parse("a  bbb cc").should == true
+          parser.results.should == {:one => "a",  :two => "bbb", :three => "cc" }
+        end
+        
+        it "passes on combination of constants and rules" do
+          parser = Parser.new(:rules => ":operation :action :date on :server where :contexts")
+          parser.parse("# logins today on my.local where stuff").should == true
+          parser.results.should == {:operation => "#",  :action => "logins",  :date => "today" ,  :server => "my.local",  :contexts => "stuff" }
+        end
+    
+        it "passes on combination of constants and rules" do
+          parser = Parser.new(:rules => ":operation :action :date on :server where :contexts")
+          parser.parse("# logins where stuff").should == true
+          parser.results.should == {:operation => "#",  :action => "logins",  :date => nil ,  :server => nil, :contexts => "stuff" }
+        end
+    
+        it "should be able to skip constants" do
+          parser = Parser.new(:rules => ":one aa :two bbb :three")
+          parser.parse("1 bbb 3")
+          parser.results.should == { :one => "1", :two => nil, :three => "3" }
+        end
+    
+        it "should handle missing constants" do
+          parser = Parser.new(:rules => ":one aa :two bbb :three")
+          parser.parse("1").should == true
+          parser.results.should == {:one => "1", :two => nil, :three => nil }
+        end
+    
+        it "should handle missing constants" do
+          parser = Parser.new(:rules => ":one aa :two bbb :three")
+          parser.parse("1 one aa 222 ").should == true
+          parser.results.should == {:one => "1 one",  :two => "222",  :three => nil }
+        end
+        
+      end
+    
+      describe "repeating" do
+        
+        it "should tokenize single repeating entities" do
+          parser = Appstats::Parser.new(:rules => ":context", :repeating => true, :tokenize => "and or || && = <= >= <> !=")
+          parser.parse("a = b").should == true
+          parser.raw_results.should == [ { :context => "a"}, "=", { :context => "b"} ]
+        end
+    
+      end
+    
+      it "should tokenize tokens" do
+        parser = Appstats::Parser.new(:rules => ":context_key = :context_value", :tokenize => "( )")
+        parser.parse("( ( a = b ) )").should == true
+        parser.raw_results.should == [ "(", "(", { :context_key => "a"}, "=", { :context_value => "b"}, ")", ")" ]
+      end
+    
+      it "should tokenize letters only if spaces between them" do
+        parser = Appstats::Parser.new(:rules => ":one :two", :tokenize => "( aa a1 )")
+        parser.parse("(blaa aa   a1   bla1)").should == true
+        parser.raw_results.should == [ "(", { :one => "blaa"}, "aa", "a1", { :two => "bla1"}, ")" ]
+      end
+    
+    
+      describe "real examples" do
+        
+        it "should find host" do
+          parser = Appstats::Parser.new(:rules => ":operation :action :date on :host where :contexts")
+          parser.parse("# logins between 2010-01-15 and 2010-01-31 on your.localnet").should == true
+          parser.results.should == {:operation => "#", :action => "logins", :date => "between 2010-01-15 and 2010-01-31", :host => "your.localnet", :contexts => nil }
+        end
+        
+      end
+      
+    end
+    
+  end
+end