appstats 0.7.0 → 0.8.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    appstats (0.7.0)
+    appstats (0.8.0)
       daemons
       net-scp
       rails (>= 2.3.0)

data/db/migrations/20110215155830_create_appstats_hosts.rb

@@ -0,0 +1,13 @@
+class CreateAppstatsHosts < ActiveRecord::Migration
+  def self.up
+    create_table :appstats_hosts do |t|
+      t.string :name
+      t.string :status
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :appstats_hosts
+  end
+end

data/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20110210225606) do
+ActiveRecord::Schema.define(:version => 20110215155830) do
 
   create_table "appstats_actions", :force => true do |t|
     t.string   "name"
@@ -56,6 +56,13 @@ ActiveRecord::Schema.define(:version => 20110210225606) do
   add_index "appstats_entries", ["year", "month"], :name => "index_entries_by_month"
   add_index "appstats_entries", ["year"], :name => "index_entries_by_year"
 
+  create_table "appstats_hosts", :force => true do |t|
+    t.string   "name"
+    t.string   "status"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
   create_table "appstats_log_collectors", :force => true do |t|
     t.string   "host"
     t.string   "filename"

data/lib/appstats.rb

@@ -12,6 +12,8 @@ require "#{File.dirname(__FILE__)}/appstats/logger"
 require "#{File.dirname(__FILE__)}/appstats/log_collector"
 require "#{File.dirname(__FILE__)}/appstats/query"
 require "#{File.dirname(__FILE__)}/appstats/result"
+require "#{File.dirname(__FILE__)}/appstats/host"
+require "#{File.dirname(__FILE__)}/appstats/parser"
 require "#{File.dirname(__FILE__)}/appstats/test_object"
 
 # required in the appstats.gemspec

data/lib/appstats/host.rb

@@ -0,0 +1,18 @@
+module Appstats
+  class Host < ActiveRecord::Base
+    set_table_name "appstats_hosts"
+    
+    attr_accessible :name, :status
+
+    def self.update_hosts
+      sql = "select distinct(host) from appstats_log_collectors where host not in (select name from appstats_hosts)"
+      count = 0
+      ActiveRecord::Base.connection.execute(sql).each do |row| 
+        Appstats::Host.create(:name => row[0], :status => 'derived')
+        count += 1
+      end
+      count
+    end
+  
+  end
+end

data/lib/appstats/parser.rb

@@ -0,0 +1,239 @@
+
+module Appstats
+  class Parser
+
+    attr_reader :raw_rules, :rules, :repeating, :raw_tokenize, :tokenize, :tokenize_no_spaces, :tokenize_regex, :tokenize_regex_no_spaces, :results, :raw_results, :constants
+
+    def initialize(data = {})
+      @raw_rules = data[:rules]
+      @raw_tokenize = data[:tokenize]
+      @repeating = data[:repeating] == true
+      @results = {}
+      @raw_results = []
+      update_tokens
+      update_rules
+    end
+
+    def parse(input)
+      @results = {}
+      @raw_results = []
+      return false if input.nil?
+      return false if @rules.size == 0
+
+      @rule_index = 0
+      @max_rule_index = @rules.size - 1
+      @previous_text_so_far = input.strip
+      @text_so_far = @previous_text_so_far
+      @remaining_constants = @constants.dup
+      
+      while !@text_so_far.blank?
+        process_constant_if_present
+        break if @rule_index > @max_rule_index && !@repeating
+        @rule_index = 0 if @rule_index > @max_rule_index
+
+        rule = @rules[@rule_index]
+        @rule_index += 1
+        
+        if rule.kind_of?(Hash)
+          if rule[:stop] == :constant
+            was_found = false
+            @remaining_constants.each_with_index do |k,index|
+              p = parse_word(@text_so_far,k,true)
+              if p[0].nil?
+                unset_rules_until(k)
+              else
+                (index-1).downto(0) do |i|
+                  @remaining_constants.delete_at(i)
+                end
+                add_results(rule[:rule],p[0])
+                @text_so_far = p[1]
+                was_found = true
+                break
+              end
+            end
+            unless was_found
+              add_results(rule[:rule],@text_so_far)
+              @text_so_far = nil
+            end
+          else
+            p = parse_word(@text_so_far,rule[:stop],false)
+            add_results(rule[:rule],p[0])
+            @text_so_far = p[1]
+          end
+        end
+        break if @previous_text_so_far == @text_so_far
+        @previous_text_so_far = @text_so_far
+      end
+      remove_tokens_at_start(@text_so_far)
+      unset_rules_until(nil)
+      true
+    end
+    
+    def self.parse_constant(current_text,constant)
+      answer = [nil,nil]
+      return answer if current_text.blank? || constant.nil?
+      current_text.strip!
+      m = current_text.match(/^(#{constant})(.*)$/im)
+      answer[0] = m[1] unless m.nil?
+      answer[1] = m.nil? ? current_text : m[2]
+      clean_parsed_word(answer)
+    end
+    
+    def self.merge_regex_filter(a,b)
+      return "" if a.blank? && b.blank?
+      return "(#{a})" if b.blank?
+      return "(#{b})" if a.blank?
+      "(#{a}|#{b})"
+    end
+    
+    def parse_word(current_text,stop_on,strict = false)
+      answer = [nil,nil]
+      return answer if current_text.blank? || stop_on.nil?
+      current_text.strip!
+
+      current_text = remove_tokens_at_start(current_text)
+
+      if stop_on == :end
+        filter = Parser.merge_regex_filter(nil,@tokenize_regex)
+        m = current_text.match(/^(.*?)(#{filter}.*)$/im)
+        if m.nil? || m[1].blank?
+          answer[0] = current_text
+        else
+          answer[0] = m[1]
+          answer[1] = m[2]
+        end
+      elsif stop_on == :space
+        filter = Parser.merge_regex_filter('\s',@tokenize_regex)
+        m = current_text.match(/^(.*?)(#{filter}.*)$/im)
+        if m.nil?
+          answer[0] = current_text
+        else
+          answer[0] = m[1]
+          answer[1] = m[2]
+        end
+      else
+        filter = Parser.merge_regex_filter(stop_on,@tokenize_regex)
+        m = current_text.match(/^(.*?)(#{filter}.*)$/im)
+        if strict
+          answer[0] = m[1] unless m.nil?
+          answer[1] = m.nil? ? current_text : m[2]
+        else
+          answer[0] = m.nil? ? current_text : m[1]
+          answer[1] = m[2] unless m.nil?
+        end
+      end
+      Parser.clean_parsed_word(answer)
+    end
+    
+    private
+      
+      def self.clean_parsed_word(answer)
+        answer[0].strip! unless answer[0].nil?
+        answer[1].strip! unless answer[1].nil?
+        answer[0] = nil if answer[0].blank?
+        answer[1] = nil if answer[1].blank?
+        answer 
+      end
+      
+      def process_constant_if_present
+        while process_tokens_if_present; end
+        to_delete = nil
+        @remaining_constants.each do |k|
+          p = Parser.parse_constant(@text_so_far,k)
+          next if p[0].nil?
+          to_delete = k
+          unset_rules_until(k)
+          add_constant(p[0])
+          @text_so_far = p[1]
+        end
+        @remaining_constants.delete(to_delete) unless to_delete.nil?
+      end
+      
+      def process_tokens_if_present
+        found = false
+        @tokenize.each do |k|
+          p = Parser.parse_constant(@text_so_far,k)
+          next if p[0].nil?
+          add_constant(p[0])
+          @text_so_far = p[1]
+          found = true
+        end
+        found
+      end
+      
+      def unset_rules_until(k)
+        @rules[@rule_index..-1].each do |rule|
+          @rule_index += 1
+          break if rule.eql?(k)
+          add_results(rule[:rule],nil) if rule.kind_of?(Hash)
+        end
+      end
+      
+      def update_tokens
+        @tokenize = []
+        @tokenize_no_spaces = []
+        @tokenize_regex = nil
+        @tokenize_regex_no_spaces = nil
+        return if @raw_tokenize.blank?
+        @raw_tokenize.split(" ").each do |token|
+          current_token = token.upcase
+          current_token.gsub!("(",'\(')
+          current_token.gsub!(")",'\)')
+          current_token.gsub!("|",'\|')
+          @tokenize_no_spaces<< current_token
+          current_token = "\\s+#{current_token}" unless current_token.match(/.*[a-z].*/i).nil?
+          @tokenize<< current_token
+        end
+        @tokenize_regex_no_spaces = @tokenize_no_spaces.join("|")
+        @tokenize_regex = @tokenize.join("|")
+      end
+      
+      def update_rules
+        @rules = []
+        @constants = []
+        current_rule = nil
+        return if @raw_rules.blank?
+        @raw_rules.split(" ").each do |rule|
+
+          if rule.starts_with?(":") && rule.size > 1
+            current_rule = { :rule => rule[1..-1].to_sym, :stop => :end }
+            previous_stop_on = :space
+          else
+            current_rule = rule.upcase
+            @constants<< current_rule
+            previous_stop_on = :constant
+          end
+
+          if @rules.last.kind_of?(Hash)
+            @rules.last[:stop] = previous_stop_on   
+          end
+          
+          @rules<< current_rule
+        end
+      end
+
+      def add_constant(value)
+        @raw_results<< value
+      end
+
+      def add_results(rule_name,value)
+        @raw_results<< { rule_name => value }
+        @results[rule_name] = value
+      end
+
+      def remove_tokens_at_start(current_text)
+        return current_text if current_text.blank?
+        current_text.blank?
+        loop do
+          break if @tokenize_regex.blank?
+          m = current_text.match(/^(#{@tokenize_regex_no_spaces})(.*)$/im)
+          break if m.nil? || m[1].blank?
+          add_constant(m[1])
+          current_text = m[2]
+          current_text.strip! unless current_text.nil?
+        end
+        current_text
+      end
+
+  end
+end

data/lib/appstats/query.rb

@@ -2,8 +2,9 @@
 module Appstats
   class Query
 
+    @@nill_query = "select 0 from appstats_entries LIMIT 1"
     @@default = "1=1"
-    attr_accessor :query, :action, :host, :date_range, :query_to_sql
+    attr_accessor :query, :action, :host, :date_range, :query_to_sql, :contexts
 
     def initialize(data = {})
       self.query=(data[:query])
@@ -28,17 +29,60 @@ module Appstats
       return @@default if m.nil?
       host = m[1]
       return @@default if host == '' or host.nil?
-      "EXISTS (select * from appstats_log_collectors where appstats_entries.appstats_log_collector_id = id and host = '#{host}' )"
+      "EXISTS (select * from appstats_log_collectors where appstats_entries.appstats_log_collector_id = appstats_log_collectors.id and host = '#{host}' )"
     end
 
-    def self.context_filter_to_sql(raw_input)
-      return @@default if raw_input.nil?
-      m = raw_input.match(/([^']+)=([^']+)/)
-      return @@default if m.nil?
-      key = m[1].strip
-      value = m[2].strip
-      return @@default if key == '' or key.nil?
-      "EXISTS (select * from appstats_contexts where appstats_entries.id = appstats_contexts.appstats_entry_id and context_key='#{key}' and context_value='#{value}' )"
+    def self.contexts_filter_to_sql(raw_input)
+      context_parser = Appstats::Parser.new(:rules => ":context", :repeating => true, :tokenize => "|| && = <= >= <> != ( )")
+      return @@default if (raw_input.blank? || !context_parser.parse(raw_input))
+      sql = "EXISTS (select * from appstats_contexts where appstats_entries.id = appstats_contexts.appstats_entry_id and ("
+      
+      status = :next
+      comparator = "="
+      context_parser.raw_results.each do |entry|
+        if entry.kind_of?(String)
+          sqlentry = sqlize(entry)
+          if Query.comparator?(entry) && status == :waiting_comparator
+            comparator = sqlize(entry)
+            status = :waiting_operand
+          else
+            sql += ")" if status == :waiting_comparator
+            sql += " #{sqlentry}"
+            status = :next
+          end
+          next
+        end
+        if status == :next
+          status = :waiting_comparator
+          sql += " (context_key='#{sqlclean(entry[:context])}'"
+        else
+          status = :next
+          sql += " and context_value#{comparator}'#{sqlclean(entry[:context])}')"
+        end
+      end
+      sql += ")" if status == :waiting_comparator
+      sql += "))"
+      sql
+    end
+    
+    def self.sqlize(input)
+      return "and" if input == "&&"
+      return "or" if input == "||"
+      return "<>" if input == "!="
+      input
+    end
+    
+    def self.sqlclean(raw_input)
+      return raw_input if raw_input.blank?
+      m = raw_input.match(/^['"](.*)['"]$/)
+      input = m.nil? ? raw_input : m[1]
+      input = input.gsub(/\\/, '\&\&').gsub(/'/, "''")
+      input
+    end
+    
+    def self.comparator?(raw_input)
+      return false if raw_input.nil?
+      ["=","!=","<>",">","<",">=","<="].include?(raw_input)
     end
     
     private
@@ -49,35 +93,50 @@ module Appstats
       end
       
       def parse_query
-        @query_to_sql = "select count(*) from appstats_entries"
-        @action = nil
-        @host = nil
-        @date_range = DateRange.new
-        return @query_to_sql if @query.nil?
-        current_query = @query
-
-        m = current_query.match(/^\s*(\#)\s*([^\s]*)\s*(.*)/)
-        return @query_to_sql if m.nil?
-        if m[1] == "#"
-          @action = normalize_action_name(m[2])
-          @query_to_sql += " where action = '#{@action}'"
-        end
-        current_query = m[3]
-
-        m_on_server = current_query.match(/^(.*)?\s*on\s*server\s*(.*)$/)
-        date_range_text = m_on_server.nil? ? current_query : m_on_server[1]
-        if date_range_text.size > 0
-          @date_range = DateRange.parse(date_range_text)
+        reset_query
+        return nil_query if @query.nil?
+        current_query = fix_legacy_structures(@query)
+        
+        parser = Appstats::Parser.new(:rules => ":operation :action :date on :host where :contexts")
+        return nil_query unless parser.parse(current_query)
+        
+        @operation = parser.results[:operation]
+        @action = normalize_action_name(parser.results[:action])
+        @date_range = DateRange.parse(parser.results[:date])
+        @host = parser.results[:host]
+        @contexts = parser.results[:contexts]
+        
+        if @operation == "#"
+          @query_to_sql = "select count(*) from appstats_entries"
+          @query_to_sql += " where action = '#{@action}'" unless @action.blank?
           @query_to_sql += " and #{@date_range.to_sql}" unless @date_range.to_sql == "1=1"
+          @query_to_sql += " and #{Query.host_filter_to_sql(@host)}" unless @host.nil?
+          @query_to_sql += " and #{Query.contexts_filter_to_sql(@contexts)}" unless @contexts.nil?
         end
-        return @query_to_sql if m_on_server.nil?
-
-        @host = m_on_server[2]
-        @query_to_sql += " and exists (select * from appstats_log_collectors where appstats_entries.appstats_log_collector_id = appstats_log_collectors.id and host = '#{@host}')"
 
         @query_to_sql
-      end      
-    
+      end    
+      
+      def fix_legacy_structures(raw_input)
+        query = raw_input.gsub(/on\s*server/,"on")
+        query
+      end  
+   
+      def sql_for_conext(context_name,contact_value)
+        "EXISTS(select * from appstats_contexts where appstats_contexts.appstats_entry_id=appstats_entries.id and context_key='#{context_name}' and context_value='#{contact_value}' )"
+      end
+      
+      def nil_query
+        @query_to_sql = @@nill_query
+        @query_to_sql
+      end
+      
+      def reset_query
+        @action = nil
+        @host = nil
+        nil_query
+        @date_range = DateRange.new
+      end
     
   end
 end