appstats 0.19.7 → 0.20.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    appstats (0.19.7)
+    appstats (0.20.0)
       daemons
       net-scp
       rails (>= 2.3.0)

data/db/migrations/20110330171745_create_appstats_action_contexts.rb

@@ -0,0 +1,14 @@
+class CreateAppstatsActionContexts < ActiveRecord::Migration
+  def self.up
+    create_table :appstats_action_context_keys do |t|
+      t.string :action_name
+      t.string :context_key
+      t.string :status
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :appstats_action_context_keys
+  end
+end

data/db/schema.rb

@@ -10,7 +10,15 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20110321154125) do
+ActiveRecord::Schema.define(:version => 20110330171745) do
+
+  create_table "appstats_action_context_keys", :force => true do |t|
+    t.string   "action_name"
+    t.string   "context_key"
+    t.string   "status"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
 
   create_table "appstats_actions", :force => true do |t|
     t.string   "name"

data/lib/appstats.rb

@@ -7,6 +7,7 @@ require "#{File.dirname(__FILE__)}/appstats/entry_date"
 require "#{File.dirname(__FILE__)}/appstats/date_range"
 require "#{File.dirname(__FILE__)}/appstats/action"
 require "#{File.dirname(__FILE__)}/appstats/context"
+require "#{File.dirname(__FILE__)}/appstats/action_context_key"
 require "#{File.dirname(__FILE__)}/appstats/tasks"
 require "#{File.dirname(__FILE__)}/appstats/logger"
 require "#{File.dirname(__FILE__)}/appstats/log_collector"
@@ -18,6 +19,7 @@ require "#{File.dirname(__FILE__)}/appstats/result_job"
 require "#{File.dirname(__FILE__)}/appstats/host"
 require "#{File.dirname(__FILE__)}/appstats/friendly_timer"
 require "#{File.dirname(__FILE__)}/appstats/context_key"
+require "#{File.dirname(__FILE__)}/appstats/appstats_query"
 require "#{File.dirname(__FILE__)}/appstats/context_value"
 require "#{File.dirname(__FILE__)}/appstats/test_object"
 require "#{File.dirname(__FILE__)}/appstats/test_query"

data/lib/appstats/action_context_key.rb

@@ -0,0 +1,23 @@
+
+module Appstats
+  class ActionContextKey < ActiveRecord::Base
+    set_table_name "appstats_action_context_keys"
+    
+    attr_accessible :action_name, :context_key, :status
+  
+    def self.update_action_context_keys
+      sql = "select action,context_key,count(*) as num
+      from appstats_entries 
+      inner join appstats_contexts on appstats_contexts.appstats_entry_id = appstats_entries.id
+      where (action,context_key) not in (select action_name, context_key from appstats_action_context_keys)
+      group by action,context_key"
+      count = 0
+      ActiveRecord::Base.connection.execute(sql).each do |row| 
+        Appstats::ActionContextKey.create(:action_name => row[0], :context_key => row[1], :status => 'derived')
+        count += 1
+      end
+      count
+    end
+    
+  end
+end

data/lib/appstats/appstats_query.rb

@@ -0,0 +1,92 @@
+class AppstatsQuery
+
+  attr_accessor :query
+  
+  @@default_query = "select 0 as num"
+  
+  @@action_to_available_contexts = 
+  {
+    "appstats_queries" => [ "action", "contexts", "group_by" ],
+    "bomb" => []
+  }
+  
+  def query_to_sql
+    return @@default_query if query.nil?
+    query.query_to_sql
+  end
+  
+  def group_query_to_sql
+    return nil if query.nil?
+    query.group_query_to_sql
+  end
+  
+  def process_query
+    return if query.nil?
+    query.query_to_sql = @@default_query
+    query.group_query_to_sql = nil
+    return if query.action.blank?
+  
+    action = query.action.pluralize.downcase
+    case action
+      when "appstats_queries"
+        count_filter = "COUNT(*)"
+        query.query_to_sql = "select #{count_filter} as num from appstats_results#{build_where_clause}"
+      when "bombs"
+        query.query_to_sql = "invalid sql"
+    end
+    query.group_query_to_sql = query.query_to_sql.sub("#{count_filter} as num","#{context_key_filter_name(action)} as context_key_filter, #{context_value_filter_name(action)} as context_value_filter, COUNT(*) as num") + " group by context_value_filter" unless query.group_by.empty?
+  end
+  
+  def run
+    query.run
+  end
+  
+  def db_connection
+    ActiveRecord::Base.connection
+  end
+  
+  def self.available_action?(action)
+    return false if action.blank?
+    return @@action_to_available_contexts.keys.include?(action.downcase.pluralize)
+  end
+  
+  private
+  
+    def build_where_clause
+      where_clause = ""
+      action = query.action.pluralize.downcase
+      available_contexts = @@action_to_available_contexts[action]
+      status = :context
+      query.parsed_contexts.each do |lookup|
+        next if status == :context && lookup.kind_of?(String)
+        next if status == :join && !lookup.kind_of?(String)
+        next if status == :context && (lookup[:context_value].nil? || !available_contexts.include?(lookup[:context_key]))
+        
+        where_clause = " where" if where_clause.blank?
+        if lookup.kind_of?(String)
+          where_clause += " #{lookup}" 
+          status = :context
+        elsif !lookup[:context_value].nil? && available_contexts.include?(lookup[:context_key])
+          where_clause += " #{database_column(action,lookup[:context_key])} #{lookup[:comparator]} '#{Appstats::Query.sqlclean(lookup[:context_value])}'"
+          status = :join
+        end
+      end
+      where_clause
+    end
+    
+    def context_key_filter_name(action)
+      "'" + query.group_by.join(",") + "'"      
+    end
+    
+    def context_value_filter_name(action)
+      database_names = query.group_by.collect do |name|
+        database_column(action,name)
+      end
+      "concat(ifnull("+ database_names.join(",'--'),',',ifnull(") +",'--'))"
+    end
+  
+    def database_column(action,name)
+      name
+    end
+
+end

data/lib/appstats/parser.rb

@@ -73,13 +73,27 @@ module Appstats
       true
     end
     
+    def self.alpha?(raw_input)
+      return false if raw_input.nil?
+      !raw_input.match(/^[A-Za-z]+$/i).nil?
+    end
+    
     def self.parse_constant(current_text,constant)
       answer = [nil,nil]
       return answer if current_text.blank? || constant.nil?
       current_text.strip!
-      m = current_text.match(/^(#{constant})(.*)$/im)
+      
+      remaining_text_index = -1
+      if alpha?(constant)
+        m = current_text.match(/^(#{constant})(\s|$)(.*)$/im)  
+        remaining_text_index = 3
+      else
+        m = current_text.match(/^(#{constant})(.*)$/im)  
+        remaining_text_index = 2
+      end
+      
       answer[0] = m[1] unless m.nil?
-      answer[1] = m.nil? ? current_text : m[2]
+      answer[1] = m.nil? ? current_text : m[remaining_text_index]
       clean_parsed_word(answer)
     end
     
@@ -186,7 +200,7 @@ module Appstats
           current_token.gsub!(")",'\)')
           current_token.gsub!("|",'\|')
           @tokenize_no_spaces<< current_token
-          current_token = "\\s+#{current_token}" unless current_token.match(/.*[a-z].*/i).nil?
+          current_token = "\\s+#{current_token}(\\s|$)" unless current_token.match(/.*[a-z].*/i).nil?
           @tokenize<< current_token
         end
         @tokenize_regex_no_spaces = @tokenize_no_spaces.join("|")
@@ -209,7 +223,7 @@ module Appstats
             current_rule = rule.upcase
             current_rule_no_spaces = current_rule
             @constants_no_spaces<< current_rule_no_spaces
-            current_rule = "\\s+#{current_rule}" unless current_rule.match(/.*[a-z].*/i).nil?
+            current_rule = "\\s+#{current_rule}(\\s|$)" unless current_rule.match(/.*[a-z].*/i).nil?
             @constants<< current_rule
             previous_stop_on = :constant
           end

data/lib/appstats/version.rb

@@ -1,3 +1,3 @@
 module Appstats
-  VERSION = "0.19.7"
+  VERSION = "0.20.0"
 end

data/lib/daemons/appstats_log_collector.rb

@@ -63,6 +63,7 @@ while($running) do
   Appstats::Host.update_hosts
   Appstats::ContextKey.update_context_keys
   Appstats::ContextValue.update_context_values
+  Appstats::ActionContextKey.update_action_context_keys
   Appstats::LogCollector.remove_remote_files(appstats_config["remote_servers"])
   ActiveRecord::Base.connection.disconnect!
 end

data/spec/action_context_key_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+
+module Appstats
+  describe ActionContextKey do
+
+    before(:each) do
+      Appstats::Entry.delete_all
+      Appstats::Context.delete_all
+      Appstats::ActionContextKey.delete_all
+      @action = Appstats::ActionContextKey.new
+    end
+    
+    
+    
+    describe "#initialize" do
+
+      it "should set action_name to nil" do
+        @action.action_name.should == nil
+      end
+
+      it "should set context_key to nil" do
+        @action.context_key.should == nil
+      end
+
+      it "should set status to nil" do
+        @action.status.should == nil
+      end
+      
+      it "should set on constructor" do
+        action = Appstats::ActionContextKey.new(:action_name => 'a', :context_key => 'b', :status => 'c')
+        action.action_name.should == 'a'
+        action.context_key.should == 'b'
+        action.status.should == 'c'
+      end
+    
+    end
+
+    describe "#update_action_context_keys" do
+      
+      it "should do nothing if no events" do
+        Appstats::ActionContextKey.update_action_context_keys.should == 0
+        Appstats::ActionContextKey.count.should == 0
+      end
+      
+      it "should ignore actions without any contexts" do
+        Appstats::Entry.create_from_logger('a')
+        Appstats::ActionContextKey.update_action_context_keys.should == 0
+        Appstats::ActionContextKey.count.should == 0
+        
+      end
+      
+      it "should add entry action / context names" do
+        Appstats::Entry.create_from_logger('a',:blah => "x")
+        Appstats::ActionContextKey.update_action_context_keys.should == 1
+        Appstats::ActionContextKey.count.should == 1
+        
+        action = Appstats::ActionContextKey.last
+        action.action_name = 'a'
+        action.context_key = 'blah'
+        action.status = 'derived'
+      end
+
+      it "should ignore existing entries action / context names" do
+        Appstats::Entry.create_from_logger('a',:blah => "x")
+        Appstats::ActionContextKey.update_action_context_keys.should == 1
+        Appstats::ActionContextKey.update_action_context_keys.should == 0
+      end
+
+      it "should ignore duplicates" do
+        Appstats::Entry.create_from_logger('a',:blah => "x")
+        Appstats::Entry.create_from_logger('a',:blah => "y")
+        Appstats::ActionContextKey.update_action_context_keys.should == 1
+        Appstats::ActionContextKey.count.should == 1
+      end
+      
+      
+    end
+
+
+  end
+end

data/spec/appstats_query_spec.rb

@@ -0,0 +1,144 @@
+require 'spec_helper'
+
+module Appstats
+
+  describe AppstatsQuery do
+
+    before(:each) do
+      Time.stub!(:now).and_return(Time.parse('2010-09-21 23:15:20'))
+      @appstats_query = AppstatsQuery.new
+    end
+
+    describe "#available_action?" do
+  
+      it "should be false for nil" do
+        AppstatsQuery.available_action?(nil).should == false
+        AppstatsQuery.available_action?("").should == false
+      end
+  
+      it "should be false for blah" do
+        AppstatsQuery.available_action?("blah").should == false
+      end
+  
+      it "should be true for appstats_queries" do
+        AppstatsQuery.available_action?("appstats_queries").should == true
+        AppstatsQuery.available_action?("Appstats_queries").should == true
+        AppstatsQuery.available_action?("appstats_query").should == true
+      end
+  
+    end
+  
+    describe "#query_to_sql" do
+  
+      it "should be nil if no query" do
+        @appstats_query.query_to_sql.should == "select 0 as num"
+      end
+  
+      it "should be based on query" do
+        @appstats_query.query = Appstats::Query.new(:query => "# appstats_queries")
+        @appstats_query.query_to_sql.should == @appstats_query.query.query_to_sql
+      end
+  
+    end
+  
+    describe "#group_query_to_sql" do
+  
+      it "should be nil if no query" do
+        @appstats_query.group_query_to_sql.should == nil
+      end
+  
+      it "should be based on query" do
+        @appstats_query.query = Appstats::Query.new(:query => "# appstats_queries")
+        @appstats_query.group_query_to_sql.should == @appstats_query.query.group_query_to_sql
+      end
+  
+    end
+  
+    describe "#process_query" do
+  
+      it "should support nil query" do
+        @appstats_query.process_query
+        @appstats_query.query_to_sql.should == "select 0 as num"
+        @appstats_query.group_query_to_sql.should == nil
+      end
+  
+      it "should be case insensitive on action" do
+        @appstats_query.query = Appstats::Query.new(:query => "# Appstats_Queries", :query_type => "Appstats::AppstatsQuery")
+        @appstats_query.process_query
+        @appstats_query.query_to_sql.should == "select COUNT(*) as num from appstats_results"
+        
+        @appstats_query.query = Appstats::Query.new(:query => "# aPpstats_Queries", :query_type => "Appstats::AppstatsQuery")
+        @appstats_query.process_query
+        @appstats_query.query_to_sql.should == "select COUNT(*) as num from appstats_results"
+      end
+        
+      it "should support singular names" do
+        @appstats_query.query = Appstats::Query.new(:query => "# appstats_query", :query_type => "Appstats::AppstatsQuery")
+        @appstats_query.process_query
+        @appstats_query.query_to_sql.should == "select COUNT(*) as num from appstats_results"
+      end
+        
+      it "should handle nil actions" do
+        @appstats_query.query = Appstats::Query.new(:query => "", :query_type => "Appstats::AppstatsQuery")
+        @appstats_query.process_query
+        @appstats_query.query_to_sql.should == "select 0 as num"
+      end
+        
+      it "should call query.run" do
+        @appstats_query.query = Appstats::Query.new(:query => "# appstats_queries", :query_type => "Appstats::AppstatsQuery")
+        @appstats_query.query.stub!(:run).and_return("call-worked")
+        @appstats_query.run.should == "call-worked"
+      end
+        
+      # THIS IS A SLOW TEST ONLY TO BE RUN IF THINGS START ACTING STRANGE
+      it "should actually execute code properly" do
+        Result.create
+        @appstats_query.query = Appstats::Query.new(:query => "# appstats_queries", :query_type => "Appstats::AppstatsQuery")
+        @appstats_query.run.count.should > 0
+      end
+        
+      describe "# appstats_queries" do
+        
+        it "should support #appstats_queries" do
+          @appstats_query.query = Appstats::Query.new(:query => "# appstats_queries", :query_type => "Appstats::AppstatsQuery")
+          @appstats_query.process_query
+          @appstats_query.query_to_sql.should == "select COUNT(*) as num from appstats_results"
+          @appstats_query.group_query_to_sql.should == nil
+        end
+
+        it "should support where clause for action, contexts, group_by" do
+          @appstats_query.query = Appstats::Query.new(:query => "# appstats_queries where action = abc AND contexts = 'def' || group_by like 'hik'", :query_type => "Appstats::AppstatsQuery")
+          @appstats_query.process_query
+          @appstats_query.query_to_sql.should == "select COUNT(*) as num from appstats_results where action = 'abc' AND contexts = 'def' or group_by like 'hik'"
+          @appstats_query.group_query_to_sql.should == nil
+        end
+        
+        it "should support group by action, contexts, group_by" do
+          @appstats_query.query = Appstats::Query.new(:query => "# appstats_queries group by action, contexts, group_by", :query_type => "Appstats::AppstatsQuery")
+          @appstats_query.process_query
+          @appstats_query.query_to_sql.should == "select COUNT(*) as num from appstats_results"
+          @appstats_query.group_query_to_sql.should == "select 'action,contexts,group_by' as context_key_filter, concat(ifnull(action,'--'),',',ifnull(contexts,'--'),',',ifnull(group_by,'--')) as context_value_filter, COUNT(*) as num from appstats_results group by context_value_filter"
+        end
+        
+      #   it "should support group by media,network" do
+      #     @appstats_query.query = Appstats::Query.new(:query => "# buildings group by media, network", :query_type => "Appstats::AppstatsQuery")        
+      #     @appstats_query.process_query
+      #     @appstats_query.query_to_sql.should == "select COUNT(DISTINCT operator_accesses.id) as num from operator_accesses left join physical_addresses on physical_addresses.id = operator_accesses.physical_address_id left join operator_networks on operator_networks.id = operator_accesses.operator_network_id left join service_providers on service_providers.id = operator_networks.service_provider_id"
+      #     @appstats_query.group_query_to_sql.should == "select 'media,network' as context_key_filter, concat(media_types.name,',',operator_networks.name) as context_value_filter, COUNT(*) as num from operator_accesses left join physical_addresses on physical_addresses.id = operator_accesses.physical_address_id left join operator_networks on operator_networks.id = operator_accesses.operator_network_id left join service_providers on service_providers.id = operator_networks.service_provider_id group by context_value_filter"
+      #   end
+        
+      end
+    end
+  
+    describe "#db_connection" do
+  
+      it "should use the extract_env" do
+        @appstats_query.query = Appstats::Query.new(:query => "# blahs on blah")
+        @appstats_query.db_connection.should == ActiveRecord::Base.connection
+      end
+  
+    end
+
+  end
+
+end

data/spec/entry_spec.rb

@@ -193,18 +193,18 @@ module Appstats
       end
       
       it "should understand an entry without contexts" do
-        entry = Entry.create_from_logger_string("0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
+        entry = Entry.create_from_logger_string("0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
         Entry.count.should == @before_count + 1
         entry.action.should == "address_search"
-        entry.raw_entry.should == "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
+        entry.raw_entry.should == "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
         entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
       end
       
       it "should understand contexts" do
-        entry = Entry.create_from_logger_string("0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
+        entry = Entry.create_from_logger_string("0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
         Entry.count.should == @before_count + 1
         entry.action.should == "address_filter"
-        entry.raw_entry.should == "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
+        entry.raw_entry.should == "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
         entry.occurred_at.should == Time.parse("2010-09-21 23:15:20")
         entry.contexts.size.should == 2
         entry.contexts[0].context_key = "app_name"
@@ -214,10 +214,10 @@ module Appstats
       end
       
       it "should handle 'action' as a context" do
-        entry = Entry.create_from_logger_string('0.19.7 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : action=save_ovcen : app_name=cdb')
+        entry = Entry.create_from_logger_string('0.20.0 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : action=save_ovcen : app_name=cdb')
         Entry.count.should == @before_count + 1
         entry.action.should == "page-view"
-        entry.raw_entry.should == "0.19.7 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : action=save_ovcen : app_name=cdb"
+        entry.raw_entry.should == "0.20.0 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : action=save_ovcen : app_name=cdb"
         entry.occurred_at.should == Time.parse("2011-02-24 12:59:57")
         entry.contexts.size.should == 2
         entry.contexts[0].context_key = "action"
@@ -228,10 +228,10 @@ module Appstats
       end
       
       it "should handle multiple of the same 'context'" do
-        entry = Entry.create_from_logger_string('0.19.7 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : app_name=market : app_name=cdb')
+        entry = Entry.create_from_logger_string('0.20.0 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : app_name=market : app_name=cdb')
         Entry.count.should == @before_count + 1
         entry.action.should == "page-view"
-        entry.raw_entry.should == "0.19.7 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : app_name=market : app_name=cdb"
+        entry.raw_entry.should == "0.20.0 setup[:,=,-n] 2011-02-24 12:59:57 action=page-view : app_name=market : app_name=cdb"
         entry.occurred_at.should == Time.parse("2011-02-24 12:59:57")
         entry.contexts.size.should == 2
         entry.contexts[0].context_key = "app_name"

data/spec/logger_spec.rb

@@ -122,12 +122,12 @@ module Appstats
       
       it "should accept numbers" do
         Appstats::Logger.entry(5, :blah => 6)   
-        Appstats::Logger.raw_read.should == ["0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=5 : blah=6"]
+        Appstats::Logger.raw_read.should == ["0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=5 : blah=6"]
       end
 
       it "should accept arrays" do
         Appstats::Logger.entry('search', :provider => [ 'one', 'two' ])   
-        Appstats::Logger.raw_read.should == ["0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=search : provider=one : provider=two"]
+        Appstats::Logger.raw_read.should == ["0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=search : provider=one : provider=two"]
       end
 
 
@@ -137,7 +137,7 @@ module Appstats
 
       it "should look similar to regular entry" do
         Appstats::Logger.exception_entry(RuntimeError.new("blah"),:on => "login")   
-        Appstats::Logger.raw_read.should == ["0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=appstats-exception : error=blah : on=login"]
+        Appstats::Logger.raw_read.should == ["0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=appstats-exception : error=blah : on=login"]
       end
       
     end
@@ -154,47 +154,47 @@ module Appstats
     
        it "should handle a statistics entry" do
          expected = { :action => "address_search", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
+         actual = Appstats::Logger.entry_to_hash("0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search")
          actual.should == expected
        end
        
        it "should handle contexts" do
          expected = { :action => "address_filter", :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => 'Market' }
-         actual = Appstats::Logger.entry_to_hash("0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live")
          actual.should == expected
        end
 
        it "should handle multiple actions" do
          expected = { :action => ["address_filter", "blah"], :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => 'Market' }
-         actual = Appstats::Logger.entry_to_hash("0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : action=blah : app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : action=blah : app_name=Market : server=Live")
          actual.should == expected
        end
 
        it "should handle multiple of same context" do
          expected = { :action => "address_filter", :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => ['Sin','Market'] }
-         actual = Appstats::Logger.entry_to_hash("0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Sin : app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Sin : app_name=Market : server=Live")
          actual.should == expected
        end
 
        it "should handle no actions" do
          expected = { :action => "UNKNOWN_ACTION", :timestamp => "2010-09-21 23:15:20", :server => "Live", :app_name => 'Market' }
-         actual = Appstats::Logger.entry_to_hash("0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 app_name=Market : server=Live")
+         actual = Appstats::Logger.entry_to_hash("0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 app_name=Market : server=Live")
          actual.should == expected
        end
     
        it "should handle actions with the delimiter (and change the delimiter)" do
          expected = { :action => "address:=search-n", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.19.7 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n")
+         actual = Appstats::Logger.entry_to_hash("0.20.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n")
          actual.should == expected
     
          expected = { :action => "address::search==--n", :timestamp => "2010-09-21 23:15:20" }
-         actual = Appstats::Logger.entry_to_hash("0.19.7 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n")
+         actual = Appstats::Logger.entry_to_hash("0.20.0 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n")
          actual.should == expected
        end
     
        it "should handle contexts with the delimiter (and change the delimiter)" do
          expected = { :action => "address", :timestamp => "2010-09-21 23:15:20", :server => "market:eval=-n" }
-         actual = Appstats::Logger.entry_to_hash("0.19.7 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n")
+         actual = Appstats::Logger.entry_to_hash("0.20.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n")
          actual.should == expected
        end
        
@@ -203,66 +203,66 @@ module Appstats
     describe "#entry_to_s" do
       
       it "should handle a statistics entry" do
-        expected = "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
+        expected = "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search"
         actual = Appstats::Logger.entry_to_s("address_search")
         actual.should == expected
       end
       
       it "should handle numbers" do
-        expected = "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=1 : note=2.2"
+        expected = "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=1 : note=2.2"
         actual = Appstats::Logger.entry_to_s(1,:note => 2.2)
         actual.should == expected
       end
       
       it "should handle default contexts" do
         Appstats::Logger.default_contexts[:app_name] = "market"
-        expected = "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : app_name=market"
+        expected = "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : app_name=market"
         actual = Appstats::Logger.entry_to_s("address_search")
         actual.should == expected
       end
       
       it "should handle contexts (and sort them by symbol)" do
-        expected = "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
+        expected = "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_filter : app_name=Market : server=Live"
         actual = Appstats::Logger.entry_to_s("address_filter", { :server => "Live", :app_name => 'Market' })
         actual.should == expected
       end
 
       it "should handle actions with the delimiter (and change the delimiter)" do
-        expected = "0.19.7 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n"
+        expected = "0.20.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=search-n"
         actual = Appstats::Logger.entry_to_s("address:=search-n")
         actual.should == expected
 
-        expected = "0.19.7 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n"
+        expected = "0.20.0 setup[:::,===,---n] 2010-09-21 23:15:20 action===address::search==--n"
         actual = Appstats::Logger.entry_to_s("address::search==--n")
         actual.should == expected
       end
 
       it "should handle contexts with the delimiter (and change the delimiter)" do
-        expected = "0.19.7 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n"
+        expected = "0.20.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address :: server==market:eval=-n"
         actual = Appstats::Logger.entry_to_s("address", :server => 'market:eval=-n')
         actual.should == expected
       end
 
       it "should ignore spaces" do
-        expected = "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address search"
+        expected = "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address search"
         actual = Appstats::Logger.entry_to_s("address search")
         actual.should == expected
       end
       
       it "should convert newlines in action" do
-        expected = "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_-nsearch"
+        expected = "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_-nsearch"
         actual = Appstats::Logger.entry_to_s("address_\nsearch")
         actual.should == expected
       end
 
       it "should convert newlines in context" do
-        expected = "0.19.7 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : blah=some-nlong-nstatement"
+        expected = "0.20.0 setup[:,=,-n] 2010-09-21 23:15:20 action=address_search : blah=some-nlong-nstatement"
         actual = Appstats::Logger.entry_to_s("address_search",:blah => "some\nlong\nstatement")
         actual.should == expected
       end
       
       it "should convert newlines based on the delimiter" do
-        expected = "0.19.7 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=--nsearch-n"
+        expected = "0.20.0 setup[::,==,--n] 2010-09-21 23:15:20 action==address:=--nsearch-n"
         actual = Appstats::Logger.entry_to_s("address:=\nsearch-n")
         actual.should == expected
       end

data/spec/parser_spec.rb

@@ -8,153 +8,153 @@ module Appstats
     end
      
     describe("#initialize") do
-      
-      it "should set rules to nil" do
-        @parser.raw_rules.should == nil
-        @parser.raw_tokenize.should == nil
-        @parser.repeating.should == false
-        @parser.tokenize_regex == nil
-        @parser.rules.should == []
-        @parser.tokenize.should == []
-        @parser.constants.should == []
-        @parser.constants_no_spaces.should == []
-      end
-      
-      it "should set rules from constructor" do
-        parser = Parser.new(:rules => ":name or :bust", :tokenize => "a bb c", :repeating => true)
-        parser.raw_rules.should == ":name or :bust"
-        parser.raw_tokenize.should == "a bb c"
-        parser.repeating.should == true
-        parser.rules.should == [ { :rule => :name, :stop => :constant }, "OR", { :rule => :bust, :stop => :end} ]
-        parser.tokenize.should == ["\\s+A","\\s+BB","\\s+C"]
-        parser.tokenize_no_spaces.should == ["A","BB","C"]
-        parser.constants.should == ["\\s+OR"]
-        parser.constants_no_spaces.should == ["OR"]
-      end
-    
-      it "should espace tokens as required" do
-        parser = Parser.new(:tokenize => "( ) abc |")
-        parser.tokenize.should == ['\(','\)','\s+ABC','\|']
-        parser.tokenize_regex.should == '\(|\)|\s+ABC|\|'
-        parser.tokenize_no_spaces.should == ['\(','\)','ABC','\|']
-        parser.tokenize_regex_no_spaces.should == '\(|\)|ABC|\|'
-      end
-    
-    end
+       
+       it "should set rules to nil" do
+         @parser.raw_rules.should == nil
+         @parser.raw_tokenize.should == nil
+         @parser.repeating.should == false
+         @parser.tokenize_regex == nil
+         @parser.rules.should == []
+         @parser.tokenize.should == []
+         @parser.constants.should == []
+         @parser.constants_no_spaces.should == []
+       end
+       
+       it "should set rules from constructor" do
+         parser = Parser.new(:rules => ":name or :bust", :tokenize => "a bb c", :repeating => true)
+         parser.raw_rules.should == ":name or :bust"
+         parser.raw_tokenize.should == "a bb c"
+         parser.repeating.should == true
+         parser.rules.should == [ { :rule => :name, :stop => :constant }, "OR", { :rule => :bust, :stop => :end} ]
+         parser.tokenize.should == ["\\s+A(\\s|$)","\\s+BB(\\s|$)","\\s+C(\\s|$)"]
+         parser.tokenize_no_spaces.should == ["A","BB","C"]
+         parser.constants.should == ["\\s+OR(\\s|$)"]
+         parser.constants_no_spaces.should == ["OR"]
+       end
      
+       it "should espace tokens as required" do
+         parser = Parser.new(:tokenize => "( ) abc |")
+         parser.tokenize.should == ['\(','\)','\s+ABC(\s|$)','\|']
+         parser.tokenize_regex.should == '\(|\)|\s+ABC(\s|$)|\|'
+         parser.tokenize_no_spaces.should == ['\(','\)','ABC','\|']
+         parser.tokenize_regex_no_spaces.should == '\(|\)|ABC|\|'
+       end
      
-    describe "#rules" do
-    
-      it "should end on constant if tokens present" do
-        Parser.new(:rules => ":name", :tokenize => ")").rules.should == [ { :rule => :name, :stop => :end } ]
-      end
-    
+     end
       
-      it "should handle one variable" do
-        Parser.new(:rules => ":name").rules.should == [ { :rule => :name, :stop => :end } ]
-      end
-    
-      it "should handle many variables" do
-        Parser.new(:rules => ":name :date").rules.should == [ { :rule => :name, :stop => :space }, { :rule => :date, :stop => :end } ]
-      end
-    
-      it "should deal with colons" do
-        Parser.new(:rules => ":name : :date").rules.should == [ { :rule => :name, :stop => :constant }, ":", { :rule => :date, :stop => :end } ]
-      end
-    
-      it "should deal with constant" do
-        Parser.new(:rules => "blah").rules.should == ["BLAH"]
-      end
-    
-      it "should deal with constant and variables" do
-        Parser.new(:rules => ":name blah :date").rules.should == [ { :rule => :name, :stop => :constant }, "BLAH", { :rule => :date, :stop => :end } ]
-      end
-    
-      it "should deal with multiple constants and variables" do
-        Parser.new(:rules => ":name blah more blah :date").rules.should == [ { :rule => :name, :stop => :constant }, "BLAH", "MORE", "BLAH", { :rule => :date, :stop => :end } ]
-      end
       
-    end
+     describe "#rules" do
      
-    describe "#constants" do
-      
-      it "should be empty if only variables" do
-        Parser.new(:rules => ":name :blah").constants.should == [ ]
-      end
-    
-      it "should track all constants" do
-        Parser.new(:rules => ":name : :date").constants.should == [ ":" ]
-      end
-    
-      it "should upper case constants" do
-        Parser.new(:rules => "blah").constants.should == ["\\s+BLAH"]
-      end
-    
-      it "should deal with multiple constants" do
-        Parser.new(:rules => ":name = :blah and :moreblah").constants.should == ["=","\\s+AND"]
-      end
-    
-    end
-    
-    describe "#constants_no_spaces" do
+       it "should end on constant if tokens present" do
+         Parser.new(:rules => ":name", :tokenize => ")").rules.should == [ { :rule => :name, :stop => :end } ]
+       end
+     
+       
+       it "should handle one variable" do
+         Parser.new(:rules => ":name").rules.should == [ { :rule => :name, :stop => :end } ]
+       end
+     
+       it "should handle many variables" do
+         Parser.new(:rules => ":name :date").rules.should == [ { :rule => :name, :stop => :space }, { :rule => :date, :stop => :end } ]
+       end
+     
+       it "should deal with colons" do
+         Parser.new(:rules => ":name : :date").rules.should == [ { :rule => :name, :stop => :constant }, ":", { :rule => :date, :stop => :end } ]
+       end
+     
+       it "should deal with constant" do
+         Parser.new(:rules => "blah").rules.should == ["BLAH"]
+       end
+     
+       it "should deal with constant and variables" do
+         Parser.new(:rules => ":name blah :date").rules.should == [ { :rule => :name, :stop => :constant }, "BLAH", { :rule => :date, :stop => :end } ]
+       end
+     
+       it "should deal with multiple constants and variables" do
+         Parser.new(:rules => ":name blah more blah :date").rules.should == [ { :rule => :name, :stop => :constant }, "BLAH", "MORE", "BLAH", { :rule => :date, :stop => :end } ]
+       end
+       
+     end
       
-      it "should be empty if only variables" do
-        Parser.new(:rules => ":name :blah").constants_no_spaces.should == [ ]
-      end
-    
-      it "should track all constants" do
-        Parser.new(:rules => ":name : :date").constants_no_spaces.should == [ ":" ]
-      end
-    
-      it "should upper case constants" do
-        Parser.new(:rules => "blah").constants_no_spaces.should == ["BLAH"]
-      end
-    
-      it "should deal with multiple constants" do
-        Parser.new(:rules => ":name = :blah and :moreblah").constants_no_spaces.should == ["=","AND"]
-      end
-    
-    end    
+     describe "#constants" do
+       
+       it "should be empty if only variables" do
+         Parser.new(:rules => ":name :blah").constants.should == [ ]
+       end
      
-    describe "#parse_constant" do
-    
-      it "should handle nil" do
-        Parser.parse_constant(nil,nil).should == [nil,nil]
-        Parser.parse_constant("",nil).should == [nil,nil]
-      end
-    
-      it "should find the constant" do
-        Parser.parse_constant("= blah blah more blah ","=").should == ["=","blah blah more blah"]
-      end
-    
-      it "should find the constants with multiple characters" do
-        Parser.parse_constant("hey blah blah more blah ","hey").should == ["hey","blah blah more blah"]
-      end
-    
-      it "should return nil if not found" do
-        Parser.parse_constant("blah blah more blah ","=").should == [nil,"blah blah more blah"]
-      end
-    
-      it "should be case insensitive" do
-        Parser.parse_constant(" blah stuff on more blah stuff ","blah").should == ["blah","stuff on more blah stuff"]
-        Parser.parse_constant(" BLAH stuff on more blah stuff ","blah").should == ["BLAH","stuff on more blah stuff"]
-        Parser.parse_constant(" blah stuff on more blah stuff ","BLAH").should == ["blah","stuff on more blah stuff"]
-      end
-    
-      it "should only find the first instance" do
-        Parser.parse_constant("one == two","==").should == [nil,"one == two"]
-      end
+       it "should track all constants" do
+         Parser.new(:rules => ":name : :date").constants.should == [ ":" ]
+       end
+     
+       it "should upper case constants" do
+         Parser.new(:rules => "blah").constants.should == ["\\s+BLAH(\\s|$)"]
+       end
+     
+       it "should deal with multiple constants" do
+         Parser.new(:rules => ":name = :blah and :moreblah").constants.should == ["=","\\s+AND(\\s|$)"]
+       end
+     
+     end
+     
+     describe "#constants_no_spaces" do
+       
+       it "should be empty if only variables" do
+         Parser.new(:rules => ":name :blah").constants_no_spaces.should == [ ]
+       end
+     
+       it "should track all constants" do
+         Parser.new(:rules => ":name : :date").constants_no_spaces.should == [ ":" ]
+       end
+     
+       it "should upper case constants" do
+         Parser.new(:rules => "blah").constants_no_spaces.should == ["BLAH"]
+       end
+     
+       it "should deal with multiple constants" do
+         Parser.new(:rules => ":name = :blah and :moreblah").constants_no_spaces.should == ["=","AND"]
+       end
+     
+     end    
       
-    end
+     describe "#parse_constant" do
+     
+       it "should handle nil" do
+         Parser.parse_constant(nil,nil).should == [nil,nil]
+         Parser.parse_constant("",nil).should == [nil,nil]
+       end
+     
+       it "should find the constant" do
+         Parser.parse_constant("= blah blah more blah ","=").should == ["=","blah blah more blah"]
+       end
+     
+       it "should find the constants with multiple characters" do
+         Parser.parse_constant("hey blah blah more blah ","hey").should == ["hey","blah blah more blah"]
+       end
+     
+       it "should return nil if not found" do
+         Parser.parse_constant("blah blah more blah ","=").should == [nil,"blah blah more blah"]
+       end
+     
+       it "should be case insensitive" do
+         Parser.parse_constant(" blah stuff on more blah stuff ","blah").should == ["blah","stuff on more blah stuff"]
+         Parser.parse_constant(" BLAH stuff on more blah stuff ","blah").should == ["BLAH","stuff on more blah stuff"]
+         Parser.parse_constant(" blah stuff on more blah stuff ","BLAH").should == ["blah","stuff on more blah stuff"]
+       end
      
-    describe "#parse_word" do
+       it "should only find the first instance" do
+         Parser.parse_constant("one == two","==").should == [nil,"one == two"]
+       end
+       
+     end
       
-      it "should handle nil" do
-        @parser.parse_word(nil,nil).should == [nil,nil]
-        @parser.parse_word("",nil).should == [nil,nil]
-      end
-    
-      it "should look for global tokens" do
+     describe "#parse_word" do
+       
+       it "should handle nil" do
+         @parser.parse_word(nil,nil).should == [nil,nil]
+         @parser.parse_word("",nil).should == [nil,nil]
+       end
+     
+       it "should look for global tokens" do
         parser = Parser.new(:tokenize => "xx")
         parser.parse_word("blah xx",:end).should == ["blah","xx"]
         parser.parse_word("blah xx stop","stop").should == ["blah","xx stop"]
@@ -259,6 +259,29 @@ module Appstats
       
     end
      
+    describe "#alpha?" do
+      
+      it "should be false for nil" do
+        Parser.alpha?(nil).should == false
+      end
+    
+      it "should be false for empty string" do
+        Parser.alpha?('').should == false
+      end
+    
+      it "should be false for non alpha string" do
+        Parser.alpha?('1abc').should == false
+        Parser.alpha?('abc2').should == false
+        Parser.alpha?('abc2def').should == false
+      end
+    
+      it "should be true for  alpha string" do
+        Parser.alpha?('a').should == true
+        Parser.alpha?('abc').should == true
+      end
+      
+    end 
+     
     describe "#parse" do
     
     
@@ -339,6 +362,12 @@ module Appstats
         parser.raw_results.should == [ "(", { :one => "blaa"}, "aa", "a1", { :two => "bla1"}, ")" ]
       end
     
+      it "should handle alpha constants as requiring spaces" do
+        parser = Appstats::Parser.new(:rules => ":one group :two")
+        parser.parse("group_by group aha")
+        parser.raw_results.should == [  { :one => "group_by"}, "group", { :two => "aha"} ]
+      end
+    
     
       describe "real examples" do
         
@@ -367,6 +396,13 @@ module Appstats
           parser.parse("# buyer-address-lookup last month").should == true
           parser.raw_results.should == [{:operation=>"#"}, {:action=>"buyer-address-lookup"}, {:host=>nil}, {:contexts=>nil}, {:date=>"last month"}]
         end
+       
+        it "should handle group_by" do
+          parser = Appstats::Parser.new(:rules => ":operation :action :date on :host where :contexts group by :group_by")
+          parser.parse("# appstats_queries where action = abc AND contexts = 'def' || group_by like 'hik'").should == true
+          parser.results.should == {:operation => "#", :action => "appstats_queries", :date => nil, :host => nil, :group_by => nil, :contexts => "action = abc AND contexts = 'def' || group_by like 'hik'" }
+        end
+       
         
       end
       

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: appstats
 version: !ruby/object:Gem::Version 
-  hash: 93
+  hash: 79
   prerelease: 
   segments: 
   - 0
-  - 19
-  - 7
-  version: 0.19.7
+  - 20
+  - 0
+  version: 0.20.0
 platform: ruby
 authors: 
 - Andrew Forward
@@ -181,10 +181,13 @@ files:
 - db/migrations/20110311214833_add_appstats_results_db_connection.rb
 - db/migrations/20110318203339_add_appstats_results_latest_flag.rb
 - db/migrations/20110321154125_add_appstats_results_query_duration_in_seconds.rb
+- db/migrations/20110330171745_create_appstats_action_contexts.rb
 - db/schema.rb
 - lib/appstats.rb
 - lib/appstats/action.rb
+- lib/appstats/action_context_key.rb
 - lib/appstats/acts_as_appstatsable.rb
+- lib/appstats/appstats_query.rb
 - lib/appstats/ci.rake
 - lib/appstats/code_injections.rb
 - lib/appstats/context.rb
@@ -227,8 +230,10 @@ files:
 - script/console
 - script/destroy
 - script/generate
+- spec/action_context_key_spec.rb
 - spec/action_spec.rb
 - spec/acts_as_appstatsble_spec.rb
+- spec/appstats_query_spec.rb
 - spec/appstats_spec.rb
 - spec/context_key_spec.rb
 - spec/context_spec.rb
@@ -282,8 +287,10 @@ signing_key:
 specification_version: 3
 summary: Provide usage statistics about how your application is being used
 test_files: 
+- spec/action_context_key_spec.rb
 - spec/action_spec.rb
 - spec/acts_as_appstatsble_spec.rb
+- spec/appstats_query_spec.rb
 - spec/appstats_spec.rb
 - spec/context_key_spec.rb
 - spec/context_spec.rb