appstats 0.10.0 → 0.11.2

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    appstats (0.10.0)
+    appstats (0.11.2)
       daemons
       net-scp
       rails (>= 2.3.0)

data/db/migrations/20110222215437_create_appstats_jobs.rb

@@ -0,0 +1,16 @@
+class CreateAppstatsJobs < ActiveRecord::Migration
+  def self.up
+    create_table :appstats_result_jobs do |t|
+      t.string :name
+      t.string :frequency
+      t.string :status
+      t.text :query
+      t.datetime :last_run_at
+      t.timestamps
+    end
+  end
+
+  def self.down
+    drop_table :appstats_result_jobs
+  end
+end

data/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20110217234136) do
+ActiveRecord::Schema.define(:version => 20110222215437) do
 
   create_table "appstats_actions", :force => true do |t|
     t.string   "name"
@@ -89,6 +89,16 @@ ActiveRecord::Schema.define(:version => 20110217234136) do
 
   add_index "appstats_log_collectors", ["host"], :name => "index_appstats_log_collectors_on_host"
 
+  create_table "appstats_result_jobs", :force => true do |t|
+    t.string   "name"
+    t.string   "frequency"
+    t.string   "status"
+    t.text     "query"
+    t.datetime "last_run_at"
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
   create_table "appstats_results", :force => true do |t|
     t.string   "name"
     t.string   "result_type"

data/lib/appstats.rb

@@ -13,6 +13,7 @@ require "#{File.dirname(__FILE__)}/appstats/log_collector"
 require "#{File.dirname(__FILE__)}/appstats/parser"
 require "#{File.dirname(__FILE__)}/appstats/query"
 require "#{File.dirname(__FILE__)}/appstats/result"
+require "#{File.dirname(__FILE__)}/appstats/result_job"
 require "#{File.dirname(__FILE__)}/appstats/host"
 require "#{File.dirname(__FILE__)}/appstats/context_key"
 require "#{File.dirname(__FILE__)}/appstats/context_value"

data/lib/appstats/date_range.rb

@@ -115,37 +115,44 @@ module Appstats
       end
 
 
-      m = input.match(/^this\s*(year|month|week|day)$/)
+      m = input.match(/^this\s*(year|quarter|month|week|day)$/)
       unless m.nil?
         range.from = EntryDate.parse(input)
-        range.format = m[1] == "week" ? :inclusive : :fixed_point
+        range.format = ["week","quarter"].include?(m[1]) ? :inclusive : :fixed_point
         return range
       end
 
-      m = input.match(/^(last|previous)\s*(year|month|week|day)$/)
+      m = input.match(/^(last|previous)\s*(year|quarter|month|week|day)$/)
       unless m.nil?
         range.from = EntryDate.parse(input)
         if m[2] == "week"
           range.to = range.from.end_of_week
           range.format = :inclusive
+        elsif m[2] == "quarter"
+          range.to = range.from.end_of_quarter
+          range.format = :inclusive
         else
           range.format = :fixed_point  
         end
         return range
       end
 
-      m = input.match(/^last\s*(.+)\s*(year|years|month|months|week|weeks|day|days)$/)
+      m = input.match(/^last\s*(.+)\s*(year|years|quarter|quarters|month|months|week|weeks|day|days)$/)
       unless m.nil?
         range.from = EntryDate.parse(input)
         range.format = :inclusive
         return range
       end
 
-      m = input.match(/^previous\s*(.+)\s*(year|month|week|day)s?$/)
+      m = input.match(/^previous\s*(.+)\s*(year|quarter|month|week|day)s?$/)
       unless m.nil?
         range.from = EntryDate.parse(input)
         to = EntryDate.parse("last #{m[2]}")
-        to = to.end_of_week if m[2] == "week"
+        if m[2] == "week"
+          to = to.end_of_week
+        elsif m[2] == "quarter"
+          to = to.end_of_quarter  
+        end
         range.to = to
         range.format = :inclusive
         return range

data/lib/appstats/entry_date.rb

@@ -22,6 +22,11 @@ module Appstats
       week  
     end
 
+    def end_of_quarter
+      t = to_time.end_of_quarter
+      EntryDate.new(:year => t.year, :month => t.month)
+    end
+
     def to_time(mode = :start)
       return Time.now if @year.nil?
       t = Time.parse("#{@year}-#{@month||'01'}-#{@day||'01'} #{@hour||'00'}:#{@min||'00'}:#{@sec||'00'}")
@@ -79,6 +84,9 @@ module Appstats
         t_parts = [:year,:month,:day]
       elsif input.match(/^this year$/)
         t_parts = [:year]
+      elsif input.match(/^this quarter$/)
+        t = t.beginning_of_quarter
+        t_parts = [:year,:month]
       elsif input.match(/^this month$/)
         t_parts = [:year,:month]
       elsif input.match(/^this week$/)
@@ -99,6 +107,13 @@ module Appstats
         t -= last_date_offset(m).year
         t_parts = [:year]
       end
+
+      m = input.match(/^(last|previous)\s*(\d*)\s*quarters?$/)
+      if m
+        t = t.beginning_of_quarter
+        last_date_offset(m).times { t = (t - 1.day).beginning_of_quarter }
+        t_parts = [:year,:month]
+      end
       
       m = input.match(/^(last|previous)\s*(\d*)\s*months?$/)
       if m

data/lib/appstats/log_collector.rb

@@ -11,6 +11,19 @@ module Appstats
       File.expand_path("#{File.dirname(__FILE__)}/../../log/appstats_remote_log_#{id}.log")
     end
 
+    def processed_filename
+      return filename if filename.nil? || filename == ''
+      m = filename.match(/(.*\/)(.*)/)
+      prefix = "__processed__"
+      return "#{prefix}#{filename}" if m.nil?
+      "#{m[1]}#{prefix}#{m[2]}"
+    end
+
+    def self.should_process(last_time)
+      return true if last_time.nil?
+      Time.now.day > last_time.day
+    end
+
     def self.find_remote_files(remote_login,path,log_template)
       begin
         Appstats.log(:info,"Looking for logs in [#{remote_login[:user]}@#{remote_login[:host]}:#{path}] labelled [#{log_template}]")
@@ -120,7 +133,28 @@ module Appstats
       Appstats.log(:info,"Processed #{count} file(s) with #{total_entries} entr(ies).")
       count
     end
+
+    def self.remove_remote_files(remote_login)
+      all = LogCollector.where("status = 'processed'").all
+      if all.empty?
+        Appstats.log(:info,"No remote logs to remove.")
+        return 0
+      end
+      
+      count = 0
+      Appstats.log(:info,"About to remove #{all.size} remote file(s) from the processing queue.")
+      all.each do |log_collector|
+        Net::SSH.start(remote_login[:host], remote_login[:user], :password => remote_login[:password] ) do |ssh|
+         ssh.exec!("mv #{log_collector.filename} #{log_collector.processed_filename}")
+         Appstats.log(:info,"  - #{remote_login[:user]}@#{remote_login[:host]}:#{log_collector.processed_filename}")
+         log_collector.status = "destroyed"
+         log_collector.save
+         count += 1
+        end
+      end
+      Appstats.log(:info,"Removed #{count} remote file(s).")
+      count
+    end
   
   end
-  
 end

data/lib/appstats/query.rb

@@ -7,9 +7,11 @@ module Appstats
 
     @@nill_query = "select 0 from appstats_entries LIMIT 1"
     @@default = "1=1"
-    attr_accessor :query, :action, :host, :date_range, :query_to_sql, :contexts
+    attr_accessor :name, :query, :action, :host, :date_range, :query_to_sql, :contexts
 
     def initialize(data = {})
+      @name = data[:name]
+      @result_type = data[:result_type] || "on_demand"
       self.query=(data[:query])
     end
     
@@ -19,7 +21,7 @@ module Appstats
     end
     
     def run
-      result = Appstats::Result.new(:result_type => :on_demand, :query => @query, :query_as_sql => @query_to_sql, :action => @action, :host => @host, :from_date => @date_range.from_date, :to_date => @date_range.to_date, :contexts => @contexts)
+      result = Appstats::Result.new(:name => @name, :result_type => @result_type, :query => @query, :query_as_sql => @query_to_sql, :action => @action, :host => @host, :from_date => @date_range.from_date, :to_date => @date_range.to_date, :contexts => @contexts)
       result.count = ActiveRecord::Base.connection.select_one(@query_to_sql)["count(*)"].to_i
       result.save
       result

data/lib/appstats/result_job.rb

@@ -0,0 +1,54 @@
+module Appstats
+  class ResultJob < ActiveRecord::Base
+    set_table_name "appstats_result_jobs"
+
+    attr_accessible :name, :frequency, :status, :query, :last_run_at
+
+    @@frequency_methods = 
+
+    def should_run
+      return true if frequency == "once" && last_run_at.nil?
+      period = { "daily" => :beginning_of_day, "weekly" => :beginning_of_week, "monthly" => :beginning_of_month, "quarterly" => :beginning_of_quarter, "yearly" => :beginning_of_year }[frequency]
+      return false if period.nil?
+      return true if last_run_at.nil?
+      last_run_at.send(period) <= (Time.now.send(period) - 1.day).send(period)
+    end
+
+    def ==(o)
+      o.class == self.class && o.send(:state) == state
+    end
+    alias_method :eql?, :==
+
+    def self.run
+      count = 0
+      all = ResultJob.where("frequency <> 'once' or last_run_at IS NULL").all
+      if all.size == 0
+        Appstats.log(:info, "No result jobs to run.")
+        return count
+      end
+      Appstats.log(:info, "About to analyze #{all.size} result job(s).")
+      all.each do |job|
+        if job.should_run
+          Appstats.log(:info, "  - Job #{job.name} run [ID #{job.id}, FREQUENCY #{job.frequency}, QUERY #{job.query}]")
+          query = Appstats::Query.new(:name => job.name, :result_type => "result_job", :query => job.query)
+          query.run
+          job.last_run_at = Time.now
+          job.save
+          count += 1
+        else
+          Appstats.log(:info, "  - Job #{job.name} NOT run [ID #{job.id}, FREQUENCY #{job.frequency}, QUERY #{job.query}]")
+        end
+      end
+      Appstats.log(:info, "Ran #{count} query(ies).")
+      count
+    end
+
+    private
+
+      def state
+        [name, frequency, status, query, last_run_at]
+      end
+
+    
+  end
+end

data/lib/appstats/version.rb

@@ -1,3 +1,3 @@
 module Appstats
-  VERSION = "0.10.0"
+  VERSION = "0.11.2"
 end

data/lib/daemons/appstats_log_collector.rb

@@ -30,19 +30,27 @@ appstats_config = YAML::load(File.open(options[:config]))
 ActiveRecord::Base.establish_connection(appstats_config['database'])
 require File.join(File.dirname(__FILE__),"..","appstats")
 
+last_processed_at = nil
 Appstats.log(:info,"Started Appstats Log Collector")
 while($running) do
+  unless Appstats::LogCollector.should_process(last_processed_at)
+    an_hour = 60*60
+    sleep an_hour
+    next
+  end
+  last_processed_at = Time.now
   ActiveRecord::Base.connection.reconnect!
   appstats_config["remote_servers"].each do |remote_server|
     Appstats::LogCollector.find_remote_files(remote_server,remote_server[:path],remote_server[:template])
   end
   Appstats::LogCollector.download_remote_files(appstats_config["remote_servers"])
   Appstats::LogCollector.process_local_files
+  Appstats::ResultJob.run
+
   Appstats::Action.update_actions
   Appstats::Host.update_hosts
   Appstats::ContextKey.update_context_keys
   Appstats::ContextValue.update_context_values
+  Appstats::LogCollector.remove_remote_files(appstats_config["remote_servers"])
   ActiveRecord::Base.connection.disconnect!
-  a_day_in_seconds = 60*60*24
-  sleep a_day_in_seconds
 end

data/spec/date_range_spec.rb

@@ -82,12 +82,16 @@ module Appstats
         DateRange.parse("  yesterday  ").should == DateRange.new(:from => EntryDate.new(:year => 2010, :month => 1, :day => 14), :to => nil, :format => :fixed_point )
       end
 
-      describe "this (year|month|week|day)" do
+      describe "this (year|quarter|month|week|day)" do
 
         it "should understand this year" do
           DateRange.parse("  this year  ").should == DateRange.new(:from => EntryDate.new(:year => 2010), :to => nil, :format => :fixed_point )
         end
 
+        it "should understand this quarter" do
+          DateRange.parse("  this quarter  ").should == DateRange.new(:from => EntryDate.new(:year => 2010, :month => 1), :to => nil, :format => :inclusive )
+        end
+
         it "should understand this month" do
           DateRange.parse("  this month  ").should == DateRange.new(:from => EntryDate.new(:year => 2010, :month => 1), :to => nil, :format => :fixed_point )
         end
@@ -102,12 +106,16 @@ module Appstats
                 
       end
 
-      describe "(last|previous) (year|month|week|day)" do
+      describe "(last|previous) (year|quarter|month|week|day)" do
 
         it "should understand last year" do
           DateRange.parse("  last year  ").should == DateRange.new(:from => EntryDate.new(:year => 2009), :to => nil, :format => :fixed_point )
         end
 
+        it "should understand last quarter" do
+          DateRange.parse("  last quarter  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 10), :to => EntryDate.new(:year => 2009, :month => 12), :format => :inclusive )
+        end
+
         it "should understand last month" do
           DateRange.parse("  last month  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 12), :to => nil, :format => :fixed_point )
         end
@@ -138,7 +146,7 @@ module Appstats
                 
       end
 
-      describe "last X (year|month|week|day)s" do
+      describe "last X (year|quarter|month|week|day)s" do
         
         it "should understand last X years" do
           DateRange.parse("  last 1 year  ").should == DateRange.new(:from => EntryDate.new(:year => 2010), :to => nil, :format => :inclusive )
@@ -146,6 +154,12 @@ module Appstats
           DateRange.parse("  last 3 years  ").should == DateRange.new(:from => EntryDate.new(:year => 2008), :to => nil, :format => :inclusive )
         end
 
+        it "should understand last X quarters" do
+          DateRange.parse("  last 1 quarter  ").should == DateRange.new(:from => EntryDate.new(:year => 2010, :month => 1), :to => nil, :format => :inclusive )
+          DateRange.parse("  last 2 quarters  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 10), :to => nil, :format => :inclusive )
+          DateRange.parse("  last 3 quarters  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 7), :to => nil, :format => :inclusive )
+        end
+
         it "should understand last X months" do
           DateRange.parse("  last 1 month  ").should == DateRange.new(:from => EntryDate.new(:year => 2010, :month => 1), :to => nil, :format => :inclusive )
           DateRange.parse("  last 2 months  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 12), :to => nil, :format => :inclusive )
@@ -166,7 +180,7 @@ module Appstats
 
       end
 
-      describe "previous X (year|month|week|day)s" do
+      describe "previous X (year|quarter|month|week|day)s" do
       
         it "should understand previous X years" do
           DateRange.parse("  previous 1 year  ").should == DateRange.new(:from => EntryDate.new(:year => 2009), :to => EntryDate.new(:year => 2009), :format => :inclusive )
@@ -174,6 +188,12 @@ module Appstats
           DateRange.parse("  previous 3 years  ").should == DateRange.new(:from => EntryDate.new(:year => 2007), :to => EntryDate.new(:year => 2009), :format => :inclusive )
         end
 
+        it "should understand previous Y quarters" do
+          DateRange.parse("  previous 1 quarter  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 10), :to => EntryDate.new(:year => 2009, :month => 12), :format => :inclusive )
+          DateRange.parse("  previous 2 quarters  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 7), :to => EntryDate.new(:year => 2009, :month => 12), :format => :inclusive )
+          DateRange.parse("  previous 3 quarters  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 4), :to => EntryDate.new(:year => 2009, :month => 12), :format => :inclusive )
+        end
+
         it "should understand previous Y months" do
           DateRange.parse("  previous 1 month  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 12), :to => EntryDate.new(:year => 2009, :month => 12), :format => :inclusive )
           DateRange.parse("  previous 2 months  ").should == DateRange.new(:from => EntryDate.new(:year => 2009, :month => 11), :to => EntryDate.new(:year => 2009, :month => 12), :format => :inclusive )

data/spec/entry_date_spec.rb

@@ -75,6 +75,15 @@ module Appstats
       end
     end
 
+    describe "#end_of_quarter" do
+      it "should return the same 'level' data points" do
+        now = EntryDate.new(:year => 2010, :month => 1, :day => 5, :hour => 10)
+        expected = EntryDate.new(:year => 2010, :month => 3)
+        now.end_of_quarter.should == expected
+      end
+    end
+
+
     describe "#parse" do
 
       it "should deal with nil" do
@@ -118,6 +127,10 @@ module Appstats
         EntryDate.parse("last year").should == EntryDate.new(:year => 2009)
       end
 
+      it "should understand last quarter" do
+        EntryDate.parse("last quarter").should == EntryDate.new(:year => 2009, :month => 10)
+      end
+
       it "should understand last month" do
         EntryDate.parse("last month").should == EntryDate.new(:year => 2009, :month => 12)
       end
@@ -134,6 +147,10 @@ module Appstats
         EntryDate.parse("previous year").should == EntryDate.new(:year => 2009)
       end
 
+      it "should understand previous quarter" do
+        EntryDate.parse("previous quarter").should == EntryDate.new(:year => 2009, :month => 10)
+      end
+
       it "should understand previous month" do
         EntryDate.parse("previous month").should == EntryDate.new(:year => 2009, :month => 12)
       end
@@ -150,6 +167,10 @@ module Appstats
         EntryDate.parse("this year").should == EntryDate.new(:year => 2010)
       end
 
+      it "should understand this quarter" do
+        EntryDate.parse("this quarter").should == EntryDate.new(:year => 2010, :month => 1)
+      end
+
       it "should understand this month" do
         EntryDate.parse("this month").should == EntryDate.new(:year => 2010, :month => 1)
       end
@@ -168,6 +189,13 @@ module Appstats
         EntryDate.parse("last 3 years").should == EntryDate.new(:year => 2008)
       end
 
+      it "should understand last X quarters" do
+        EntryDate.parse("last 1 quarter").should == EntryDate.new(:year => 2010, :month => 1)
+        EntryDate.parse("last 2 quarters").should == EntryDate.new(:year => 2009, :month => 10)
+        EntryDate.parse("last 3 quarters").should == EntryDate.new(:year => 2009, :month => 7)
+      end
+
+
       it "should understand last X months" do
         EntryDate.parse("last 1 month").should == EntryDate.new(:year => 2010, :month => 1)
         EntryDate.parse("last 2 months").should == EntryDate.new(:year => 2009, :month => 12)
@@ -192,6 +220,12 @@ module Appstats
         EntryDate.parse("previous 3 years").should == EntryDate.new(:year => 2007)
       end
 
+      it "should understand previous X quarters" do
+        EntryDate.parse("previous 1 quarter").should == EntryDate.new(:year => 2009, :month => 10)
+        EntryDate.parse("previous 2 quarters").should == EntryDate.new(:year => 2009, :month => 7)
+        EntryDate.parse("previous 3 quarters").should == EntryDate.new(:year => 2009, :month => 4)
+      end
+
       it "should understand previous X months" do
         EntryDate.parse("previous 1 month").should == EntryDate.new(:year => 2009, :month => 12)
         EntryDate.parse("previous 2 months").should == EntryDate.new(:year => 2009, :month => 11)