sir_tracks_alot 0.4.0 → 0.6.2

data/Gemfile

@@ -2,11 +2,12 @@ source :gemcutter
 gem 'logging'
 gem 'twitter'
 gem 'ruport'
-gem 'ohm'
+gem 'ohm', '=0.0.38'
 gem 'redis'
 
 group :development do
 	gem 'rspec'
+	gem 'color'
 	gem 'rspec_hpricot_matchers'
 	gem 'hpricot'
 end

data/Gemfile.lock

@@ -0,0 +1,54 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    color (1.4.1)
+    crack (0.1.8)
+    fastercsv (1.5.3)
+    hashie (0.4.0)
+    hoe (2.6.2)
+      rake (>= 0.8.7)
+      rubyforge (>= 2.0.4)
+    hpricot (0.8.2)
+    httparty (0.6.1)
+      crack (= 0.1.8)
+    json_pure (1.4.6)
+    little-plugger (1.1.2)
+    logging (1.4.3)
+      little-plugger (>= 1.1.2)
+    multi_json (0.0.4)
+    oauth (0.4.3)
+    ohm (0.0.38)
+      redis (>= 1.0.4)
+    pdf-writer (1.1.8)
+      color (>= 1.4.0)
+      transaction-simple (~> 1.3)
+    rake (0.8.7)
+    redis (2.0.10)
+    rspec (1.3.0)
+    rspec_hpricot_matchers (1.0)
+    rubyforge (2.0.4)
+      json_pure (>= 1.1.7)
+    ruport (1.6.3)
+      fastercsv
+      pdf-writer (= 1.1.8)
+    transaction-simple (1.4.0)
+      hoe (>= 1.1.7)
+    twitter (0.9.12)
+      hashie (~> 0.4.0)
+      httparty (~> 0.6.1)
+      multi_json (~> 0.0.4)
+      oauth (~> 0.4.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  color
+  hpricot
+  logging
+  ohm (= 0.0.38)
+  redis
+  rspec
+  rspec_hpricot_matchers
+  ruport
+  twitter

data/README.rdoc

@@ -1,6 +1,34 @@
 = sir_tracks_alot
 
-Description goes here.
+Sir Tracks Alot is a ruby library designed to aid in the tracking, collating and analysis of arbitrary usage data. You can use it to record anything! Then summarize what's been recorded into useful graphs. The primary use case for Sir Tracks Alot was to instrument web usage of Rails applications. 
+
+For example, to that a specific user has viewed a specific page:
+
+	SirTracksAlot.record_activity :owner => 'owner', :actor => '/users/123', :target => '/pages/1'
+
+To include information taken from a web request (such as the user agent), simple include it:
+
+	SirTracksAlot.record_activity :owner => 'owner', :actor => '/users/123', :target => '/pages/1', :request => request
+
+You can also include the notion of action type:
+
+	SirTracksAlot.record_activity :owner => 'owner', :actor => '/users/123', :target => '/pages/1', :action => 'create'
+	
+To investigate what's been stored you can filter activities with a variety of method and any attribute can be queried. 
+
+Direct queries work on strings. For example, to search for all activities by a certain owner:
+
+	Activity.filter(:owner => 'owner')
+	
+Regular expressions can be used:
+
+	Activity.filter(:owner => 'owner', :target => /\/pages\/\d+/)
+
+You can also combine permutations of various options. For example, to find all 'create' and 'view' actions for both '/pages' and '/profiles' targets:
+
+	Activity.filter(:owner => 'owner', :target => ['/pages', '/profiles'], :actions => ['create', 'destroy'])
+	
+
 
 == Note on Patches/Pull Requests
  

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.6.2

data/benchmarks/activity_benchmark.rb

@@ -1,35 +1,34 @@
-require 'rubygems'
-require 'ohm'
-require 'lib/trackable'
+require 'benchmarks/benchmark_helper'
 
-Ohm.connect(:host => 'localhost.com', :timeout => 10)
+@owner = 'owner'
 
-@owner = '/events/md10'
+RedisSpecHelper.reset
+DataBuilder.build(1000)
 
-activities_count = Trackable::Activity.filter(:owner => @owner).size
-events_count = 0; Trackable::Activity.filter(:owner => @owner).each{|a| events_count += a.events.size}
-
-puts "**** Running benchmarks against dataset with #{activities_count} activities and #{events_count} events:"
+Benchmark.bm do |b|    
+  b.report('filtering by owner') do 
+    SirTracksAlot::Activity.filter(:owner => @owner)
+  end  
 
-Benchmark.bmbm do |b|  
-  
-  b.report('grabbing all activities, and iterating') do
-    Trackable::Activity.filter(:owner => @owner).collect{|a| a.target}
+  b.report('filtering by target with regex (all match)') do 
+    SirTracksAlot::Activity.filter(:owner => @owner, :target => /.+/)
   end
   
-  b.report('filter only') do 
-    Trackable::Activity.filter(:owner => @owner)
+  b.report('filtering by target with regex (no matches)') do 
+    SirTracksAlot::Activity.filter(:owner => @owner, :target => /xxxxxxx/)
+  end  
+  
+  b.report('filtering by owner, and iterating') do
+    SirTracksAlot::Activity.filter(:owner => @owner).collect{|a| a.target}
   end
   
-  b.report('filter and retrieve last_event') do 
-    Trackable::Activity.filter(:owner => @owner).map{|a| a.last_event}
+  b.report('filtering and retrieve last_event') do 
+    SirTracksAlot::Activity.filter(:owner => @owner).map{|a| a.last_event}
   end
 
-  b.report('filter and retrieve all events') do 
-    Trackable::Activity.filter(:owner => @owner).map{|a| a.events.map{|e| e}}
+  b.report('filtering and retrieve all events') do 
+    SirTracksAlot::Activity.filter(:owner => @owner).map{|a| a.events.map{|e| e}}
   end
-
-  b.report('count_by daily') do 
-    # counts = Trackable::Activity.count_by(:daily, :owner => @owner)    
-  end  
 end
+
+RedisSpecHelper.reset

data/benchmarks/benchmark_helper.rb

@@ -0,0 +1,30 @@
+require "rubygems"
+require "bundler"
+Bundler.setup(:development)
+Bundler.require(:development)
+require 'lib/sir_tracks_alot'
+require 'benchmark'
+require 'spec/redis_spec_helper'
+
+Ohm.connect(:host => '127.0.0.1', :timeout => 2, :db => 15)
+
+class DataBuilder
+  START_DATE = 1272559940
+  ONE_HOUR = 3600
+  
+  def self.build(count = 1000, owner = 'owner', ref_count = 100)
+    puts "** Building activities..."
+    
+    count.times do |i|
+      event = START_DATE + (ONE_HOUR * i)
+      target = "/targets/#{rand ref_count}"
+      actor = "/actors/#{rand ref_count}"
+      user_agent = "user_agents-#{rand ref_count}"
+      view = 'view'
+      
+      SirTracksAlot.record(:owner => owner, :event => event, :target => target, :actor => actor, :user_agent => user_agent, :action => view)
+    end
+    
+    puts "** Done building #{count} activities."
+  end
+end

data/benchmarks/count_benchmark.rb

@@ -0,0 +1,33 @@
+require 'benchmarks/benchmark_helper'
+
+@owner = 'owner'
+RedisSpecHelper.reset
+DataBuilder.build(1000)
+
+@results = {}
+
+Benchmark.bm do |b|
+  b.report('counting activities') do
+    SirTracksAlot::Count.count({:owner => @owner}, {:limit => 999999}, :daily)
+  end
+
+  b.report('building rows for reports') do
+    SirTracksAlot::Count.rows(:owner => @owner)
+  end
+
+  b.report('filtering counts by owner') do
+    SirTracksAlot::Count.find(:owner => @owner)
+  end
+
+  b.report('filtering counts with regex') do
+    SirTracksAlot::Count.rows(:owner => @owner, :target => /.+/)
+  end
+end
+
+counts_count = SirTracksAlot::Count.all.size
+summaries_count = SirTracksAlot::Summary.all.size
+activities_count = SirTracksAlot::Activity.all.size
+
+puts "*** Generated #{activities_count}, #{counts_count} counts (#{(counts_count+0.1)/(activities_count+0.1)}), #{summaries_count} summaries"
+
+RedisSpecHelper.reset

data/lib/sir_tracks_alot.rb

@@ -1,16 +1,18 @@
 $:.unshift(File.dirname(__FILE__)) unless
   $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
 
-require "rubygems"
-require "bundler"
-Bundler.setup
-Bundler.require 
-
-# require 'logging'
-# require 'twitter'
-# require 'ruport'
-# require 'ohm'
-# require 'redis'
+begin
+  require "rubygems"
+  require "bundler"
+  Bundler.setup
+  Bundler.require
+rescue LoadError
+  require 'logging'
+  require 'twitter'
+  require 'ruport'
+  require 'ohm'
+  require 'redis'  
+end
 
 module SirTracksAlot
   TRACKABLE_ROOT = "#{File.dirname(__FILE__)}/.."
@@ -20,6 +22,7 @@ module SirTracksAlot
   autoload :EventHelper, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot event_helper.rb])
   autoload :FilterHelper, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot filter_helper.rb])
   autoload :Count, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot count.rb])
+  autoload :Summary, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot summary.rb])
   autoload :Clock, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot clock.rb])  
   
   class SirTracksAlotError < StandardError    
@@ -57,9 +60,7 @@ module SirTracksAlot
     
     raise RecordInvalidError.new("Activity not valid: #{activity.errors.inspect}") unless activity.valid?
 
-raise "EVENT NIL" if event.nil?
-
-    activity.update(:last_event => event)
+    activity.update(:last_event => event, :counted => '0')
     activity.events << event
     activity
   end
@@ -94,7 +95,7 @@ raise "EVENT NIL" if event.nil?
     return @log if @log
     
     @log = Logging::Logger[self]
-    @log.level = :debug
+    @log.level = ENV['SIR_TRACKS_ALOT_LOG_LEVEL'] || 'warn'
     @log
   end  
   
@@ -121,6 +122,7 @@ raise "EVENT NIL" if event.nil?
   
   module Reports
     autoload :Report, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot reports report.rb])
+    autoload :SimpleReport, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot reports simple_report.rb])
     autoload :BasicReport, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot reports basic_report.rb])
     autoload :SirTracksAlotReport, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot reports trackable_report.rb])
     autoload :TargetReport, File.join(File.dirname(__FILE__), *%w[sir_tracks_alot reports target_report.rb])

data/lib/sir_tracks_alot/activity.rb

@@ -4,7 +4,6 @@ module SirTracksAlot
     extend FilterHelper 
     
     ACTIONS = [:create, :view, :login, :search, :update, :destroy]    
-    LIMIT = 500
 
     attribute :created_at
     attribute :last_event # Clock.now
@@ -15,7 +14,7 @@ module SirTracksAlot
     attribute :action     # create, view, login, etc.
     attribute :user_agent # IE/Safari Windows/Mac etc.
     attribute :counted    # true/false
-    list      :events     # Clock.now's
+    list      :events    # Clock.now's
 
     index :owner
     index :actor
@@ -36,19 +35,21 @@ module SirTracksAlot
     end
 
     # find recent activities
-    def self.recent(options_for_find, options_for_sort = {:order => 'DESC', :limit => LIMIT})
+    def self.recent(options_for_find, options_for_sort = {:order => 'DESC', :limit => 500})
       find(options_for_find).sort_by(:last_event, options_for_sort)
     end              
 
-    # Delete counted activities, leaving a default of the 500 most recent for the provided search criteria
-    # If left blank, purges all but the most recent Limit
-    def self.purge!(options_for_find = {}, options_for_sort = {:order => 'DESC'})
+    # Delete 500 counted activities, leaving a default of the 500 most recent for the provided search criteria
+    def self.purge!(options_for_find = {}, options_for_sort = {:order => 'DESC', :start => 500, :limit => 500})
       recent(options_for_find.merge(:counted => 1), options_for_sort).each{|a| a.delete}
     end
 
     # set this activity to counted
+    # and clear the recorded events
     def counted!
       update(:counted => '1')
+      events.clear
+      true
     end
 
     # is this activity counted?

data/lib/sir_tracks_alot/clock.rb

@@ -3,5 +3,9 @@ module SirTracksAlot
     def self.now
       Time.now.utc.to_i
     end
+    
+    def self.convert(human)
+      Time.parse(human).to_i
+    end
   end
 end

data/lib/sir_tracks_alot/count.rb

@@ -5,75 +5,123 @@ module SirTracksAlot
     ACTIONS = [:create, :view, :login, :search, :update, :destroy]
     
     attribute :created_at 
-    attribute :date       # 10/01/2010
-    attribute :hour       # 1 - 23
     attribute :owner      # 123123
     attribute :actor      # /users/peter-brown   
     attribute :target     # /discussions/23423
-    attribute :category   
-    attribute :views
-    attribute :visits
+    attribute :category   # /discussions    
+    attribute :action     # create
+    set       :summaries, Summary    
     
-    index :date
-    index :hour
     index :owner
     index :actor
     index :target
     index :category
+    index :action
 
-    def self.count(options)
-      options = OpenStruct.new(options) if options.kind_of?(Hash)
+    def self.total(what, options_for_find)
+      what = [what] unless what.kind_of?(Array)      
+      views, visits = 0, 0
+
+      filter(options_for_find).each do |count|
+        count.summaries.each do |summary|
+          views +=  summary.views.to_i if what.include?(:views)
+          visits += summary.visits.to_i if what.include?(:visits)
+        end
+      end      
+
+      return [views, visits] if what == [:views, :visits]
+      return [visits, views] if what == [:visits, :views]
+      return views if what == [:views]
+      return visits if what == [:visits]
+      raise ArgumentError("what must be one or both of :views, :visits")      
+    end
+    
+    def self.rows(options_for_find, title = nil)
+      groups = {}
       
-      rollup(Activity.filter(:owner => options.owner, 
-          :target => options.target, 
-          :action => options.action||[], 
-          :category => options.category || [], 
-          :counted => '0'), 
-        options.session_duration
-      )
+      filter(options_for_find).each do |count|
+        t = title || count.target || count.actor # don't currently support counts with both actor and target set
+        
+        count.summaries.each do |summary|
+          groups[t] ||= [0,0]
+          groups[t][0] += summary.views.to_i
+          groups[t][1] += summary.visits.to_i
+        end
+      end
+      
+      groups.collect{|g| g.flatten}
+    end
+
+    # Convert a default of 500 activities into counts
+    def self.count(options_for_find = {}, options_for_sort = {:limit => 500}, resolution = :hourly)
+      options_for_find = options_for_find.to_hash
+      session_duration = options_for_find.delete(:session_duration)
+      activities = Activity.find(options_for_find.merge(:counted => 0, :action => [], :category => [])).sort(options_for_sort)
+      rollup(activities, session_duration, resolution)
     end
     
-    def self.rollup(activities, session_duration)
+    # Summarize counts daily or hourly
+    # All times are in UTC
+    def self.summarize(resolution = :daily, options_for_find = {})
+      counts = {}
+
+      filter(options_for_find).each do |count|
+        count.summaries.each do |summary|          
+          key = case resolution
+            when :daily
+              Time.at(summary.date.to_i).utc.strftime('%Y/%m/%d')
+            when :hourly
+              Time.at(summary.date.to_i).utc.strftime('%Y/%m/%d %H:00 UTC')
+            end
+        
+          counts[key]   ||= [0,0]
+          counts[key][0] += summary.views.to_i
+          counts[key][1] += summary.visits.to_i
+        end
+      end
+      
+      counts
+    end
+    
+    # Rollup all activities by resolution
+    # Create one count for actor and one count for target
+    def self.rollup(activities, session_duration, resolution = :hourly)
       counts = []
 
       activities.each do |activity|        
-        activity.views(:hourly).each do |time, views|          
-          date, hour = time.split(' ')
-          counts << create_by_activity({:owner => activity.owner, :category => activity.category, :target => activity.target, :date => date, :hour => hour}, views, 0)
-          counts << create_by_activity({:owner => activity.owner, :category => activity.category, :actor => activity.actor, :date => date, :hour => hour}, views, 0)
+        count = Count.find_or_create(:owner => activity.owner, :category => activity.category,  :target => activity.target, :action => activity.action)
+
+        activity.views(resolution).each do |time, views|
+          date = Time.parse(time).to_i
+          summary = count.summaries.find(:date => date).first || Summary.create(:date => date)
+          count.summaries << summary
+          summary.update(:views => (summary.views||0).to_i + views.to_i)
         end
 
-        activity.visits(session_duration, :hourly).each do |time, visits|
-          date, hour = time.split(' ')
-          counts << create_by_activity({:owner => activity.owner, :category => activity.category, :target => activity.target, :date => date, :hour => hour}, 0, visits)
-          counts << create_by_activity({:owner => activity.owner, :category => activity.category, :actor => activity.actor, :date => date, :hour => hour}, 0, visits)
+        activity.visits(session_duration, resolution).each do |time, visits|
+          date = Time.parse(time).to_i          
+          summary = count.summaries.find(:date => date).first || Summary.create(:date => date)
+          count.summaries << summary
+          summary.update(:visits => (summary.visits||0).to_i + visits.to_i)
         end
 
         activity.counted!        
+        counts << count
       end
       
       counts
     end
     
-    def self.create_by_activity(attributes, views = 0, visits = 0)
-      count = Count.find_or_create(attributes)
-        
-      count.views ||= 0; count.visits ||= 0
+    def to_hash
+      {:owner => owner, :actor => actor, :target => target, :category => category, :id => id}
+    end    
+
 
-      count.views  = count.views.to_i + views 
-      count.visits = count.visits.to_i + visits
-      
-      count.save        
-    end
-    
-            
     private
 
     def validate
       assert_present :owner
-      assert_present :views
-      assert_present :visits
-      assert_unique([:owner, :actor, :target, :category, :date, :hour])
+      assert_unique([:owner, :actor, :target, :category, :action])
     end
     
   end