sir_tracks_alot 0.2.0

data/Gemfile

@@ -0,0 +1,12 @@
+source :gemcutter
+gem 'logging'
+gem 'twitter'
+gem 'ruport'
+gem 'ohm'
+gem 'redis'
+
+group :development do
+	gem 'rspec'
+	gem 'rspec_hpricot_matchers'
+	gem 'hpricot'
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Peter T. Brown
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,17 @@
+= sir_tracks_alot
+
+Description goes here.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Peter T. Brown. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'rake'
+require 'bundler'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "sir_tracks_alot"
+    gem.summary = %Q{A high speed general purpose tracking and reporting tool which uses Redis.}
+    gem.description = %Q{A high speed general purpose tracking and reporting tool which uses Redis.}
+    gem.email = "peter@flippyhead.com"
+    gem.homepage = "http://github.com/flippyhead/sir_tracks_alot"
+    gem.authors = ["Peter T. Brown"]
+    gem.add_bundler_dependencies
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/benchmarks/activity_benchmark.rb

@@ -0,0 +1,35 @@
+require 'rubygems'
+require 'ohm'
+require 'lib/trackable'
+
+Ohm.connect(:host => 'localhost.com', :timeout => 10)
+
+@owner = '/events/md10'
+
+activities_count = Trackable::Activity.filter(:owner => @owner).size
+events_count = 0; Trackable::Activity.filter(:owner => @owner).each{|a| events_count += a.events.size}
+
+puts "**** Running benchmarks against dataset with #{activities_count} activities and #{events_count} events:"
+
+Benchmark.bmbm do |b|  
+  
+  b.report('grabbing all activities, and iterating') do
+    Trackable::Activity.filter(:owner => @owner).collect{|a| a.target}
+  end
+  
+  b.report('filter only') do 
+    Trackable::Activity.filter(:owner => @owner)
+  end
+  
+  b.report('filter and retrieve last_event') do 
+    Trackable::Activity.filter(:owner => @owner).map{|a| a.last_event}
+  end
+
+  b.report('filter and retrieve all events') do 
+    Trackable::Activity.filter(:owner => @owner).map{|a| a.events.map{|e| e}}
+  end
+
+  b.report('count_by daily') do 
+    counts = Trackable::Activity.count_by(:daily, :owner => @owner)    
+  end  
+end

data/benchmarks/report_benchmark.rb

@@ -0,0 +1,21 @@
+require 'rubygems'
+require 'ohm'
+require 'lib/trackable'
+
+Ohm.connect(:host => 'localhost.com', :timeout => 10)
+
+@owner = '/events/md10'
+@categories = Trackable::Activity.filter(:owner => @owner).collect{|a| a.category}.uniq
+@actions = Trackable::Activity.filter(:owner => @owner).collect{|a| a.action}.uniq
+
+activities_count = Trackable::Activity.filter(:owner => @owner).size
+events_count = 0; Trackable::Activity.filter(:owner => @owner).each{|a| events_count += a.events.size}
+
+puts "** Running benchmarks against dataset with #{activities_count} activities, #{events_count} events, and #{@categories.size} categories:"
+
+Benchmark.bmbm do |b|  
+  b.report('targer report') do     
+    report = Trackable::Reports::TargetReport.render_html(:owner => @owner, :roots => @categories, :actions => @actions)
+    # puts report
+  end  
+end

data/lib/sir_tracks_alot/activity.rb

@@ -0,0 +1,139 @@
+module SirTracksAlot
+  class Activity < Persistable
+    ACTIONS = [:create, :view, :login, :search, :update, :destroy]
+    DATE_FORMATS = {:hourly => '%Y/%m/%d %H', :daily => '%Y/%m/%d'}
+    LIMIT = 1000
+    
+    attribute :created_at
+    attribute :last_event # Clock.now
+    attribute :owner      # 123123
+    attribute :actor      # /users/peter-brown   
+    attribute :target     # /discussions/23423
+    attribute :category   # **automatically set**
+    attribute :action     # create, view, login, etc.
+    attribute :user_agent # IE/Safari Windows/Mac etc.
+    
+    index :last_event
+    index :owner
+    index :actor
+    index :target
+    index :category
+    index :action
+    index :user_agent
+    
+    list :events          # Clock.now's
+        
+    # Find activities that match attributes 
+    # Strings are passed to Ohm
+    # Regular Expression filters match against retrieved attribute values
+    # 
+    # filter(:category => 'category', :target => /\/targets\/\d+/)
+    def self.filter(options_for_find, &block)      
+      activities = []
+      sort = options_for_find.delete(:sort) || {}
+      
+      strings = {}
+      matchers = {}
+      
+      options_for_find.each do |key, candidate|
+        matchers[key] = candidate if candidate.kind_of?(Regexp)
+        strings[key]  = candidate if candidate.kind_of?(String)
+      end      
+      
+      all = SirTracksAlot::Activity.find(strings).sort(sort)
+      
+      all.each do |activity|
+        pass = true
+        
+        matchers.each do |key, matcher|
+          pass = false if !matcher.match(activity.send(key))
+        end
+
+        next unless pass
+        
+        yield activity if block_given?
+        
+        activities << activity
+      end
+      
+      activities      
+    end
+      
+    # resolution can be: hourly = %Y/%m/%d %H, daily = %Y/%m/%d or any valid strftime string
+    def self.count_by(resolution, options_for_find = {})      
+      groups = {}
+      
+      filter(options_for_find) do |activity|
+        activity.views(resolution).each do |time, count| 
+          groups[time] ||= 0
+          groups[time] += count
+        end
+      end
+      
+      groups
+    end
+    
+    def self.recent(options_for_find, options_for_sort = {:order => 'DESC', :limit => LIMIT})
+      SirTracksAlot::Activity.find(options_for_find).sort_by(:last_event, options_for_sort)
+    end      
+        
+    def self.count(what, options_for_find)      
+      what = [what] unless what.kind_of?(Array)      
+      views, visits = 0, 0      
+      session_duration = options_for_find.delete(:session_duration)
+      resolution = options_for_find.delete(:resolution)
+      
+      filter(options_for_find) do |activity|
+        views += activity.views(resolution) if what.include?(:views)
+        visits += activity.visits(session_duration) if what.include?(:visits)
+      end      
+            
+      return [views, visits] if what == [:views, :visits]
+      return [visits, views] if what == [:visits, :views]
+      return views if what == [:views]
+      return visits if what == [:visits]
+      raise ArgumentError("what must be one or more of :views, :visits")
+    end        
+        
+    def views(resolution = nil)
+      return events.size if resolution.nil?
+            
+      groups = {}
+      
+      date_format = resolution.kind_of?(String) ? resolution : DATE_FORMATS[resolution]
+
+      events.each do |event|      
+        time = Time.at(event.to_i).utc.strftime(date_format) # lop of some detail
+        groups[time.to_s] ||= 0
+        groups[time.to_s] += 1 
+      end
+      
+      return groups
+    end
+    
+    # Count one visit for all events every session_duration (in seconds)
+    # e.g. 3 visits within 15 minutes counts as one, 1 visit 2 days later counts as another
+    def visits(sd = 1800)
+      session_duration = sd || 1800
+      count = events.size > 0 ? 1 : 0
+      last_event = events.first
+      
+      events.each do |event|
+        count += 1 if (event.to_i - last_event.to_i > session_duration)
+        last_event = event
+      end
+      
+      count
+    end
+        
+    private
+    
+    def validate
+      assert_present :owner
+      assert_present :target
+      assert_present :action
+      assert_unique([:owner, :target, :action, :category, :user_agent, :actor])
+    end
+    
+  end
+end

data/lib/sir_tracks_alot/clock.rb

@@ -0,0 +1,7 @@
+module SirTracksAlot
+  class Clock
+    def self.now
+      Time.now.utc.to_i
+    end
+  end
+end

data/lib/sir_tracks_alot/persistable.rb

@@ -0,0 +1,17 @@
+require 'ohm'
+
+module SirTracksAlot
+  class Persistable < Ohm::Model
+    attribute :created_at
+
+    def self.find_or_create(attributes)
+      models = self.find(attributes)
+      models.empty? ? self.create(attributes.merge(:created_at => Clock.now)) : models.first
+    end
+
+    protected    
+    
+    def validate
+    end        
+  end
+end

data/lib/sir_tracks_alot/queue/queue_helper.rb

@@ -0,0 +1,24 @@
+module SirTracksAlot
+  module Queue
+    class QueueHelper
+      def self.camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true)
+        if first_letter_in_uppercase
+          lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
+        else
+          lower_case_and_underscored_word.first.downcase + camelize(lower_case_and_underscored_word)[1..-1]
+        end
+      end
+
+      def self.constantize(camel_cased_word)
+        names = camel_cased_word.split('::')
+        names.shift if names.empty? || names.first.empty?
+
+        constant = Object
+        names.each do |name|
+          constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+        end
+        constant
+      end
+    end
+  end
+end

data/lib/sir_tracks_alot/queue/report_cache.rb

@@ -0,0 +1,13 @@
+module SirTracksAlot
+  module Queue
+    class ReportCache < Persistable    
+      attribute :created_at
+      attribute :report
+      attribute :owner
+      attribute :html
+    
+      index :report
+      index :owner
+    end
+  end
+end

data/lib/sir_tracks_alot/queue/report_config.rb

@@ -0,0 +1,30 @@
+module SirTracksAlot
+  module Queue
+    class ReportConfig < Persistable
+      SESSION_DURATION = 1800
+      LIMIT = 10000
+    
+      attribute :created_at
+      attribute :report
+      attribute :owner
+      attribute :options_store
+
+      index :report
+      index :owner
+
+      def options
+        self.options = {:limit => LIMIT, :session_duration => SESSION_DURATION} if options_store.nil?
+        Marshal.load(options_store)
+      end
+    
+      def options=(opts)
+        update(:options_store => Marshal.dump(opts))
+      end
+        
+      def validate
+        assert_present :report
+        assert_present :owner      
+      end
+    end
+  end
+end

data/lib/sir_tracks_alot/queue/report_queue.rb

@@ -0,0 +1,59 @@
+module SirTracksAlot
+  module Queue
+    class ReportQueue < Persistable
+      QUEUE_NAME = 'trackable_queue'
+
+      attribute :created_at
+      attribute :name
+      index :name
+      list :queue # of ReportConfigs
+
+      def self.push(owner, report, options)
+        config = ReportConfig.find_or_create(:owner => owner.to_s, :report => report.to_s)
+        config.options = options.merge(:owner => owner.to_s) # reports require owner, should be sync'd with config owner
+
+        queue = self.find_or_create(:name => QUEUE_NAME)
+        queue.queue << config.id
+      end
+
+      def self.pop
+        queue = self.find(:name => QUEUE_NAME)
+        
+        if queue.nil? || queue.first.nil?
+          SirTracksAlot.log.info("Reports queue has not been created yet.")
+          return false
+        end
+        
+        queue = queue.first
+        
+        config = ReportConfig[queue.queue.pop] # raw gets us just the id's, pop to remove the las one, it's a queue!
+      
+        process(config)
+      end
+    
+      def self.process(config)
+        return false if config.nil?
+        
+        SirTracksAlot.log.info("building report: #{config.inspect}")
+        
+        begin
+          report = QueueHelper.constantize(QueueHelper.camelize("SirTracksAlot::Reports::#{config.report.capitalize}"))                
+          html = report.render_html(config.options)
+      
+          cache = ReportCache.find_or_create(:owner => config.owner, :report => config.report)
+          cache.update(:html => html)
+        rescue Exception => e
+          SirTracksAlot.log.fatal("Error building report #{config.report} for #{config.owner}: #{e}")
+        end
+                
+        true
+      end      
+    end
+  end
+end
+
+# TODO: ability to easily run single reports
+# e = Event.find_by_domain 'community.mitcio.com'
+# SirTracksAlot::Queue::ReportConfig.find(:owner => e.to_trackable_id)
+# c = SirTracksAlot::Queue::ReportConfig.find(:owner => e.to_trackable_id).all[-1]
+# SirTracksAlot::Queue::ReportQueue.process(c)

data/lib/sir_tracks_alot/reports/activity_report.rb

@@ -0,0 +1,37 @@
+module SirTracksAlot
+  module Reports
+    class ActivityReport < SirTracksAlotReport            
+      COLUMN_NAMES = ['actor', 'action', 'target', 'event']
+      LIMIT = 10000
+      
+      stage :report      
+      required_option :owner
+      
+      def setup
+        super    
+        options.column_names ||= COLUMN_NAMES
+        
+        activities = Activity.recent({:owner => options.owner}, :order => 'DESC', :limit => (options.limit || LIMIT))        
+        events = activities.collect{|a| [a.actor, a.action, a.target, a.last_event]}.reject{|a| a[2].nil?}
+        
+        table = Table(options.column_names, :data => events)
+        table.sort_rows_by!('event', :order => :descending)
+        
+        self.data = table
+      end  
+                  
+      module Helpers
+        include Report::Helpers        
+      end
+
+
+      class HTML < Ruport::Formatter::HTML
+        renders :html, :for => ActivityReport
+
+        build :report do
+          output << erb((options.template || TRACKABLE_ROOT+"/views/reports/table.html.erb"), :binding => binding)
+        end    
+      end                  
+    end
+  end
+end

data/lib/sir_tracks_alot/reports/actor_activity_report.rb

@@ -0,0 +1,47 @@
+module SirTracksAlot
+  module Reports
+    class ActorActivityReport < SirTracksAlotReport
+      COLUMN_NAMES = ['target', 'page views', 'visits']
+      
+      stage :main
+      required_option :owner, :actor
+      
+      def setup
+        super
+        counts = {}
+        options.column_names ||= COLUMN_NAMES
+        
+        activities = Activity.find(:owner => options.owner, :actor => options.actor)
+        
+        activities.each do |activity|
+          counts[activity.target] ||= [0,0]
+          counts[activity.target][0] += activity.views
+          counts[activity.target][1] += activity.visits(options.session_duration)          
+        end
+
+        table = Table(options.column_names) do |t|
+          counts.each do |target, count|
+            t << [target, count[0], count[1]]
+          end          
+        end
+        
+        table.sort_rows_by!('page views', :order => :descending)
+        
+        self.data = table
+      end  
+                  
+      module Helpers
+        include Report::Helpers        
+      end
+
+
+      class HTML < Ruport::Formatter::HTML
+        renders :html, :for => ActorActivityReport
+
+        build :main do
+          output << erb((options.template || TRACKABLE_ROOT+"/views/reports/table.html.erb"), :binding => binding)
+        end    
+      end                  
+    end
+  end
+end

data/lib/sir_tracks_alot/reports/actor_report.rb

@@ -0,0 +1,54 @@
+module SirTracksAlot
+  module Reports
+    class ActorReport < SirTracksAlotReport                
+      COLUMN_NAMES = ['actor', 'page views', 'visits']
+      
+      stage :actor
+      required_option :owner
+      
+      def setup
+        super        
+        counts = {}        
+        options.column_names ||= COLUMN_NAMES
+        options.actions ||= []
+        options.roots ||= []
+        
+        options.roots.each do |root|
+          options.actions.each do |action|            
+            activities = Activity.find(:owner => options.owner, :action => action, :category => root)
+            
+            activities.each do |activity|
+              counts[activity.actor] ||= [0,0]
+              counts[activity.actor][0] += activity.views
+              counts[activity.actor][1] += activity.visits(options.session_duration)
+            end
+          end
+        end
+
+        table = Table(options.column_names) do |t|
+          counts.each do |target, count|
+            t << [target, count[0], count[1]]
+          end
+        end
+        
+        table.sort_rows_by!('page views', :order => :descending)
+        
+        self.data = table
+
+      end  
+                  
+      module Helpers
+        include Report::Helpers        
+      end
+
+
+      class HTML < Ruport::Formatter::HTML
+        renders :html, :for => ActorReport
+
+        build :actor do
+          output << erb((options.template || TRACKABLE_ROOT+"/views/reports/table.html.erb"), :binding => binding)
+        end    
+      end                  
+    end
+  end
+end

data/lib/sir_tracks_alot/reports/basic_report.rb

@@ -0,0 +1,30 @@
+module SirTracksAlot
+  module Reports
+    class BasicReport < Report
+      COLUMN_NAMES = ['group', 'title', 'count']
+      stage :basic
+      
+      def setup      
+        options.column_names ||= COLUMN_NAMES          
+        
+        table = Table(options.column_names, :data => options.data)
+        grouping = Grouping(table, :by => "group", :order => 'group')
+        grouping.each{|n, g| g.sort_rows_by!('count', :order => :descending)}
+
+        self.data = grouping
+      end  
+                  
+      module Helpers
+        include Report::Helpers
+      end
+
+      class HTML < Ruport::Formatter::HTML
+        renders :html, :for => BasicReport        
+
+        build :basic do
+          output << erb((options.template || TRACKABLE_ROOT+"/views/reports/group.html.erb"), :binding => binding)
+        end    
+      end                  
+    end
+  end
+end

data/lib/sir_tracks_alot/reports/filter_report.rb

@@ -0,0 +1,55 @@
+module SirTracksAlot
+  module Reports
+    class FilterReport < SirTracksAlotReport      
+      COLUMN_NAMES = ['title', 'page views', 'visits']
+      
+      stage :base      
+      required_option :owner      
+      
+      # Build up reports by filtering things. Filters are applied and assigned a row title.
+      #
+      # SirTracksAlot::Reports::FilterReport.render_html(
+      #   :actions => ['view', 'search']
+      # 
+      # :filters = {
+      #   'Title' => {:category => 'category', :target => /\/categories/}
+      # }
+      #      
+      # :filters => {
+      #   :only => {Profile Pages' => {:target => /^\/user_profiles\/.+$/}}
+      #   :except => {Profile Index' => {:target => /^\/user_profiles$/}}
+      # })      
+      def setup
+        super
+        
+        counts = []
+        options.filters ||= {}
+        options.column_names ||= COLUMN_NAMES
+                
+        options.filters.each do |title, options_for_find|          
+          views, visits = Activity.count([:views, :visits], options_for_find.merge(:owner => options.owner))          
+          counts << [title, views, visits] if views > 0
+        end
+        
+        
+        table = Table(options.column_names, :data => counts)
+
+        table.sort_rows_by!('page views', :order => :descending)
+        
+        self.data = table
+      end
+            
+      module Helpers
+        include Report::Helpers
+      end
+
+      class HTML < Ruport::Formatter::HTML    
+        renders :html, :for => FilterReport
+
+        build :base do
+          output << erb(options.template || TRACKABLE_ROOT+"/views/reports/table.html.erb", :binding => binding)
+        end    
+      end      
+    end
+  end
+end