cohortly 0.0.7 → 0.0.8

data/app/controllers/cohortly/reports_controller.rb

@@ -4,13 +4,22 @@ class Cohortly::ReportsController < Cohortly::CohortlyController
     @metric_search = Cohortly::Metric.new(params[:cohortly_metric])
     tags = @metric_search.tags.any? ? @metric_search.tags : nil
     groups = @metric_search.groups    
-    @report_name =  Cohortly::Metric.report_table_name(tags, groups, true)
+      
+    @report_name =  Cohortly::Metric.report_table_name(tags, groups, true)        
     # run this in background would be better
-    Cohortly::Metric.weekly_cohort_chart_for_tag(tags, groups)
+    
+    if Cohortly::Metric.respond_to? :delay
+      Cohortly::Metric.delay.weekly_cohort_chart_for_tag(tags, groups)      
+    else
+      Cohortly::Metric.weekly_cohort_chart_for_tag(tags, groups)
+    end
     @report = Cohortly::Report.new( tags, groups, true )
+    
     respond_to do |format|
       format.html
-      format.js { render :json => @report }
+      format.js {
+        render :json => @report
+      }
     end
   end
 end

data/app/models/cohortly/metric.rb

@@ -12,6 +12,7 @@ module Cohortly
     timestamps!
     
     ensure_index :tags
+    ensure_index :user_start_date
     
     def self.store!(args)
       data = args[4]
@@ -34,14 +35,21 @@ module Cohortly
       collection_name = self.report_table_name(tags, groups, weekly)
       # incremental map_reduce pattern
       meta = Cohortly::ReportMeta.find_or_create_by_collection_name(collection_name)
-      query[:created_at] = { :$gt => meta.last_update_on.utc } if meta.last_update_on
+
+      if meta.last_update_on
+        query[:created_at] = { :$gt => meta.last_update_on.utc }
+        out_collection = { :reduce => meta.store_name }
+      else
+        out_collection = meta.store_name
+      end
+
+      meta.last_update_on = Time.now.utc 
+      meta.save                    
       self.collection.map_reduce(weekly ? self.week_map : self.month_map,
                                  self.reduce,
-                                 { :out => meta.last_update_on ? { :reduce => collection_name } : collection_name,
+                                 { :out => out_collection,
                                    :raw => true,
                                    :query => query})        
-      meta.last_update_on = Time.now.utc
-      meta.save        
     end
     
     def self.cohort_chart_for_tag(tags = nil, groups = nil)

data/app/models/cohortly/report.rb

@@ -1,9 +1,10 @@
 module Cohortly
   class Report
     # this is the reduced collection
-    attr_accessor :collection, :weekly, :key_pattern, :groups, :tags
+    attr_accessor :collection, :weekly, :key_pattern, :groups, :tags, :report_meta
     def initialize( tags = nil, groups = nil, weekly = true )
       self.collection = Cohortly::Metric.report_table_name(tags, groups, weekly)
+      self.report_meta = ReportMeta.find_or_create_by_collection_name(self.collection)
       self.groups = groups
       self.tags = tags
       self.weekly = weekly
@@ -11,7 +12,7 @@ module Cohortly
     end
 
     def data
-      @data ||= (Cohortly::Metric.database[self.collection].find().collect {|x| x}).sort_by {|x| x['_id'] }
+      @data ||= (Cohortly::Metric.database[self.report_meta.store_name].find().collect {|x| x}).sort_by {|x| x['_id'] }
     end
 
     def fix_data_lines
@@ -45,8 +46,8 @@ module Cohortly
     end
 
     def user_count_in_cohort(report_key)
-      params = { :user_start_date => { :$gt => key_to_time(report_key) - self.offset,
-                                       :$lt => (key_to_time(report_key))}}
+      params = { :user_start_date => { :$gt => key_to_time(report_key),
+                                       :$lt => (key_to_time(report_key) + 1.week)}}
       params[:tags] = { :$in => groups } if self.groups
       Cohortly::Metric.collection.distinct(:user_id, params).length
     end
@@ -56,8 +57,9 @@ module Cohortly
       start_time = key_to_time(start_key)
       end_time = key_to_time(end_key)
       cur_time = start_time
-      res = []
-      while(cur_time <= end_time) do
+      res = [start_key]
+      cur_time += self.offset
+      while(cur_time < end_time) do
         res << time_to_key(cur_time)
         cur_time += self.offset
       end
@@ -99,7 +101,12 @@ module Cohortly
     
     def as_json(*args)
       fix_data_lines
-      { :data => data,
+      { :name => report_meta.collection_name,
+        :store_name => report_meta.store_name,
+        :groups => self.groups,
+        :tags   => self.tags,
+        :weekly => self.weekly,
+        :data => data,
         :base_n => base_n 
       }
     end

data/app/models/cohortly/report_meta.rb

@@ -4,5 +4,14 @@ module Cohortly
 
     key :collection_name, String
     key :last_update_on, Time
+
+    def store_name
+      "cohortly_report_#{self.id}"
+    end
+
+    def run
+      args = Metric.report_name_to_args(self.collection_name)
+      Cohortly::Metric.cohort_chart(*args)
+    end
   end
 end

data/app/views/cohortly/metrics/_groups.html.erb

@@ -1,9 +1,14 @@
-<div class="groups">
-  <% Cohortly::TagConfig.all_groups.each do |group| %>
-    <span class"group">
-      <%= check_box_tag 'cohortly_metric[groups][]', group,
-      params[:cohortly_metric] && params[:cohortly_metric][:groups] &&
-      params[:cohortly_metric][:groups].include?(group) %><label><%= group %></label>
-    </span>
+<div class="tags">
+  <% Cohortly::TagConfig.all_groups.sort.each_slice(3) do |groups| %>
+    <div class="tag_group" style="float:left; padding: 10px;">
+      <% groups.sort.each do |group| %>
+        <div class"tag">
+          <%= check_box_tag 'cohortly_metric[groups][]', group,
+          params[:cohortly_metric] && params[:cohortly_metric][:groups] &&
+          params[:cohortly_metric][:groups].include?(group) %><label><%= group %></label>
+        </div>
+      <% end %>
+    </div>
   <% end %>
+  <div style="clear:both;"></div>
 </div>

data/app/views/cohortly/metrics/_metric.html.erb

@@ -4,5 +4,5 @@
   <td><%= metric.created_at.strftime("%m/%d/%Y %H:%M") %></td>
   <td><%= metric.controller %></td>
   <td><%= metric.action %></td>
-  <td><%= metric.tags.join(', ')  %></td>
+  <td><%= (metric.tags - Cohortly::TagConfig.all_groups).sort.join(', ')  %></td>
 </tr>

data/app/views/cohortly/metrics/_tags.html.erb

@@ -1,7 +1,7 @@
 <div class="tags">
-  <% (Cohortly::Metric.collection.distinct(:tags) - Cohortly::TagConfig.all_groups).each_slice(8) do |tags| %>
+  <% (Cohortly::Metric.collection.distinct(:tags) - Cohortly::TagConfig.all_groups).sort.each_slice(8) do |tags| %>
     <div class="tag_group" style="float:left; padding: 10px;">
-      <% tags.each do |tag| %>
+      <% tags.sort.each do |tag| %>
         <div class"tag">
           <%= check_box_tag 'cohortly_metric[tags][]', tag,
           params[:cohortly_metric] && params[:cohortly_metric][:tags] &&

data/app/views/layouts/cohortly/application.html.erb

@@ -5,8 +5,8 @@
   <%= csrf_meta_tag %>
 
   <%= javascript_include_tag 'https://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.min.js' %>
-  <%= javascript_include_tag 'http://cdnjs.cloudflare.com/ajax/libs/underscore.js/1.1.7/underscore-min.js' %>
-  <%= javascript_include_tag 'http://cdnjs.cloudflare.com/ajax/libs/backbone.js/0.5.3/backbone-min.js' %>
+  <%= javascript_include_tag 'vendor/underscore.js' %>
+  <%= javascript_include_tag 'vendor/backbone.js' %>
 
   <script type="text/javascript">
     var Cohortly = {};
@@ -24,14 +24,21 @@
         'click input[type=submit]': 'submitQuery'
       },
       submitQuery: function(e) {
+        $('.result_table', this.el).html('<h1>Loading .......</h1>');
         $.get( '/cohortly/reports.js?' + $(this.el).serialize(), _(function(data) {
           this.model.set(data);
+          this.render();
         }).bind(this), 'json')
         e.preventDefault();
       },
       render: function() {
         $('.result_table', this.el).html(
-           ['<table class="one-column-emphasis">',
+           ['<h3>',
+             this.model.get('groups') ? 'Groups: ' + this.model.get('groups').join(', ') + ' | ':  'Groups: __ | ',
+            '',
+             this.model.get('tags') ? 'Tags: ' + this.model.get('tags').join(', ') :  'Tags: __ ',
+            '</h3>',
+            '<table class="one-column-emphasis">',
             '<colgroup><col class="oce-first"></colgroup>',
             this.render_header(),
             this.render_rows(), 
@@ -43,7 +50,7 @@
           '<thead><tr>',
             '<th>Week</th>',
             '<th>N</th>',
-            _.range(1,13).map(function(x){ return '<th>W' + x + '</th>'; }).join(''),
+            _.range(1,15).map(function(x){ return '<th>W' + x + '</th>'; }).join(''),
           '</tr></thead>'
         ].join('');
       },
@@ -62,7 +69,7 @@
         ].join('');
       },
       render_cells: function(row, base_n) {
-        return _(_(row.value).keys()).sortBy(function(x){return x}).slice(0,12).map(function(key) {
+        return _(_(row.value).keys()).sortBy(function(x){return x}).slice(0,14).map(function(key) {
           var num_users = _(row.value[key]).keys().length;
           var percent = (base_n > 0) ? (num_users / base_n) : 0;
           return [

data/lib/cohortly/version.rb

@@ -1,3 +1,3 @@
 module Cohortly
-  VERSION = "0.0.7"
+  VERSION = "0.0.8"
 end

data/lib/tasks/run_reports.rake

@@ -1,23 +1,29 @@
 namespace :cohortly do
   namespace :run do
     desc "run the reports for all the tags"
-    task :reports => :environment do
-      report_names = Cohortly::Metric.database.collections.select { |c| c.name =~ /^cohortly_report/ }.collect &:name
-      report_names.each do |name|
-        args = Cohortly::Metric.report_name_to_args(name)
-        Cohortly::Metric.cohort_chart(*args)
-        puts name  
-      end
-      (Cohortly::Metric.collection.distinct(:tags) - Cohortly::TagConfig.all_groups).each do |tag|
+    task :reseed_reports => :environment do
+      Cohortly::ReportMeta.delete_all
+      Cohortly::Metric.cohort_chart(nil, nil, true)
+      puts "main report"
+      real_tags = (Cohortly::Metric.collection.distinct(:tags) - Cohortly::TagConfig.all_groups)
+      real_tags.each do |tag|
         Cohortly::Metric.cohort_chart([tag], nil, true)
         puts "tag: #{tag}"
       end
       Cohortly::TagConfig.all_groups.each do |group|
-        Cohortly::TagConfig.all_tags.each do |tag|
+        real_tags.each do |tag|
           puts "tag: #{tag} group: #{group}"          
           Cohortly::Metric.cohort_chart([tag], [group], true)
         end        
       end
     end
+
+    desc "update all existing reports"
+    task :updates => :environment do
+      Cohortly::ReportMeta.all.each do |rep|
+        puts rep.collection_name
+        rep.run
+      end
+    end
   end
 end

data/test/cohortly_test.rb

@@ -66,11 +66,12 @@ class CohortlyTest < ActiveSupport::TestCase
   end
 
   test "one day of data" do
-    payload = { :user_start_date => Time.now,
+    payload = { :user_start_date => Time.now.utc,
                 :user_id         => 5,
                 :user_email => "jordon@example.com",
                 :controller => "session",
-                :action => "login"
+                :action => "login",
+                :created_at => Time.now.utc
                 }
 
     ActiveSupport::Notifications.instrument("cohortly.event", payload)
@@ -81,7 +82,7 @@ class CohortlyTest < ActiveSupport::TestCase
 
   test "weekly" do
     Cohortly::Metric.delete_all
-    setup_weekly_data_to_report_on
+    weekly_data
     Cohortly::Metric.weekly_cohort_chart_for_tag
 
     report = Cohortly::Report.new()
@@ -95,9 +96,9 @@ class CohortlyTest < ActiveSupport::TestCase
     assert_equal report.time_to_key(Time.utc(2011,8)), '2011-31'
     assert_equal report.time_to_key(Time.utc(2011,1)), '2011-00'
     assert_equal report.start_key, report.time_to_key(Time.now.utc - 15.weeks)
-    assert_equal report.period_cohorts.length, 16
+    assert_equal report.period_cohorts.length, 15
 
-    assert_equal report.report_totals, [[16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
+    assert_equal report.report_totals, [
                                         [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
                                         [14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
                                         [13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
@@ -115,19 +116,25 @@ class CohortlyTest < ActiveSupport::TestCase
                                         [1]]
   end
   
-  test "report map reduce" do
-    setup_data_to_report_on
-    Cohortly::Metric.cohort_chart_for_tag
-    assert_equal (Cohortly::Metric.all.collect &:user_id).uniq.length, 136
+  test "absolutely correct wwekly data chart" do
+    Cohortly::Metric.delete_all
+    weekly_data
+    Cohortly::Metric.weekly_cohort_chart_for_tag
 
-    report = Cohortly::Report.new(nil,nil,false)
-    assert_equal report.key_to_time('2011-08'), Time.utc(2011, 8)
-    assert_equal report.time_to_key(Time.utc(2011,8)), '2011-08'
-    assert_equal report.start_key, (Time.now - 15.months).year.to_s + '-0' + (Time.now - 15.months).month.to_s
-    assert_equal report.period_cohorts.length, 16
+    report = Cohortly::Report.new()
+    assert report.weekly
+    
+    time = DateTime.strptime('2011-08', '%Y-%W').utc
+    assert_equal report.key_to_time('2011-08'), time
+    assert_equal report.key_to_time(report.time_to_key(time)), time     
 
-    assert_equal report.report_totals, [[16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
-                                        [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
+    
+    assert_equal report.time_to_key(Time.utc(2011,8)), '2011-31'
+    assert_equal report.time_to_key(Time.utc(2011,1)), '2011-00'
+    assert_equal report.start_key, report.time_to_key(Time.now.utc - 15.weeks)
+    assert_equal report.period_cohorts.length, 15
+
+    assert_equal report.report_totals, [[15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
                                         [14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
                                         [13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
                                         [12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
@@ -142,6 +149,35 @@ class CohortlyTest < ActiveSupport::TestCase
                                         [3, 2, 1],
                                         [2, 1],
                                         [1]]
+  end
+
+  
+  test "report map reduce" do
+    setup_data_to_report_on
+    Cohortly::Metric.cohort_chart_for_tag
+    assert_equal (Cohortly::Metric.all.collect &:user_id).uniq.length, 136
+
+    report = Cohortly::Report.new(nil,nil,false)
+    assert_equal report.key_to_time('2011-08'), Time.utc(2011, 8)
+    assert_equal report.time_to_key(Time.utc(2011,8)), '2011-08'
+    assert_equal report.start_key, (Time.now - 15.months).year.to_s + '-0' + (Time.now - 15.months).month.to_s
+    assert_equal report.period_cohorts.length, 15
+
+    assert_equal report.report_totals, [[16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2],
+                                        [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2],
+                                        [14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2],
+                                        [13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2],
+                                        [12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2],
+                                        [11, 10, 9, 8, 7, 6, 5, 4, 3, 2],
+                                        [10, 9, 8, 7, 6, 5, 4, 3, 2],
+                                        [9, 8, 7, 6, 5, 4, 3, 2],
+                                        [8, 7, 6, 5, 4, 3, 2],
+                                        [7, 6, 5, 4, 3, 2],
+                                        [6, 5, 4, 3, 2],
+                                        [5, 4, 3, 2],
+                                        [4, 3, 2],
+                                        [3, 2],
+                                        [2]]
 
   end
 
@@ -149,11 +185,11 @@ class CohortlyTest < ActiveSupport::TestCase
     setup_weekly_data_to_report_on
     Cohortly::Metric.weekly_cohort_chart_for_tag()
     report = Cohortly::Report.new()
-    start_week = report.start_key
-    start_week_time = report.key_to_time(report.start_key)
-    next_week  = report.time_to_key(start_week_time + 1.week)
+
+    start_week_time = report.key_to_time(report.start_key) 
+    next_week  = report.time_to_key(start_week_time + 1.week)    
     
-    assert_equal report.user_count_in_cohort(start_week), 16
+    assert_equal report.user_count_in_cohort(report.start_key), 16
     assert_equal report.user_count_in_cohort(next_week), 15    
   end
 
@@ -161,8 +197,8 @@ class CohortlyTest < ActiveSupport::TestCase
     setup_weekly_data_to_report_on
     Cohortly::Metric.weekly_cohort_chart_for_tag
     report = Cohortly::Report.new
-    line = report.percent_line(report.start_key)
-    assert_equal line, [16, 100, 94, 88, 81, 75, 69, 63, 56, 50, 44, 38, 31, 25, 19, 13, 6]
+    line = report.percent_line(report.time_to_key(report.key_to_time(report.start_key) + 1.week ))
+    assert_equal line, [15, 100, 93, 87, 80, 73, 67, 60, 53, 47, 40, 33, 27, 20, 13]
   end
   
   test "javascript day of year" do
@@ -172,19 +208,23 @@ class CohortlyTest < ActiveSupport::TestCase
   end
 
   test "javascript week of year" do
+    
     StoredProcedures.store_procedures
     assert_equal Time.now.utc.strftime('%W').to_i, StoredProcedures.execute(:week_of_year, Time.now.utc)
     assert_equal (Time.now.utc + 1.week).strftime('%W').to_i, StoredProcedures.execute(:week_of_year, Time.now.utc + 1.week)
+    week_end_minus_15 = Time.now.end_of_week - 15.hours
+    # these are one second off of eachother bleh :P
     30.times { |x|
-      assert_equal (Time.now.utc + x.days).strftime('%W').to_i, StoredProcedures.execute(:week_of_year, Time.now.utc + x.days)
+      assert_equal (week_end_minus_15 + x.hours + 1.second).utc.strftime('%W').to_i, StoredProcedures.execute(:week_of_year, week_end_minus_15 + x.hours)
     }
   end
 
   test "javascript time to week key" do
     StoredProcedures.store_procedures
     assert_equal Time.now.utc.strftime('%Y-%W'), StoredProcedures.execute(:week_key, Time.now.utc)
+    week_end_minus_15 = Time.now.end_of_week - 15.hours
     30.times { |x|
-      assert_equal (Time.now.utc + x.days).strftime('%Y-%W'), StoredProcedures.execute(:week_key, Time.now.utc + x.days)
+      assert_equal (week_end_minus_15 + x.days + 1.second).strftime('%Y-%W'), StoredProcedures.execute(:week_key, week_end_minus_15 + x.days)
     }
   end
   
@@ -196,6 +236,35 @@ class CohortlyTest < ActiveSupport::TestCase
     assert_equal Metric.report_name_to_args("cohortly_report-tags=rand_1:rand_5-monthly"), [['rand_1', 'rand_5'], nil, false]        
   end
   
+  def weekly_data_generate
+    data = (1..15).to_a.reverse.collect {|x| (1..x).to_a.reverse}
+    user_id_level = 0
+    data.collect do |ds|
+      start_date = ds.first.weeks.ago;
+      user_base = user_id_level * 1000
+      ds.each do |d|
+        occured_on = d.weeks.ago
+        d.times { |user_id| yield user_base + user_id + 1, start_date, occured_on }
+      end
+      user_id_level += 1
+    end
+  end
+
+  def weekly_data
+     payload = {
+        :controller => "session",
+        :action => "login",
+        :add_tags => ['login']
+    }
+    weekly_data_generate do |user_id, started_on, occurred_on|
+      payload[:user_id] = user_id
+      payload[:username] = "user-#{user_id}"      
+      payload[:user_start_date] = started_on
+      payload[:created_at] = occurred_on
+      Cohortly::Metric.store! [nil, nil, nil, nil, payload]       
+    end
+  end
+  
   def setup_data_to_report_on
     payload = { :user_start_date => Time.now,
                 :user_id         => 5,