merry_go_round 0.0.5 → 0.0.6

data/lib/merry_go_round/aggregator.rb

@@ -43,6 +43,16 @@ module MerryGoRound
 
     # Aggregate existing aggregations (i.e. hour -> day)
     def compound
+      # Get list of known keys
+      keys = Aggregation.select('DISTINCT(key) AS key').collect(&:key)
+
+      # Aggregate each key
+      keys.each do |key|
+        compound_key(key)
+      end
+    end
+
+    def compound_key(key)
       now = Time.now.utc
 
       # Loop through all of the granularities except for the first one
@@ -50,46 +60,60 @@ module MerryGoRound
         # Skip minutes since the Redis aggregator does those
         next if index == 0
 
+        # Get starting point
+        start = start_timestamp(key, granularity).to_i
+
         # Time interval for granularity
+        # TODO: Use `window` instead of `seconds`
         interval = seconds(granularity)
 
         # Get the previous granularity
         prev_gran = MerryGoRound.granularities[index - 1]
 
-        # Get where we left off for this granularity
-        results = Aggregation.select('MAX(timestamp) AS timestamp, key').where(granularity: prev_gran).group(:key).order('timestamp DESC')
-
         # Loop through the results
-        results.each do |result|
-          timestamp = result.timestamp.utc.to_i
-          while timestamp < now.to_i
-            range = window(granularity, Time.at(timestamp).utc)
-            break unless range.first < now and range.last < now
-
-            # Check time interval to see if we have enough data to do this granularity
-            break if Aggregation.where('key = ? AND granularity = ? AND timestamp = ?', result.key, granularity, range.first).first
+        while start < now.to_i
+          range = window(granularity, Time.at(start).utc)
+          break unless range.first < now
 
-            children = Aggregation.where('key = ? AND granularity = ? AND timestamp >= ? AND timestamp <= ?', result.key, prev_gran, range.first, range.last).order('timestamp ASC')
-            break unless children.length > 0
+          # Check time interval to see if we have enough data to do this granularity
+          break if Aggregation.where('key = ? AND granularity = ? AND timestamp = ?', key, granularity, range.first).first
 
-            # Get the aggregated value
-            # TODO: This should probably done in SQL
-            value = children.collect(&:value).reduce(:+)
+          children = Aggregation.where('key = ? AND granularity = ? AND timestamp >= ? AND timestamp <= ?', key, prev_gran, range.first, range.last).order('timestamp ASC')
+          break unless children.length > 0
 
-            # Create the aggregation
-            agg = Aggregation.create(key: result.key, value: value, granularity: granularity, timestamp: range.first)
+          # Get the aggregated value
+          # TODO: This should probably done in SQL
+          value = children.collect(&:value).reduce(:+)
 
-            # Associate the children with their parent
-            children.each do |child|
-              child.parent_id = agg.id
-              child.save
-            end
+          # Create the aggregation
+          agg = Aggregation.create(key: key, value: value, granularity: granularity, timestamp: range.first)
 
-            # Increment the timestamp
-            timestamp += interval
+          # Associate the children with their parent
+          children.each do |child|
+            child.parent_id = agg.id
+            child.save
           end
+
+          # Increment the timestamp
+          start += interval
         end
       end
     end
+
+    def start_timestamp(key, granularity)
+      # Search at this level
+      result = Aggregation.select('MAX(timestamp) AS timestamp').where(key: key, granularity: granularity).order('timestamp DESC').first
+      if result and result.timestamp
+        # TODO: Possibily add 1
+        return result.timestamp
+      end
+
+      # Go down a level
+      prev_gran = MerryGoRound.granularities[MerryGoRound.granularities.index(granularity) - 1]
+      result = Aggregation.select('MIN(timestamp) AS timestamp').where(key: key, granularity: prev_gran).order('timestamp ASC').first
+      return result.timestamp if result and result.timestamp
+
+      nil
+    end
   end
 end

data/lib/merry_go_round/version.rb

@@ -1,3 +1,3 @@
 module MerryGoRound
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
 end

data/test/fixtures/votes.csv

@@ -0,0 +1,179 @@
+key,value,timestamp,granularity
+vote.create,17,2013-02-13 00:01:00,minute
+vote.create,5,2013-02-13 00:02:00,minute
+vote.create,15,2013-02-13 00:03:00,minute
+vote.create,9,2013-02-13 00:04:00,minute
+vote.create,6,2013-02-13 00:05:00,minute
+vote.create,4,2013-02-13 00:06:00,minute
+vote.create,6,2013-02-13 00:07:00,minute
+vote.create,2,2013-02-13 00:08:00,minute
+vote.create,2,2013-02-13 00:09:00,minute
+vote.create,3,2013-02-13 00:10:00,minute
+vote.create,8,2013-02-13 00:11:00,minute
+vote.create,8,2013-02-13 00:12:00,minute
+vote.create,7,2013-02-13 00:13:00,minute
+vote.create,15,2013-02-13 00:14:00,minute
+vote.create,10,2013-02-13 00:15:00,minute
+vote.create,2,2013-02-13 00:16:00,minute
+vote.create,15,2013-02-13 00:17:00,minute
+vote.create,13,2013-02-13 00:18:00,minute
+vote.create,24,2013-02-13 00:19:00,minute
+vote.create,20,2013-02-13 00:20:00,minute
+vote.create,19,2013-02-13 00:21:00,minute
+vote.create,24,2013-02-13 00:22:00,minute
+vote.create,17,2013-02-13 00:23:00,minute
+vote.create,9,2013-02-13 00:24:00,minute
+vote.create,16,2013-02-13 00:25:00,minute
+vote.create,12,2013-02-13 00:26:00,minute
+vote.create,18,2013-02-13 00:27:00,minute
+vote.create,19,2013-02-13 00:28:00,minute
+vote.create,11,2013-02-13 00:29:00,minute
+vote.create,8,2013-02-13 00:30:00,minute
+vote.create,2,2013-02-13 00:31:00,minute
+vote.create,6,2013-02-13 00:32:00,minute
+vote.create,12,2013-02-13 00:33:00,minute
+vote.create,11,2013-02-13 00:34:00,minute
+vote.create,8,2013-02-13 00:35:00,minute
+vote.create,3,2013-02-13 00:36:00,minute
+vote.create,3,2013-02-13 00:37:00,minute
+vote.create,2,2013-02-13 00:38:00,minute
+vote.create,16,2013-02-13 00:39:00,minute
+vote.create,16,2013-02-13 00:40:00,minute
+vote.create,25,2013-02-13 00:41:00,minute
+vote.create,25,2013-02-13 00:42:00,minute
+vote.create,14,2013-02-13 00:43:00,minute
+vote.create,11,2013-02-13 00:44:00,minute
+vote.create,10,2013-02-13 00:45:00,minute
+vote.create,3,2013-02-13 00:46:00,minute
+vote.create,12,2013-02-13 00:47:00,minute
+vote.create,12,2013-02-13 00:48:00,minute
+vote.create,25,2013-02-13 00:49:00,minute
+vote.create,36,2013-02-13 00:50:00,minute
+vote.create,32,2013-02-13 00:51:00,minute
+vote.create,18,2013-02-13 00:52:00,minute
+vote.create,13,2013-02-13 00:53:00,minute
+vote.create,9,2013-02-13 00:54:00,minute
+vote.create,21,2013-02-13 00:55:00,minute
+vote.create,19,2013-02-13 00:56:00,minute
+vote.create,10,2013-02-13 00:57:00,minute
+vote.create,11,2013-02-13 00:58:00,minute
+vote.create,8,2013-02-13 00:59:00,minute
+vote.create,27,2013-02-13 01:00:00,minute
+vote.create,14,2013-02-13 01:01:00,minute
+vote.create,14,2013-02-13 01:02:00,minute
+vote.create,12,2013-02-13 01:03:00,minute
+vote.create,1,2013-02-13 01:04:00,minute
+vote.create,4,2013-02-13 01:05:00,minute
+vote.create,20,2013-02-13 01:06:00,minute
+vote.create,15,2013-02-13 01:07:00,minute
+vote.create,6,2013-02-13 01:08:00,minute
+vote.create,14,2013-02-13 01:09:00,minute
+vote.create,11,2013-02-13 01:10:00,minute
+vote.create,9,2013-02-13 01:11:00,minute
+vote.create,7,2013-02-13 01:12:00,minute
+vote.create,16,2013-02-13 01:13:00,minute
+vote.create,10,2013-02-13 01:14:00,minute
+vote.create,4,2013-02-13 01:15:00,minute
+vote.create,7,2013-02-13 01:16:00,minute
+vote.create,1,2013-02-13 01:17:00,minute
+vote.create,4,2013-02-13 01:18:00,minute
+vote.create,10,2013-02-13 01:19:00,minute
+vote.create,5,2013-02-13 01:20:00,minute
+vote.create,7,2013-02-13 01:21:00,minute
+vote.create,10,2013-02-13 01:22:00,minute
+vote.create,10,2013-02-13 01:23:00,minute
+vote.create,11,2013-02-13 01:24:00,minute
+vote.create,3,2013-02-13 01:25:00,minute
+vote.create,5,2013-02-13 01:26:00,minute
+vote.create,6,2013-02-13 01:27:00,minute
+vote.create,1,2013-02-13 01:28:00,minute
+vote.create,3,2013-02-13 01:29:00,minute
+vote.create,14,2013-02-13 01:30:00,minute
+vote.create,7,2013-02-13 01:31:00,minute
+vote.create,5,2013-02-13 01:32:00,minute
+vote.create,2,2013-02-13 01:33:00,minute
+vote.create,8,2013-02-13 01:34:00,minute
+vote.create,7,2013-02-13 01:35:00,minute
+vote.create,6,2013-02-13 01:37:00,minute
+vote.create,14,2013-02-13 01:38:00,minute
+vote.create,10,2013-02-13 01:39:00,minute
+vote.create,17,2013-02-13 01:40:00,minute
+vote.create,7,2013-02-13 01:41:00,minute
+vote.create,3,2013-02-13 01:42:00,minute
+vote.create,8,2013-02-13 01:43:00,minute
+vote.create,15,2013-02-13 01:44:00,minute
+vote.create,18,2013-02-13 01:45:00,minute
+vote.create,11,2013-02-13 01:46:00,minute
+vote.create,7,2013-02-13 01:47:00,minute
+vote.create,10,2013-02-13 01:48:00,minute
+vote.create,15,2013-02-13 01:49:00,minute
+vote.create,11,2013-02-13 01:50:00,minute
+vote.create,9,2013-02-13 01:51:00,minute
+vote.create,4,2013-02-13 01:52:00,minute
+vote.create,3,2013-02-13 01:53:00,minute
+vote.create,2,2013-02-13 01:54:00,minute
+vote.create,3,2013-02-13 01:55:00,minute
+vote.create,10,2013-02-13 01:56:00,minute
+vote.create,12,2013-02-13 01:57:00,minute
+vote.create,8,2013-02-13 01:58:00,minute
+vote.create,15,2013-02-13 01:59:00,minute
+vote.create,24,2013-02-13 02:00:00,minute
+vote.create,24,2013-02-13 02:01:00,minute
+vote.create,12,2013-02-13 02:02:00,minute
+vote.create,2,2013-02-13 02:02:00,minute
+vote.create,9,2013-02-13 02:03:00,minute
+vote.create,3,2013-02-13 02:04:00,minute
+vote.create,3,2013-02-13 02:05:00,minute
+vote.create,3,2013-02-13 02:06:00,minute
+vote.create,5,2013-02-13 02:07:00,minute
+vote.create,4,2013-02-13 02:08:00,minute
+vote.create,6,2013-02-13 02:09:00,minute
+vote.create,6,2013-02-13 02:10:00,minute
+vote.create,11,2013-02-13 02:11:00,minute
+vote.create,18,2013-02-13 02:12:00,minute
+vote.create,12,2013-02-13 02:13:00,minute
+vote.create,22,2013-02-13 02:14:00,minute
+vote.create,10,2013-02-13 02:15:00,minute
+vote.create,21,2013-02-13 02:16:00,minute
+vote.create,14,2013-02-13 02:17:00,minute
+vote.create,16,2013-02-13 02:18:00,minute
+vote.create,17,2013-02-13 02:19:00,minute
+vote.create,16,2013-02-13 02:20:00,minute
+vote.create,15,2013-02-13 02:21:00,minute
+vote.create,21,2013-02-13 02:22:00,minute
+vote.create,4,2013-02-13 02:23:00,minute
+vote.create,6,2013-02-13 02:24:00,minute
+vote.create,11,2013-02-13 02:25:00,minute
+vote.create,13,2013-02-13 02:26:00,minute
+vote.create,31,2013-02-13 02:27:00,minute
+vote.create,22,2013-02-13 02:28:00,minute
+vote.create,21,2013-02-13 02:29:00,minute
+vote.create,18,2013-02-13 02:30:00,minute
+vote.create,23,2013-02-13 02:31:00,minute
+vote.create,9,2013-02-13 02:32:00,minute
+vote.create,17,2013-02-13 02:33:00,minute
+vote.create,15,2013-02-13 02:34:00,minute
+vote.create,9,2013-02-13 02:35:00,minute
+vote.create,11,2013-02-13 02:36:00,minute
+vote.create,5,2013-02-13 02:37:00,minute
+vote.create,11,2013-02-13 02:38:00,minute
+vote.create,25,2013-02-13 02:39:00,minute
+vote.create,18,2013-02-13 02:40:00,minute
+vote.create,6,2013-02-13 02:41:00,minute
+vote.create,2,2013-02-13 02:42:00,minute
+vote.create,1,2013-02-13 02:43:00,minute
+vote.create,1,2013-02-13 02:44:00,minute
+vote.create,2,2013-02-13 02:45:00,minute
+vote.create,1,2013-02-13 02:47:00,minute
+vote.create,2,2013-02-13 02:48:00,minute
+vote.create,7,2013-02-13 02:49:00,minute
+vote.create,6,2013-02-13 02:50:00,minute
+vote.create,10,2013-02-13 02:51:00,minute
+vote.create,12,2013-02-13 02:52:00,minute
+vote.create,9,2013-02-13 02:53:00,minute
+vote.create,14,2013-02-13 02:54:00,minute
+vote.create,16,2013-02-13 02:55:00,minute
+vote.create,20,2013-02-13 02:56:00,minute
+vote.create,9,2013-02-13 02:57:00,minute
+vote.create,11,2013-02-13 02:58:00,minute
+vote.create,3,2013-02-13 02:59:00,minute

data/test/merry_go_round/aggregator_test.rb

@@ -5,7 +5,7 @@ module MerryGoRound
     def test_aggregating_from_redis
       # Load the fixture. There is 60 minutes of data with 3 keys in each minute
       Timecop.freeze(Time.at(1360695600).utc) do
-        redis_fixture('hour')
+        redis_fixture('minutes')
         MerryGoRound.aggregate!
         MerryGoRound.aggregate!
       end
@@ -16,12 +16,20 @@ module MerryGoRound
       # 1 hour * 3 keys
       assert_equal 3, Aggregation.where(granularity: 'hour').count
 
-      # No days since there is only an hour of data
-      assert_equal 0, Aggregation.where(granularity: 'day').count
-
       # The minute aggregations should belong to an hour one
       refute_nil Aggregation.where(granularity: 'minute').first.parent_id
       assert_equal 'hour', Aggregation.where(granularity: 'minute').first.parent.granularity
     end
+
+    def test_compound_aggregations
+      Timecop.freeze(Time.at(1360792056).utc) do
+        csv_fixture('votes')
+        MerryGoRound.aggregate!
+        assert_equal 3, Aggregation.where(granularity: 'hour').count
+
+        MerryGoRound.aggregate!
+        assert_equal 3, Aggregation.where(granularity: 'hour').count
+      end
+    end
   end
 end

data/test/merry_go_round/query_test.rb

@@ -4,7 +4,7 @@ module MerryGoRound
   class QueryTest < TestCase
     def test_execute
       Timecop.freeze(Time.at(1360695600).utc) do
-        redis_fixture('hour')
+        redis_fixture('minutes')
         MerryGoRound.aggregate!
       end
 

data/test/support/fixtures.rb

@@ -1,11 +1,13 @@
+require 'csv'
+
 module FixtureMacros
-  def fixture(name)
+  def yml_fixture(name)
     path = File.expand_path("../../fixtures/#{name}.yml", __FILE__)
     YAML::load(File.open(path))
   end
 
   def redis_fixture(name)
-    hash = fixture(name)
+    hash = yml_fixture(name)
     redis = MerryGoRound.redis
 
     hash.each do |key, value|
@@ -14,4 +16,11 @@ module FixtureMacros
       end
     end
   end
+
+  def csv_fixture(name)
+    path = File.expand_path("../../fixtures/#{name}.csv", __FILE__)
+    CSV.foreach(path) do |row|
+      MerryGoRound::Aggregation.create(key: row[0], value: row[1], timestamp: row[2], granularity: row[3])
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: merry_go_round
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -120,7 +120,8 @@ files:
 - lib/rails/generators/merry_go_round/install/templates/add_aggregations.rb
 - lib/rails/generators/merry_go_round/install/templates/merry_go_round.rake
 - merry_go_round.gemspec
-- test/fixtures/hour.yml
+- test/fixtures/minutes.yml
+- test/fixtures/votes.csv
 - test/merry_go_round/aggregation_test.rb
 - test/merry_go_round/aggregator_test.rb
 - test/merry_go_round/query_test.rb
@@ -152,7 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3985102410336867870
+      hash: -2084216448482753946
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -160,7 +161,8 @@ signing_key:
 specification_version: 3
 summary: Simple data-warehousing with Redis and PostgreSQL.
 test_files:
-- test/fixtures/hour.yml
+- test/fixtures/minutes.yml
+- test/fixtures/votes.csv
 - test/merry_go_round/aggregation_test.rb
 - test/merry_go_round/aggregator_test.rb
 - test/merry_go_round/query_test.rb