merry_go_round 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +1 -0
- data/Gemfile +6 -0
- data/Readme.markdown +25 -3
- data/lib/merry_go_round.rb +16 -6
- data/lib/merry_go_round/aggregation.rb +11 -4
- data/lib/merry_go_round/aggregator.rb +49 -21
- data/lib/merry_go_round/core_ext/time.rb +32 -0
- data/lib/merry_go_round/query.rb +71 -0
- data/lib/merry_go_round/utils.rb +23 -0
- data/lib/merry_go_round/version.rb +1 -1
- data/lib/merry_go_round/web.rb +36 -0
- data/lib/rails/generators/merry_go_round/install/templates/add_aggregations.rb +1 -1
- data/merry_go_round.gemspec +2 -1
- data/test/fixtures/hour.yml +240 -0
- data/test/merry_go_round/aggregation_test.rb +17 -0
- data/test/merry_go_round/aggregator_test.rb +26 -0
- data/test/merry_go_round/query_test.rb +71 -0
- data/test/merry_go_round/utils_test.rb +7 -0
- data/test/merry_go_round/web_test.rb +15 -0
- data/test/schema.rb +14 -0
- data/test/support/active_record.rb +2 -0
- data/test/support/fixtures.rb +17 -0
- data/test/test_helper.rb +6 -0
- metadata +43 -7
- data/bin/merry +0 -7
data/.gitignore
CHANGED
data/Gemfile
CHANGED
data/Readme.markdown
CHANGED
@@ -26,27 +26,49 @@ Or install it yourself as:
|
|
26
26
|
|
27
27
|
## Usage
|
28
28
|
|
29
|
-
You'll need to add the migrations to your app. Simply run:
|
29
|
+
You'll need to add the migrations and rake tasks to your app. Simply run:
|
30
30
|
|
31
31
|
$ rails generate merry_go_round:install
|
32
32
|
|
33
|
+
### Configuring
|
34
|
+
|
35
|
+
All of the configuration is optional.
|
36
|
+
|
33
37
|
Merry Go Round will automatically detect most Redis servers. If you need to configure it, add an initializer with:
|
34
38
|
|
35
39
|
``` ruby
|
36
40
|
MerryGoRound.configure do |config|
|
41
|
+
# You can set a hash with a url and a custom namespace
|
37
42
|
config.redis = { url: 'redis://redis.example.com:7372/12', namespace: 'mynamespace' }
|
43
|
+
|
44
|
+
# You can also set an existing redis client or a new one
|
45
|
+
config.redis = Redis.new
|
38
46
|
end
|
39
47
|
```
|
40
48
|
|
49
|
+
By default, it will namespace to `merry_go_round` using [redis-namespace](https://github.com/defunkt/redis-namespace).
|
50
|
+
|
41
51
|
### Aggregating
|
42
52
|
|
43
53
|
To run the aggregator, run:
|
44
54
|
|
45
55
|
$ rake merry_go_round:aggregate
|
46
56
|
|
47
|
-
|
57
|
+
It will aggregate data into minute, hour, day, week, month, quarter and year buckets. All you need to do is run the command and it will automatically aggregate for each time period. If that time period is not complete, it will not aggregate them (i.e. if you have been running Merry Go Round for 5 days, it won't generate week and higher aggregations).
|
58
|
+
|
59
|
+
It is recommened to run the aggregator via cron (or [Heroku Scheduler](https://devcenter.heroku.com/articles/scheduler)) every 5 minutes. Since it uses ActiveRecord to store the aggregations, it will have to load your environment which can take a bit. Running the aggregator more than once at the same time is currently not supported and will potentially do bad things.
|
60
|
+
|
61
|
+
### Querying
|
62
|
+
|
63
|
+
Once you have some data in Merry Go Round, it's easy to get it out.
|
64
|
+
|
65
|
+
``` ruby
|
66
|
+
MerryGoRound::Aggregation.for_key('users', :day)
|
67
|
+
```
|
68
|
+
|
69
|
+
This will return an array of `MerryGoRound::Aggregation` objects for every day in this month. You can pass other options to customize the fetching. To get just the values, add `.collect(&:value)` to the end. It's usefull to work with the aggregation objects since they have a timestamp on them.
|
48
70
|
|
49
|
-
It
|
71
|
+
It's worth noting, they have a `children` method that will return the next level down. For example, calling `children` on an object returned from the above call will return 24 objects that have `:hour` as their granularity. You can further drill down all the way to minute.
|
50
72
|
|
51
73
|
## Contributing
|
52
74
|
|
data/lib/merry_go_round.rb
CHANGED
@@ -1,13 +1,9 @@
|
|
1
|
-
require 'merry_go_round/version'
|
2
|
-
require 'merry_go_round/aggregator'
|
3
|
-
require 'merry_go_round/entry'
|
4
|
-
require 'merry_go_round/aggregation'
|
5
|
-
require 'merry_go_round/railtie' if defined? Rails
|
6
|
-
|
7
1
|
require 'redis'
|
8
2
|
require 'redis/namespace'
|
9
3
|
|
10
4
|
module MerryGoRound
|
5
|
+
GRANULARITIES = [:minute, :hour, :day, :week, :year].freeze
|
6
|
+
|
11
7
|
def self.configure
|
12
8
|
yield self
|
13
9
|
end
|
@@ -43,6 +39,10 @@ module MerryGoRound
|
|
43
39
|
@@redis = Redis::Namespace.new(namespace, redis: client)
|
44
40
|
end
|
45
41
|
|
42
|
+
def self.granularities
|
43
|
+
GRANULARITIES
|
44
|
+
end
|
45
|
+
|
46
46
|
private
|
47
47
|
|
48
48
|
def self.determine_redis_provider
|
@@ -51,3 +51,13 @@ private
|
|
51
51
|
ENV[provider]
|
52
52
|
end
|
53
53
|
end
|
54
|
+
|
55
|
+
require 'merry_go_round/version'
|
56
|
+
require 'merry_go_round/utils'
|
57
|
+
require 'merry_go_round/aggregator'
|
58
|
+
require 'merry_go_round/entry'
|
59
|
+
require 'merry_go_round/aggregation'
|
60
|
+
require 'merry_go_round/query'
|
61
|
+
require 'merry_go_round/web'
|
62
|
+
require 'merry_go_round/railtie' if defined? Rails
|
63
|
+
|
@@ -4,19 +4,26 @@ module MerryGoRound
|
|
4
4
|
# Aggregation is the ActiveRecord store for aggregated data points. You should
|
5
5
|
# never create aggregations directly unless you know what you're doing.
|
6
6
|
class Aggregation < ActiveRecord::Base
|
7
|
-
GRANULARITIES = [:minute, :hour, :day, :week, :year]
|
8
|
-
|
9
7
|
attr_accessible :key, :value, :timestamp, :granularity, :parent_id
|
10
8
|
|
11
9
|
belongs_to :parent, class_name: 'MerryGoRound::Aggregation'
|
12
10
|
has_many :children, class_name: 'MerryGoRound::Aggregation', foreign_key: :parent_id
|
13
11
|
|
12
|
+
validates :key, :value, :timestamp, :granularity, presence: true
|
13
|
+
validate :valid_granularity
|
14
|
+
|
14
15
|
def self.table_name
|
15
16
|
'merry_go_round_aggregations'
|
16
17
|
end
|
17
18
|
|
18
|
-
def self.for_key(key, granularity =
|
19
|
-
|
19
|
+
def self.for_key(key, granularity = MerryGoRound.granularities.first)
|
20
|
+
where('key = ? AND granularity = ?', key, granularity)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def valid_granularity
|
26
|
+
errors.add(:message, 'is not a valid granularity') unless MerryGoRound.granularities.include?(self.granularity.to_sym)
|
20
27
|
end
|
21
28
|
end
|
22
29
|
end
|
@@ -1,8 +1,10 @@
|
|
1
1
|
module MerryGoRound
|
2
2
|
class Aggregator
|
3
|
+
include Utils
|
4
|
+
|
3
5
|
def aggregate!
|
4
6
|
from_redis
|
5
|
-
|
7
|
+
compound
|
6
8
|
end
|
7
9
|
|
8
10
|
private
|
@@ -14,7 +16,7 @@ module MerryGoRound
|
|
14
16
|
# Aggregate stuff since the last aggregation in Redis
|
15
17
|
def from_redis
|
16
18
|
# Aggregate to the first granularity
|
17
|
-
redis_granularity =
|
19
|
+
redis_granularity = MerryGoRound.granularities.first
|
18
20
|
|
19
21
|
# Get entry keys from Redis
|
20
22
|
redis_keys = redis.keys 'entry-*'
|
@@ -22,7 +24,7 @@ module MerryGoRound
|
|
22
24
|
# Loop through the Redis keys
|
23
25
|
redis_keys.each do |redis_key|
|
24
26
|
# Extract the timestamp
|
25
|
-
timestamp = Time.
|
27
|
+
timestamp = Time.at(redis_key.sub('entry-', '').to_i).utc
|
26
28
|
|
27
29
|
# TODO: Make sure this timestamp hasn't already been aggregated
|
28
30
|
|
@@ -39,28 +41,54 @@ module MerryGoRound
|
|
39
41
|
end
|
40
42
|
end
|
41
43
|
|
42
|
-
# Aggregate existing
|
43
|
-
def
|
44
|
+
# Aggregate existing aggregations (i.e. hour -> day)
|
45
|
+
def compound
|
46
|
+
now = Time.now.utc.to_i
|
47
|
+
|
44
48
|
# Loop through all of the granularities except for the first one
|
45
|
-
|
46
|
-
|
49
|
+
MerryGoRound.granularities.each_with_index do |granularity, index|
|
50
|
+
next if index == 0
|
51
|
+
|
52
|
+
# TODO: Check time interval to see if we have enough data to do this granularity
|
53
|
+
|
54
|
+
# Time interval for granularity
|
55
|
+
interval = seconds(granularity)
|
56
|
+
|
47
57
|
# Get the previous granularity
|
48
|
-
prev_gran =
|
58
|
+
prev_gran = MerryGoRound.granularities[index - 1]
|
49
59
|
|
50
|
-
# Get
|
51
|
-
|
52
|
-
end
|
53
|
-
end
|
60
|
+
# Get where we left off for this granularity
|
61
|
+
results = Aggregation.select('MAX(timestamp) AS timestamp, key').where(granularity: prev_gran).group(:key).order('timestamp DESC')
|
54
62
|
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
63
|
+
# Loop through the results
|
64
|
+
results.each do |result|
|
65
|
+
timestamp = result.timestamp.to_i
|
66
|
+
while timestamp < now
|
67
|
+
range = window(granularity, Time.at(timestamp))
|
68
|
+
break unless range.last < Time.now
|
69
|
+
|
70
|
+
children = Aggregation.where('key = ? AND granularity = ? AND timestamp >= ? AND timestamp <= ?', result.key, prev_gran, range.first, range.last).order('timestamp ASC')
|
71
|
+
break unless children.length > 0
|
72
|
+
|
73
|
+
# Get the aggregated value
|
74
|
+
# TODO: This should probably done in SQL
|
75
|
+
value = children.collect(&:value).reduce(:+)
|
76
|
+
|
77
|
+
# Create the aggregation
|
78
|
+
agg = Aggregation.create(key: result.key, value: value, granularity: granularity, timestamp: range.first)
|
79
|
+
# puts agg.inspect
|
80
|
+
|
81
|
+
# Associate the children with their parent
|
82
|
+
children.each do |child|
|
83
|
+
child.parent_id = agg.id
|
84
|
+
child.save
|
85
|
+
end
|
86
|
+
|
87
|
+
# Increment the timestamp
|
88
|
+
timestamp += interval
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
64
92
|
end
|
65
93
|
end
|
66
94
|
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
class Time
|
2
|
+
def beginning_of_minute
|
3
|
+
change(sec: 0)
|
4
|
+
end
|
5
|
+
|
6
|
+
def end_of_minute
|
7
|
+
change(
|
8
|
+
sec: 59,
|
9
|
+
usec: Rational(999999999, 1000)
|
10
|
+
)
|
11
|
+
end
|
12
|
+
|
13
|
+
def beginning_of_hour
|
14
|
+
change(min: 0)
|
15
|
+
end
|
16
|
+
|
17
|
+
def end_of_hour
|
18
|
+
change(
|
19
|
+
:min => 59,
|
20
|
+
:sec => 59,
|
21
|
+
:usec => Rational(999999999, 1000)
|
22
|
+
)
|
23
|
+
end
|
24
|
+
|
25
|
+
def all_minute
|
26
|
+
beginning_of_minute..end_of_minute
|
27
|
+
end
|
28
|
+
|
29
|
+
def all_hour
|
30
|
+
beginning_of_hour..end_of_hour
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
module MerryGoRound
|
2
|
+
class Query
|
3
|
+
include Utils
|
4
|
+
|
5
|
+
attr_accessor :keys, :granularity, :min, :max
|
6
|
+
|
7
|
+
def initialize(params = {})
|
8
|
+
self.granularity = params[:granularity] || :hour
|
9
|
+
self.keys = [params[:keys]].flatten.compact
|
10
|
+
parse_max(params)
|
11
|
+
parse_min(params)
|
12
|
+
end
|
13
|
+
|
14
|
+
def execute
|
15
|
+
query = MerryGoRound::Aggregation
|
16
|
+
if keys && keys.length > 0
|
17
|
+
query = query.where('key in (?)', keys)
|
18
|
+
end
|
19
|
+
query = query.where('granularity = ? and timestamp >= ? and timestamp <= ?', granularity, min, max)
|
20
|
+
query = query.order('key, timestamp')
|
21
|
+
|
22
|
+
records = query.all
|
23
|
+
keys = records.map(&:key).uniq
|
24
|
+
|
25
|
+
results = keys.inject({}) do |out, current_key|
|
26
|
+
out[current_key] = records.select{|record| record.key == current_key }.map do |item|
|
27
|
+
{
|
28
|
+
id: item.id,
|
29
|
+
parent_id: item.parent_id,
|
30
|
+
timestamp: item.timestamp,
|
31
|
+
value: item.value,
|
32
|
+
granularity: item.granularity
|
33
|
+
}
|
34
|
+
end
|
35
|
+
|
36
|
+
out
|
37
|
+
end
|
38
|
+
|
39
|
+
{
|
40
|
+
results: results,
|
41
|
+
query: self.as_json
|
42
|
+
}
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def parse_max(params)
|
48
|
+
max = Time.at((params[:max] || Time.now).to_i).utc
|
49
|
+
# remove seconds
|
50
|
+
self.max = max - max.sec
|
51
|
+
end
|
52
|
+
|
53
|
+
def parse_min(params)
|
54
|
+
if params[:min]
|
55
|
+
min = Time.at(params[:min].to_i).utc
|
56
|
+
else
|
57
|
+
current_granularity = MerryGoRound.granularities.index(granularity)
|
58
|
+
|
59
|
+
unless less_granularity = MerryGoRound.granularities[current_granularity + 1]
|
60
|
+
less_granularity = query.granularity
|
61
|
+
end
|
62
|
+
|
63
|
+
min = Time.now.utc - seconds(less_granularity)
|
64
|
+
end
|
65
|
+
|
66
|
+
# remove seconds
|
67
|
+
self.min = min - min.sec
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'active_support/time'
|
2
|
+
require 'merry_go_round/core_ext/time'
|
3
|
+
|
4
|
+
module MerryGoRound
|
5
|
+
module Utils
|
6
|
+
module_function
|
7
|
+
|
8
|
+
def seconds(granularity)
|
9
|
+
map = {
|
10
|
+
minute: 60,
|
11
|
+
hour: 3600,
|
12
|
+
day: 86400,
|
13
|
+
week: 604800,
|
14
|
+
year: 31536000
|
15
|
+
}
|
16
|
+
map[granularity]
|
17
|
+
end
|
18
|
+
|
19
|
+
def window(granularity, timestamp)
|
20
|
+
timestamp.utc.send(:"all_#{granularity.to_s}")
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'sinatra'
|
2
|
+
require 'multi_json'
|
3
|
+
require 'merry_go_round/aggregation'
|
4
|
+
|
5
|
+
class MerryGoRound::Web < Sinatra::Application
|
6
|
+
|
7
|
+
before do
|
8
|
+
content_type 'application/json'
|
9
|
+
end
|
10
|
+
|
11
|
+
# Homepage
|
12
|
+
get '/config' do
|
13
|
+
response_hash = {
|
14
|
+
}
|
15
|
+
body { MultiJson.dump(response_hash) }
|
16
|
+
end
|
17
|
+
|
18
|
+
get '/query' do
|
19
|
+
results = MerryGoRound::Query.new(params).execute
|
20
|
+
body { MultiJson.dump(results)}
|
21
|
+
end
|
22
|
+
|
23
|
+
protected
|
24
|
+
|
25
|
+
def render_error(message, status = 500)
|
26
|
+
|
27
|
+
response_hash = {
|
28
|
+
error: {
|
29
|
+
status: status,
|
30
|
+
message: message
|
31
|
+
}
|
32
|
+
}
|
33
|
+
|
34
|
+
halt status, MultiJson.dump(response_hash)
|
35
|
+
end
|
36
|
+
end
|
@@ -17,7 +17,7 @@ class AddMerryGoRoundAggregations < ActiveRecord::Migration
|
|
17
17
|
t.integer :parent_id
|
18
18
|
end
|
19
19
|
|
20
|
-
add_index :merry_go_round_aggregations, [:
|
20
|
+
add_index :merry_go_round_aggregations, [:granularity, :timestamp, :key, :parent_id], name: 'merry_go_round_aggregations_kgtp'
|
21
21
|
add_index :merry_go_round_aggregations, [:parent_id, :granularity, :timestamp], name: 'merry_go_round_aggregations_pgt'
|
22
22
|
end
|
23
23
|
end
|
data/merry_go_round.gemspec
CHANGED
@@ -20,7 +20,8 @@ Gem::Specification.new do |gem|
|
|
20
20
|
|
21
21
|
gem.required_ruby_version = '>= 1.9.2'
|
22
22
|
gem.add_dependency 'activerecord', '>= 3.0.0'
|
23
|
+
gem.add_dependency 'activesupport', '>= 3.0.0'
|
23
24
|
gem.add_dependency 'redis', '>= 3.0.0'
|
24
25
|
gem.add_dependency 'redis-namespace'
|
25
|
-
gem.add_dependency '
|
26
|
+
gem.add_dependency 'sinatra'
|
26
27
|
end
|
@@ -0,0 +1,240 @@
|
|
1
|
+
entry-1360692000:
|
2
|
+
user: 167
|
3
|
+
vote: 589
|
4
|
+
comment: 40
|
5
|
+
entry-1360692060:
|
6
|
+
user: 17
|
7
|
+
vote: 448
|
8
|
+
comment: 49
|
9
|
+
entry-1360692120:
|
10
|
+
user: 77
|
11
|
+
vote: 320
|
12
|
+
comment: 45
|
13
|
+
entry-1360692180:
|
14
|
+
user: 143
|
15
|
+
vote: 101
|
16
|
+
comment: 2
|
17
|
+
entry-1360692240:
|
18
|
+
user: 19
|
19
|
+
vote: 389
|
20
|
+
comment: 22
|
21
|
+
entry-1360692300:
|
22
|
+
user: 77
|
23
|
+
vote: 895
|
24
|
+
comment: 36
|
25
|
+
entry-1360692360:
|
26
|
+
user: 59
|
27
|
+
vote: 205
|
28
|
+
comment: 18
|
29
|
+
entry-1360692420:
|
30
|
+
user: 136
|
31
|
+
vote: 891
|
32
|
+
comment: 20
|
33
|
+
entry-1360692480:
|
34
|
+
user: 68
|
35
|
+
vote: 233
|
36
|
+
comment: 32
|
37
|
+
entry-1360692540:
|
38
|
+
user: 144
|
39
|
+
vote: 392
|
40
|
+
comment: 32
|
41
|
+
entry-1360692600:
|
42
|
+
user: 74
|
43
|
+
vote: 984
|
44
|
+
comment: 19
|
45
|
+
entry-1360692660:
|
46
|
+
user: 116
|
47
|
+
vote: 283
|
48
|
+
comment: 38
|
49
|
+
entry-1360692720:
|
50
|
+
user: 102
|
51
|
+
vote: 750
|
52
|
+
comment: 38
|
53
|
+
entry-1360692780:
|
54
|
+
user: 100
|
55
|
+
vote: 214
|
56
|
+
comment: 15
|
57
|
+
entry-1360692840:
|
58
|
+
user: 198
|
59
|
+
vote: 622
|
60
|
+
comment: 38
|
61
|
+
entry-1360692900:
|
62
|
+
user: 130
|
63
|
+
vote: 503
|
64
|
+
comment: 10
|
65
|
+
entry-1360692960:
|
66
|
+
user: 24
|
67
|
+
vote: 372
|
68
|
+
comment: 21
|
69
|
+
entry-1360693020:
|
70
|
+
user: 88
|
71
|
+
vote: 669
|
72
|
+
comment: 10
|
73
|
+
entry-1360693080:
|
74
|
+
user: 45
|
75
|
+
vote: 414
|
76
|
+
comment: 8
|
77
|
+
entry-1360693140:
|
78
|
+
user: 28
|
79
|
+
vote: 367
|
80
|
+
comment: 42
|
81
|
+
entry-1360693200:
|
82
|
+
user: 70
|
83
|
+
vote: 168
|
84
|
+
comment: 10
|
85
|
+
entry-1360693260:
|
86
|
+
user: 28
|
87
|
+
vote: 156
|
88
|
+
comment: 13
|
89
|
+
entry-1360693320:
|
90
|
+
user: 151
|
91
|
+
vote: 175
|
92
|
+
comment: 38
|
93
|
+
entry-1360693380:
|
94
|
+
user: 186
|
95
|
+
vote: 737
|
96
|
+
comment: 29
|
97
|
+
entry-1360693440:
|
98
|
+
user: 175
|
99
|
+
vote: 946
|
100
|
+
comment: 39
|
101
|
+
entry-1360693500:
|
102
|
+
user: 33
|
103
|
+
vote: 735
|
104
|
+
comment: 21
|
105
|
+
entry-1360693560:
|
106
|
+
user: 177
|
107
|
+
vote: 328
|
108
|
+
comment: 37
|
109
|
+
entry-1360693620:
|
110
|
+
user: 45
|
111
|
+
vote: 722
|
112
|
+
comment: 43
|
113
|
+
entry-1360693680:
|
114
|
+
user: 91
|
115
|
+
vote: 250
|
116
|
+
comment: 31
|
117
|
+
entry-1360693740:
|
118
|
+
user: 153
|
119
|
+
vote: 501
|
120
|
+
comment: 25
|
121
|
+
entry-1360693800:
|
122
|
+
user: 43
|
123
|
+
vote: 890
|
124
|
+
comment: 18
|
125
|
+
entry-1360693860:
|
126
|
+
user: 4
|
127
|
+
vote: 937
|
128
|
+
comment: 45
|
129
|
+
entry-1360693920:
|
130
|
+
user: 135
|
131
|
+
vote: 527
|
132
|
+
comment: 12
|
133
|
+
entry-1360693980:
|
134
|
+
user: 133
|
135
|
+
vote: 662
|
136
|
+
comment: 9
|
137
|
+
entry-1360694040:
|
138
|
+
user: 129
|
139
|
+
vote: 793
|
140
|
+
comment: 25
|
141
|
+
entry-1360694100:
|
142
|
+
user: 43
|
143
|
+
vote: 963
|
144
|
+
comment: 28
|
145
|
+
entry-1360694160:
|
146
|
+
user: 11
|
147
|
+
vote: 104
|
148
|
+
comment: 27
|
149
|
+
entry-1360694220:
|
150
|
+
user: 7
|
151
|
+
vote: 748
|
152
|
+
comment: 8
|
153
|
+
entry-1360694280:
|
154
|
+
user: 177
|
155
|
+
vote: 302
|
156
|
+
comment: 12
|
157
|
+
entry-1360694340:
|
158
|
+
user: 174
|
159
|
+
vote: 122
|
160
|
+
comment: 17
|
161
|
+
entry-1360694400:
|
162
|
+
user: 79
|
163
|
+
vote: 489
|
164
|
+
comment: 28
|
165
|
+
entry-1360694460:
|
166
|
+
user: 11
|
167
|
+
vote: 973
|
168
|
+
comment: 3
|
169
|
+
entry-1360694520:
|
170
|
+
user: 46
|
171
|
+
vote: 272
|
172
|
+
comment: 22
|
173
|
+
entry-1360694580:
|
174
|
+
user: 173
|
175
|
+
vote: 877
|
176
|
+
comment: 24
|
177
|
+
entry-1360694640:
|
178
|
+
user: 130
|
179
|
+
vote: 379
|
180
|
+
comment: 25
|
181
|
+
entry-1360694700:
|
182
|
+
user: 130
|
183
|
+
vote: 537
|
184
|
+
comment: 48
|
185
|
+
entry-1360694760:
|
186
|
+
user: 124
|
187
|
+
vote: 771
|
188
|
+
comment: 7
|
189
|
+
entry-1360694820:
|
190
|
+
user: 135
|
191
|
+
vote: 618
|
192
|
+
comment: 17
|
193
|
+
entry-1360694880:
|
194
|
+
user: 192
|
195
|
+
vote: 844
|
196
|
+
comment: 50
|
197
|
+
entry-1360694940:
|
198
|
+
user: 186
|
199
|
+
vote: 150
|
200
|
+
comment: 16
|
201
|
+
entry-1360695000:
|
202
|
+
user: 193
|
203
|
+
vote: 142
|
204
|
+
comment: 6
|
205
|
+
entry-1360695060:
|
206
|
+
user: 56
|
207
|
+
vote: 786
|
208
|
+
comment: 39
|
209
|
+
entry-1360695120:
|
210
|
+
user: 72
|
211
|
+
vote: 473
|
212
|
+
comment: 16
|
213
|
+
entry-1360695180:
|
214
|
+
user: 9
|
215
|
+
vote: 904
|
216
|
+
comment: 14
|
217
|
+
entry-1360695240:
|
218
|
+
user: 33
|
219
|
+
vote: 450
|
220
|
+
comment: 50
|
221
|
+
entry-1360695300:
|
222
|
+
user: 175
|
223
|
+
vote: 836
|
224
|
+
comment: 43
|
225
|
+
entry-1360695360:
|
226
|
+
user: 91
|
227
|
+
vote: 694
|
228
|
+
comment: 6
|
229
|
+
entry-1360695420:
|
230
|
+
user: 61
|
231
|
+
vote: 788
|
232
|
+
comment: 48
|
233
|
+
entry-1360695480:
|
234
|
+
user: 193
|
235
|
+
vote: 918
|
236
|
+
comment: 30
|
237
|
+
entry-1360695540:
|
238
|
+
user: 119
|
239
|
+
vote: 721
|
240
|
+
comment: 11
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module MerryGoRound
|
4
|
+
class AggregationTest < TestCase
|
5
|
+
def test_creating
|
6
|
+
Aggregation.create(key: 'test', value: 1, granularity: 'minute', timestamp: Time.now)
|
7
|
+
end
|
8
|
+
|
9
|
+
def test_validating_granularities
|
10
|
+
agg = Aggregation.new(key: 'test', value: 2, granularity: 'crap', timestamp: Time.now)
|
11
|
+
refute agg.save
|
12
|
+
|
13
|
+
agg.granularity = :minute
|
14
|
+
assert agg.save
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module MerryGoRound
|
4
|
+
class AggregatorTest < TestCase
|
5
|
+
def test_aggregating_from_redis
|
6
|
+
# Load the fixture. There is 60 minutes of data with 3 keys in each minute
|
7
|
+
redis_fixture('hour')
|
8
|
+
|
9
|
+
# Aggregate!
|
10
|
+
MerryGoRound.aggregate!
|
11
|
+
|
12
|
+
# 60 minutes * 3 keys
|
13
|
+
assert_equal 60 * 3, Aggregation.where(granularity: 'minute').count
|
14
|
+
|
15
|
+
# 1 hour * 3 keys
|
16
|
+
assert_equal 3, Aggregation.where(granularity: 'hour').count
|
17
|
+
|
18
|
+
# No days since there is only an hour of data
|
19
|
+
assert_equal 0, Aggregation.where(granularity: 'day').count
|
20
|
+
|
21
|
+
# The minute aggregations should belong to an hour one
|
22
|
+
refute_nil Aggregation.where(granularity: 'minute').first.parent_id
|
23
|
+
assert_equal 'hour', Aggregation.where(granularity: 'minute').first.parent.granularity
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module MerryGoRound
|
4
|
+
class QueryTest < TestCase
|
5
|
+
def test_execute
|
6
|
+
redis_fixture('hour')
|
7
|
+
MerryGoRound.aggregate!
|
8
|
+
|
9
|
+
min = MerryGoRound::Aggregation.minimum(:timestamp).to_i
|
10
|
+
max = MerryGoRound::Aggregation.maximum(:timestamp).to_i
|
11
|
+
|
12
|
+
query = Query.new(max: max, granularity: :hour)
|
13
|
+
out = query.execute
|
14
|
+
|
15
|
+
assert_equal 2, out.keys.length
|
16
|
+
out[:results].keys.each do |key|
|
17
|
+
assert_equal 1, out[:results][key].length
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def test_parsing_with_no_arguments
|
22
|
+
Timecop.freeze(Time.now) do
|
23
|
+
max_time = Time.at(Time.now.to_i)
|
24
|
+
|
25
|
+
query = Query.new(keys: 'test')
|
26
|
+
# should default to now, sans seconds
|
27
|
+
expected_max = max_time.utc - max_time.utc.sec
|
28
|
+
# should default to a day ago
|
29
|
+
expected_min = expected_max - 86400
|
30
|
+
|
31
|
+
assert_query_match query, expected_min, expected_max, 'test', :hour
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def test_parsing_with_min_and_max
|
36
|
+
Timecop.freeze(Time.now) do
|
37
|
+
# add some arbitrary offset
|
38
|
+
max_time = Time.now - 5000
|
39
|
+
# back in time by a day
|
40
|
+
min_time = max_time - 86400
|
41
|
+
|
42
|
+
query = Query.new(min: min_time.to_i, max: max_time.to_i, keys: 'test')
|
43
|
+
expected_max = max_time.utc - max_time.utc.sec
|
44
|
+
expected_min = min_time.utc - min_time.utc.sec
|
45
|
+
|
46
|
+
assert_query_match query, expected_min, expected_max, 'test', :hour
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def test_parsing_with_granularity
|
51
|
+
Timecop.freeze(Time.now) do
|
52
|
+
max_time = Time.at(Time.now.to_i)
|
53
|
+
|
54
|
+
query = Query.new(granularity: :week, keys: 'test')
|
55
|
+
# should default to now, sans seconds
|
56
|
+
expected_max = max_time.utc - max_time.utc.sec
|
57
|
+
# should default to a day ago
|
58
|
+
expected_min = expected_max - 31536000
|
59
|
+
|
60
|
+
assert_query_match query, expected_min, expected_max, 'test', :week
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def assert_query_match(query, expected_min, expected_max, expected_key, expected_granularity)
|
65
|
+
assert_equal expected_max.to_i, query.max.to_i
|
66
|
+
assert_equal expected_min.to_i, query.min.to_i
|
67
|
+
assert_equal expected_granularity, query.granularity
|
68
|
+
assert_equal [expected_key], query.keys
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
data/test/schema.rb
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
ActiveRecord::Schema.define(:version => 20130212042134) do
|
4
|
+
create_table "merry_go_round_aggregations", :force => true do |t|
|
5
|
+
t.string "key", :null => false
|
6
|
+
t.integer "value", :default => 0, :null => false
|
7
|
+
t.datetime "timestamp", :null => false
|
8
|
+
t.string "granularity", :null => false
|
9
|
+
t.integer "parent_id"
|
10
|
+
end
|
11
|
+
|
12
|
+
add_index "merry_go_round_aggregations", ["granularity", "timestamp", "key", "parent_id"], :name => "merry_go_round_aggregations_kgtp"
|
13
|
+
add_index "merry_go_round_aggregations", ["parent_id", "granularity", "timestamp"], :name => "merry_go_round_aggregations_pgt"
|
14
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module FixtureMacros
|
2
|
+
def fixture(name)
|
3
|
+
path = File.expand_path("../../fixtures/#{name}.yml", __FILE__)
|
4
|
+
YAML::load(File.open(path))
|
5
|
+
end
|
6
|
+
|
7
|
+
def redis_fixture(name)
|
8
|
+
hash = fixture(name)
|
9
|
+
redis = MerryGoRound.redis
|
10
|
+
|
11
|
+
hash.each do |key, value|
|
12
|
+
value.each do |k, v|
|
13
|
+
redis.hset key, k, v
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/test/test_helper.rb
CHANGED
@@ -14,4 +14,10 @@ Dir["#{File.expand_path(File.dirname(__FILE__))}/support/*.rb"].each do |file|
|
|
14
14
|
end
|
15
15
|
|
16
16
|
class MerryGoRound::TestCase < MiniTest::Unit::TestCase
|
17
|
+
include FixtureMacros
|
18
|
+
|
19
|
+
def teardown
|
20
|
+
MerryGoRound::Aggregation.delete_all
|
21
|
+
MerryGoRound.redis.flushdb
|
22
|
+
end
|
17
23
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: merry_go_round
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-02-
|
12
|
+
date: 2013-02-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activerecord
|
@@ -27,6 +27,22 @@ dependencies:
|
|
27
27
|
- - ! '>='
|
28
28
|
- !ruby/object:Gem::Version
|
29
29
|
version: 3.0.0
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: activesupport
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 3.0.0
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 3.0.0
|
30
46
|
- !ruby/object:Gem::Dependency
|
31
47
|
name: redis
|
32
48
|
requirement: !ruby/object:Gem::Requirement
|
@@ -60,7 +76,7 @@ dependencies:
|
|
60
76
|
- !ruby/object:Gem::Version
|
61
77
|
version: '0'
|
62
78
|
- !ruby/object:Gem::Dependency
|
63
|
-
name:
|
79
|
+
name: sinatra
|
64
80
|
requirement: !ruby/object:Gem::Requirement
|
65
81
|
none: false
|
66
82
|
requirements:
|
@@ -78,8 +94,7 @@ dependencies:
|
|
78
94
|
description: Simple data-warehousing.
|
79
95
|
email:
|
80
96
|
- sam@soff.es
|
81
|
-
executables:
|
82
|
-
- merry
|
97
|
+
executables: []
|
83
98
|
extensions: []
|
84
99
|
extra_rdoc_files: []
|
85
100
|
files:
|
@@ -90,19 +105,31 @@ files:
|
|
90
105
|
- LICENSE
|
91
106
|
- Rakefile
|
92
107
|
- Readme.markdown
|
93
|
-
- bin/merry
|
94
108
|
- lib/merry_go_round.rb
|
95
109
|
- lib/merry_go_round/aggregation.rb
|
96
110
|
- lib/merry_go_round/aggregator.rb
|
111
|
+
- lib/merry_go_round/core_ext/time.rb
|
97
112
|
- lib/merry_go_round/entry.rb
|
113
|
+
- lib/merry_go_round/query.rb
|
98
114
|
- lib/merry_go_round/railtie.rb
|
115
|
+
- lib/merry_go_round/utils.rb
|
99
116
|
- lib/merry_go_round/version.rb
|
117
|
+
- lib/merry_go_round/web.rb
|
100
118
|
- lib/rails/generators/merry_go_round/install/USAGE
|
101
119
|
- lib/rails/generators/merry_go_round/install/install_generator.rb
|
102
120
|
- lib/rails/generators/merry_go_round/install/templates/add_aggregations.rb
|
103
121
|
- lib/rails/generators/merry_go_round/install/templates/merry_go_round.rake
|
104
122
|
- merry_go_round.gemspec
|
123
|
+
- test/fixtures/hour.yml
|
124
|
+
- test/merry_go_round/aggregation_test.rb
|
125
|
+
- test/merry_go_round/aggregator_test.rb
|
126
|
+
- test/merry_go_round/query_test.rb
|
127
|
+
- test/merry_go_round/utils_test.rb
|
128
|
+
- test/merry_go_round/web_test.rb
|
105
129
|
- test/merry_go_round_test.rb
|
130
|
+
- test/schema.rb
|
131
|
+
- test/support/active_record.rb
|
132
|
+
- test/support/fixtures.rb
|
106
133
|
- test/test_helper.rb
|
107
134
|
homepage: https://github.com/seesawco/merry_go_round
|
108
135
|
licenses:
|
@@ -125,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
125
152
|
version: '0'
|
126
153
|
segments:
|
127
154
|
- 0
|
128
|
-
hash:
|
155
|
+
hash: -1990269655294854867
|
129
156
|
requirements: []
|
130
157
|
rubyforge_project:
|
131
158
|
rubygems_version: 1.8.23
|
@@ -133,6 +160,15 @@ signing_key:
|
|
133
160
|
specification_version: 3
|
134
161
|
summary: Simple data-warehousing with Redis and PostgreSQL.
|
135
162
|
test_files:
|
163
|
+
- test/fixtures/hour.yml
|
164
|
+
- test/merry_go_round/aggregation_test.rb
|
165
|
+
- test/merry_go_round/aggregator_test.rb
|
166
|
+
- test/merry_go_round/query_test.rb
|
167
|
+
- test/merry_go_round/utils_test.rb
|
168
|
+
- test/merry_go_round/web_test.rb
|
136
169
|
- test/merry_go_round_test.rb
|
170
|
+
- test/schema.rb
|
171
|
+
- test/support/active_record.rb
|
172
|
+
- test/support/fixtures.rb
|
137
173
|
- test/test_helper.rb
|
138
174
|
has_rdoc:
|