merry_go_round 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+bundler_args: --without development
+rvm:
+  - 1.9.3

data/Contributing.markdown

@@ -0,0 +1,19 @@
+## Submitting a Pull Request
+
+1. [Fork the repository.][fork]
+2. [Create a topic branch.][branch]
+3. Add tests for your unimplemented feature or bug fix.
+4. Run `bundle exec rake`. If your tests pass, return to step 3.
+5. Implement your feature or bug fix.
+6. Run `bundle exec rake`. If your tests fail, return to step 5.
+7. Run `open coverage/index.html`. If your changes are not completely covered
+   by your tests, return to step 3.
+8. Add documentation for your feature or bug fix.
+9. Run `bundle exec rake doc`. If your changes are not 100% documented, go
+   back to step 8.
+10. Add, commit, and push your changes.
+11. [Submit a pull request.][pr]
+
+[fork]: http://help.github.com/fork-a-repo/
+[branch]: http://learn.github.com/p/branching.html
+[pr]: http://help.github.com/send-pull-requests/

data/Gemfile

@@ -0,0 +1,19 @@
+source 'https://rubygems.org'
+
+# Gem dependencies
+gemspec
+
+gem 'rake', group: [:development, :test]
+
+# Development dependencies
+group :development do
+  gem 'yard'
+  gem 'redcarpet'
+end
+
+# Testing dependencies
+group :test do
+  gem 'minitest'
+  gem 'minitest-wscolor'
+  gem 'simplecov', require: false
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Seesaw Decisions Corporation
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,20 @@
+require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+end
+task default: :test
+
+begin
+  require 'yard'
+  YARD::Rake::YardocTask.new(:doc) do |task|
+    task.files   = ['Readme.markdown', 'LICENSE', 'lib/**/*.rb']
+    task.options = [
+      '--output-dir', 'doc',
+      '--markup', 'markdown',
+    ]
+  end
+rescue LoadError
+end

data/Readme.markdown

@@ -0,0 +1,53 @@
+# Merry Go Round
+
+*Note: This is a work in progress and it not production ready. If want to help, we'll love you forever.*
+
+Simple data-warehousing with Redis and ActiveRecord.
+
+All data is stored in Redis until it is aggregated. Once it is aggregated, it is stored in ActiveRecord. Merry Go Round handles all of the collecting, aggregating, and querying for you. All you have to do it put your data in and get your data out. All of the hard work of warehousing is done for you automatically.
+
+Redis is just used as a temporary store until the data can be warehoused in ActiveRecord. You can blow away Redis at anytime and only lose data since the last aggregation.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+``` ruby
+gem 'merry_go_round'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install merry_go_round
+
+## Usage
+
+You'll need to add the migrations to your app. Simply run:
+
+    $ merry install
+
+Merry Go Round will automatically detect most Redis servers. If you need to configure it, add an initializer with:
+
+``` ruby
+MerryGoRound.configure do |config|
+  config.redis = { url: 'redis://redis.example.com:7372/12', namespace: 'mynamespace' }
+end
+```
+
+### Aggregating
+
+To run the aggregator, run:
+
+    $ merry go
+
+By default, it will aggregate everything at 1 minute, 1 hour, 1 day, 1 week, 1 month, and 1 year. All you need to do is run the command and it will automatically aggregate for each time period. If that time period is not complete, it will not aggregate them (i.e. if you have been running Merry Go Round for 5 days, it won't generate week and higher aggregations).
+
+It is recommened to run the aggregator via cron (or [Heroku Scheduler](https://devcenter.heroku.com/articles/scheduler)) every few minutes. Since it uses ActiveRecord to store the aggregations, it will have to load your environment.
+
+## Contributing
+
+See the [contributing guide](Contributing.markdown).

data/bin/merry

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path('../../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'merry_go_round/cli'
+
+MerryGoRound::Cli.start

data/lib/merry_go_round/aggregation.rb

@@ -0,0 +1,16 @@
+require 'active_record'
+
+module MerryGoRound
+  # Aggregation is the ActiveRecord store for aggregated data points. You should
+  # never create aggregations directly unless you know what you're doing.
+  class Aggregation < ActiveRecord::Base
+    attr_accessible :key, :value, :timestamp, :granularity, :parent_id
+
+    belongs_to :aggregation, foreign_key: :parent_id
+    has_many :aggregations, foreign_key: :parent_id
+
+    def self.values_for_key(key, granularity = 60)
+      where('key = ? AND granularity = ?', granularity)
+    end
+  end
+end

data/lib/merry_go_round/aggregator.rb

@@ -0,0 +1,69 @@
+module MerryGoRound
+  class Aggregator
+    GRANULARITIES = [:minute, :hour, :day, :week, :year] # :month, :quarter
+
+    def aggregate!
+      from_redis
+      existing_slivers
+    end
+
+  private
+
+    def redis
+      MerryGoRound.redis
+    end
+
+    # Aggregate stuff since the last aggregation in Redis
+    def from_redis
+      # Aggregate to the first granularity
+      redis_granularity = GRANULARITIES.first
+
+      # Get entry keys from Redis
+      redis_keys = redis.keys 'entry-*'
+
+      # Loop through the Redis keys
+      redis_keys.each do |redis_key|
+        # Extract the timestamp
+        timestamp = Time.utc(redis_key.sub('entry-', '').to_i)
+
+        # Make sure this timestamp hasn't already been aggregated
+        redis.multi do
+          # Get the hash of Merry Go Round keys and values
+          hash = redis.hgetall redis_key
+
+          # Loop through each key/value in the hash
+          hash.each do |key, value|
+            Sliver.create(key: key, value: value, timestamp: timestamp, granularity: redis_granularity)
+          end
+
+          # Delete the key
+          redis.del redis_key
+        end
+      end
+    end
+
+    # Aggregate existing slivers (i.e. hour -> day)
+    def existing_slivers
+      # Loop through all of the granularities except for the first one
+      grans = GRANULARITIES.slice(1, GRANULARITIES.length - 1)
+      grans.each_with_index do |granularity, index|
+        # Get the previous granularity
+        prev_gran = GRANULARITIES[index - 1]
+
+        # Get slivers with the previous granularity
+        Sliver.where(granularity: prev_gran, )
+      end
+    end
+
+    def self.seconds(granularity)
+      map = {
+        minute: 60,
+        hour: 3600,
+        day: 86400,
+        week: 604800,
+        year: 31536000
+      }
+      map[granularity]
+    end
+  end
+end

data/lib/merry_go_round/cli.rb

@@ -0,0 +1,17 @@
+require 'thor'
+require 'merry_go_round'
+
+module MerryGoRound
+  class Cli < Thor
+    desc 'go', 'Aggregate collected data in Redis into ActiveRecord Slivers.'
+    def go
+      MerryGoRound.aggregate!
+    end
+
+    desc 'install', 'Install the Merry Go Round migration'
+    def install
+      system 'rails generate merry_go_round:install'
+      puts "\n\nThe migratation has been installed. Don't forget to run `rake db:migrate`.\n"
+    end
+  end
+end

data/lib/merry_go_round/entry.rb

@@ -0,0 +1,26 @@
+module MerryGoRound
+  class Entry
+    def initialize(key, increment = 1, timestamp = Time.now)
+      @key = key
+      @increment = increment
+
+      # Note, storing in the past isn't supported. If you set this to something the
+      # aggregator has already run, the results are undefined.
+      @timestamp = timestamp
+    end
+
+    def record!
+      # Normalize time. Drop seconds and convert to UTC
+      time = Time.at(@timestamp.to_i - @timestamp.sec).utc.to_i
+
+      # Store in Redis
+      redis.hincrby "entry-#{time.to_s}", @key, @increment
+    end
+
+  private
+
+    def redis
+      MerryGoRound.redis
+    end
+  end
+end

data/lib/merry_go_round/railtie.rb

@@ -0,0 +1,4 @@
+module MerryGoRound
+  class Railtie < Rails::Railtie
+  end
+end

data/lib/merry_go_round/version.rb

@@ -0,0 +1,3 @@
+module MerryGoRound
+  VERSION = '0.0.1'
+end

data/lib/merry_go_round.rb

@@ -0,0 +1,53 @@
+require 'merry_go_round/version'
+require 'merry_go_round/aggregator'
+require 'merry_go_round/entry'
+require 'merry_go_round/aggregation'
+require 'merry_go_round/railtie' if defined? Rails
+
+require 'redis'
+require 'redis/namespace'
+
+module MerryGoRound
+  def self.configure
+    yield self
+  end
+
+  def self.aggregate!
+    Aggregator.new.aggregate!
+  end
+
+  def self.record(*args)
+    Entry.new(*args).record!
+  end
+
+  def self.redis
+    # Set redis to nothing make the setter run and setup a default if it's nothing
+    self.redis = {} unless defined? @@redis
+
+    # Return the namespaced Redis instance
+    @@redis
+  end
+
+  def self.redis=(options = {})
+    client = nil
+    if options.is_a?(Redis)
+      client = options
+    else
+      url = options[:url] || determine_redis_provider || 'redis://localhost:6379/0'
+      driver = options[:driver] || 'ruby'
+      namespace = options[:namespace] || 'merry_go_round'
+
+      client = Redis.connect(url: url, driver: driver)
+    end
+
+    @@redis = Redis::Namespace.new(namespace, redis: client)
+  end
+
+private
+
+  def self.determine_redis_provider
+    return ENV['REDISTOGO_URL'] if ENV['REDISTOGO_URL']
+    provider = ENV['REDIS_PROVIDER'] || 'REDIS_URL'
+    ENV[provider]
+  end
+end

data/lib/rails/generators/merry_go_round/install/USAGE

@@ -0,0 +1,3 @@
+Description:
+  The merry_go_round:install generator creates the necessary migrations to
+  use Merry Go Round.

data/lib/rails/generators/merry_go_round/install/install_generator.rb

@@ -0,0 +1,23 @@
+module MerryGoRound
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include Rails::Generators::Migration
+
+      source_root File.expand_path('../templates', __FILE__)
+      desc 'Add Merry Go Round migrations'
+
+      def self.next_migration_number(path)
+        unless @prev_migration_nr
+          @prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+        else
+          @prev_migration_nr += 1
+        end
+        @prev_migration_nr.to_s
+      end
+
+      def copy_migrations
+        migration_template 'add_aggregations.rb', 'db/migrate/add_merry_go_round_aggregations.rb'
+      end
+    end
+  end
+end

data/lib/rails/generators/merry_go_round/install/templates/add_aggregations.rb

@@ -0,0 +1,23 @@
+class AddMerryGoRoundAggregations < ActiveRecord::Migration
+  def change
+    create_table :merry_go_round_aggregations do |t|
+      # The name of whatever you're tracking
+      t.string :key, null: false
+
+      # The integer value of your count
+      t.integer :value, null: false, default: 0
+
+      # When it was counted rounded to a normalized time
+      t.datetime :timestamp, null: false
+
+      # A string representing the granularity (minute, hour, day, week, month, year)
+      t.string :granularity, null: false
+
+      # Foreign key for tree structure
+      t.integer :parent_id
+    end
+
+    add_index :merry_go_round_aggregations, [:key, :granularity, :timestamp, :parent_id], name: 'merry_go_round_aggregations_kgtp'
+    add_index :merry_go_round_aggregations, [:parent_id, :granularity, :timestamp], name: 'merry_go_round_aggregations_pgt'
+  end
+end

data/merry_go_round.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'merry_go_round/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'merry_go_round'
+  gem.version       = MerryGoRound::VERSION
+  gem.authors       = ['Sam Soffes']
+  gem.email         = ['sam@soff.es']
+  gem.description   = 'Simple data-warehousing.'
+  gem.summary       = 'Simple data-warehousing with Redis and PostgreSQL.'
+  gem.homepage      = 'https://github.com/seesawco/merry_go_round'
+  gem.license       = 'MIT'
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.required_ruby_version = '>= 1.9.2'
+  gem.add_dependency 'activerecord', '>= 3.0.0'
+  gem.add_dependency 'redis', '>= 3.0.0'
+  gem.add_dependency 'redis-namespace'
+  gem.add_dependency 'thor'
+end

data/test/merry_go_round_test.rb

@@ -0,0 +1,13 @@
+require 'test_helper'
+
+class MerryGoRoundTest < MerryGoRound::TestCase
+  def test_configure
+    redis = Redis.new
+    MerryGoRound.configure do |config|
+      config.redis = redis
+    end
+
+    # It returns a namespaced Redis, so call `.redis` again on it
+    assert_equal redis, MerryGoRound.redis.redis
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'bundler'
+Bundler.require :test
+
+require 'simplecov'
+SimpleCov.start
+
+require 'minitest/autorun'
+require 'merry_go_round'
+
+# Support files
+Dir["#{File.expand_path(File.dirname(__FILE__))}/support/*.rb"].each do |file|
+  require file
+end
+
+class MerryGoRound::TestCase < MiniTest::Unit::TestCase
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: merry_go_round
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Sam Soffes
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: redis-namespace
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Simple data-warehousing.
+email:
+- sam@soff.es
+executables:
+- merry
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Contributing.markdown
+- Gemfile
+- LICENSE
+- Rakefile
+- Readme.markdown
+- bin/merry
+- lib/merry_go_round.rb
+- lib/merry_go_round/aggregation.rb
+- lib/merry_go_round/aggregator.rb
+- lib/merry_go_round/cli.rb
+- lib/merry_go_round/entry.rb
+- lib/merry_go_round/railtie.rb
+- lib/merry_go_round/version.rb
+- lib/rails/generators/merry_go_round/install/USAGE
+- lib/rails/generators/merry_go_round/install/install_generator.rb
+- lib/rails/generators/merry_go_round/install/templates/add_aggregations.rb
+- merry_go_round.gemspec
+- test/merry_go_round_test.rb
+- test/test_helper.rb
+homepage: https://github.com/seesawco/merry_go_round
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 3503042711735793145
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Simple data-warehousing with Redis and PostgreSQL.
+test_files:
+- test/merry_go_round_test.rb
+- test/test_helper.rb
+has_rdoc: