reduceable 0.1.0

data/.gitignore

@@ -0,0 +1,4 @@
+pkg/*
+*.gem
+.bundle
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,9 @@
+source :rubygems
+gem "mongo_mapper", "~>0.9.0"
+gem "bson_ext"
+gem "activesupport", "~>3.0.7"
+
+source :gemcutter
+
+# Specify your gem's dependencies in the .gemspec file
+gemspec

data/README.markdown

@@ -0,0 +1,99 @@
+Reducable
+=========
+
+This is a module for MongoMapper which provides an easy way to add in some simple 
+map/reduce functions to your data. If you have time series data and you want to show
+some sort of counter per date or time, then this should do it.
+
+Concept
+-------
+You have a bunch of objects in your MongoDB. You need to get some basic information
+about them such as:
+Simple aggregation of documents per key,
+Finding an average of a value,
+Counting the number of documents that contain a key.
+
+You've probably read you can do this sort of stuff with MongoDB's map/reduce
+functionality, maybe you already know exactly how that works or maybe you don't
+really have a clue. Every guide I've seen for MongoMapper recommends you execute
+the map/reduce calculation every single time it's accessed, and they all demand
+that you write your own map and reduce functions.
+
+Here are some use cases
+
+```ruby
+# Count how many times each tag is used
+Model.count_by(:tag, query = {})
+# Sum all the weights of the different types of wresters 
+Model.sum_of(:weight, :wrester_type, query = {})
+```
+
+Coming Soon
+----------
++ mongoid support
++ Sum by composite index
++ Averages
++ More Unit Tests :(
+
+Installation
+------------
+```
+gem install reduceable
+# or
+sudo gem install reduceable
+```
+
+Usage
+-----
+```ruby
+require 'mongo_mapper'
+require 'reduceable'
+
+MongoMapper.database = 'my_database_name'
+
+class BlogPost
+  include MongoMapper::Document
+  include Reduceable
+
+  key :article_body, String
+  key :categories, Array
+  key :time_posted, Time
+  key :article_length, Integer
+end
+
+# Insert some data
+
+BlogPost.count_by(:categories).to_a.each do |x| 
+  puts "You have posted #{x['value']} posts from catefory #{x['_id']}"
+end
+BlogPost.sum_of(:article_length, :categories).to_a.each do |x|
+  puts "You have written #{x['value']} characters in category #{x['_id']}"
+end
+```
+
+See example.rb
+
+```ruby
+# require the example model
+require './example.rb'  #=> true
+# setup some base data
+setup #=> #<Test _id: BSON: ...... 
+#
+# Calculate how many times each tag is used
+# You will use a similar map/reduce for a tag cloud
+Test.count_by(:tags).to_a
+#=> [{"_id"=>"alternative", "value"=>1.0}, {"_id"=>"book", "value"=>5.0}, {"_id"=>"classical", "value"=>1.0}, {"_id"=>"fantasy", "value"=>2.0}, {"_id"=>"fiction", "value"=>2.0}, {"_id"=>"music", "value"=>4.0}, {"_id"=>"non-fiction", "value"=>1.0}, {"_id"=>"pop", "value"=>1.0}, {"_id"=>"rock", "value"=>1.0}]
+
+# Sum up the sale_amounts per tag
+Test.sum_of(:sale_amount, :tags).to_a
+
+# Sum up the sale_amounts per tag where tags contains 'book'
+Test.sum_of(:sale_amount, :tags, {:tags => 'book'}).to_a
+# you can optionally pass in a mongo query that limits the initial dataset being
+# fed to the map function.
+```
+
+For such a small collection the speed benefits aren't present, but once you get to
+several hundred thousand record, recreating the map_reduce collection on every call 
+really slows things down. Reduceable solves that problem.
+

data/Rakefile

@@ -0,0 +1,15 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:core) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rspec_opts = %w(-fs --color --format documentation)
+end
+
+Dir["tasks/*.rake"].each do |rake_file|
+  load rake_file
+end
+
+task :default => [:core]

data/example.rb

@@ -0,0 +1,29 @@
+require 'mongo_mapper'
+require 'reduceable'
+
+MongoMapper.database = 'mr_test'
+
+def setup
+  # add some data
+  Test.collection.remove
+  Test.create(:date => Date.today, :tags => ['book', 'fiction'], :sale_amount => 40)
+  Test.create(:date => Date.today, :tags => ['music', 'rock'], :sale_amount => 20.5)
+  Test.create(:date => Date.today, :tags => ['music', 'pop'], :sale_amount => 20.5)
+  Test.create(:date => Date.today, :tags => ['music', 'classical'], :sale_amount => 20.5)
+  Test.create(:date => Date.today, :tags => ['music', 'alternative'], :sale_amount => 20.5)
+  Test.create(:date => Date.today, :tags => ['book', 'fiction'], :sale_amount => 40)
+  Test.create(:date => Date.today, :tags => ['book', 'fantasy'], :sale_amount => 40)
+  Test.create(:date => Date.today, :tags => ['book', 'fantasy'], :sale_amount => 40)
+  Test.create(:date => Date.today, :tags => ['book', 'non-fiction'], :sale_amount => 40)
+end
+
+class Test
+  include MongoMapper::Document
+  include Reduceable
+  
+  key :date, String # or Time YYYYMMDD format
+  key :sale_amount, Float
+  key :tags, Array # a list of tags you might want to query on
+                   # eg: you can query on 'book' to find out 
+                   # how many book sales you have per day
+end

data/lib/reduceable.rb

@@ -0,0 +1,121 @@
+require 'active_support'
+require 'mongo_mapper'
+require 'base64'
+
+class MrStatus
+  include MongoMapper::Document
+  safe
+
+  # mr collection to query
+  key :collection_name, String, :unique => true
+  # the base class the mr results are calculated from
+  key :base_class, String 
+  key :status, Boolean # true = dirty, clean = false
+  
+end
+
+module Reduceable
+  extend ActiveSupport::Concern
+
+  included do
+    after_save :mr_dirty!
+  end
+
+  module ClassMethods
+    def sum_of(property, index, query={})
+      collection = mr_collection_name("sum_of_#{property}_by_#{index}", query) 
+      map = sum_map(property, index)
+      reduce = sum_reduce
+      return build(collection, map, reduce, query).find
+    end
+    def sum_map(property, index)
+      index = index.to_s if index.is_a? Symbol
+      if self.keys[index].type == Array
+        "function(){var amount = this.#{property};this.#{index}.forEach(function(value){emit(value, amount);});}"
+      else
+        "function(){emit(this.#{index}, this.#{property});}"
+      end
+    end
+    def sum_reduce 
+      <<-REDUCE
+        function(key, values) {
+          var total = 0;
+          for (var i=0; i<values.length; i++){
+            total += values[i];
+          }
+          return total;
+        }
+      REDUCE
+    end
+
+    def count_by(index, query={})
+      collection = mr_collection_name("count_by_#{index}", query) 
+      map = count_map(index)
+      reduce = count_reduce
+      
+      return build(collection, map, reduce, query).find
+    end
+    def count_map(key)
+      # Not sure how to handle hashes yet
+      key = key.to_s if key.is_a? Symbol
+      if self.keys[key].type == Array
+        "function(){this.#{key}.forEach(function(value){emit(value, 1);});}"
+      else
+        "function(){emit(this.#{key}, 1);}"
+      end
+    end
+    def count_reduce
+      <<-REDUCE
+        function(key, values) {
+          var total = 0;
+          for (var i=0; i<values.length; i++){
+            total += values[i];
+          }
+          return total;
+        }
+      REDUCE
+    end
+
+    def mr_collection_name(action, query = {})
+      # we need a unique collection name based off the query
+      # this introduces a fun bug where if your query params are in a random order
+      # you won't get the performance increase of reusing map/reduce collections
+      # TODO: come up with a better way of getting the collection name
+      name = (self.to_s.downcase + "#{action}_mr_" + Base64.urlsafe_encode64(query.to_s)).gsub('=','_').gsub(':','_')
+      return name
+    end
+
+    def build(collection, map, reduce, query = {})
+      if requires_mr_update collection
+        mr_status = MrStatus.new
+        mr_status.collection_name = collection
+        mr_status.status = false
+        mr_status.base_class = self.to_s
+        mr_status.save
+        opts = {:out => {:replace => collection}, :query => query}
+        self.collection.map_reduce(map, reduce, opts)
+      else
+        self.database[collection]
+      end
+    end
+
+    # Does this particular map reduce require an update?
+    def requires_mr_update(collection)
+      status_list = MrStatus.where(:collection_name => collection).all
+      return true if status_list.count == 0
+      status_list.each do |status|
+        return status.status
+      end
+    end
+  end
+
+  module InstanceMethods
+    def mr_dirty!
+      MrStatus.where({:base_class => self.class.to_s}).all.each do |m|
+        m.status = true
+        m.save
+      end
+    end
+  end
+end
+

data/lib/reduceable/version.rb

@@ -0,0 +1,3 @@
+module Reduceable
+  VERSION = "0.1.0"
+end

data/reduceable.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path("../lib/reduceable/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "reduceable"
+  s.version     = Reduceable::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Leonard Garvey']
+  s.email       = ['lengarvey@gmail.com']
+  s.homepage    = "http://github.com/lengarvey/reduceable"
+  s.summary     = "Reduceable makes map reduce in mongo easy"
+  s.description = ""
+
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.add_development_dependency "bundler", ">= 1.0.0"
+  s.add_development_dependency "rspec", ">= 2.3.0"
+
+  s.files        = `git ls-files`.split("\n")
+  s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
+  s.require_path = 'lib'
+end

data/spec/helpers.rb

@@ -0,0 +1,44 @@
+require 'mongo_mapper'
+require './lib/reduceable'
+
+module Helpers
+  MongoMapper.database = 'reduceable_dev'
+
+  def clear_database
+    MongoMapper.database.collections.each do |collection|
+      begin   
+        collection.drop
+      rescue Exception => e   
+      end   
+    end
+  end
+  def count_answers
+    {'book'=>5.0, 'fiction'=>2.0, 'music'=>4.0, 'fantasy'=>2.0, 'non-fiction'=>1.0, 'rock'=>1.0, 'pop'=>1.0, 'classical'=>1.0, 'alternative'=>1.0}
+  end
+  def sum_answers
+    {'book'=>200.0, 'fiction'=>80.0, 'music'=>82.0, 'fantasy'=>80.0, 'non-fiction'=>40.0, 'rock'=>20.5, 'pop'=>20.5, 'classical'=>20.5, 'alternative'=>20.5}
+  end
+  def load_data 
+    # add some data
+    Sale.create(:date => Date.today, :tags => ['book', 'fiction'], :sale_amount => 40)
+    Sale.create(:date => Date.today, :tags => ['music', 'rock'], :sale_amount => 20.5)
+    Sale.create(:date => Date.today, :tags => ['music', 'pop'], :sale_amount => 20.5)
+    Sale.create(:date => Date.today, :tags => ['music', 'classical'], :sale_amount => 20.5)
+    Sale.create(:date => Date.today, :tags => ['music', 'alternative'], :sale_amount => 20.5)
+    Sale.create(:date => Date.today, :tags => ['book', 'fiction'], :sale_amount => 40)
+    Sale.create(:date => Date.today, :tags => ['book', 'fantasy'], :sale_amount => 40)
+    Sale.create(:date => Date.today, :tags => ['book', 'fantasy'], :sale_amount => 40)
+    Sale.create(:date => Date.today, :tags => ['book', 'non-fiction'], :sale_amount => 40)
+  end
+
+  class Sale 
+    include MongoMapper::Document
+    include Reduceable
+
+    key :date, String # or Time YYYYMMDD format
+    key :sale_amount, Float
+    key :tags, Array # a list of tags you might want to query on
+    # eg: you can query on 'book' to find out 
+    # how many book sales you have per day
+  end
+end

data/spec/reduceable_spec.rb

@@ -0,0 +1,41 @@
+$: << File.dirname(__FILE__)
+require 'helpers'
+
+RSpec.configure do |c|
+    c.include Helpers
+end
+
+describe "Reduceable" do
+  before(:each) do
+    clear_database
+    load_data  
+  end
+  it "should count_by" do
+    Sale.respond_to?(:count_by).should eql true
+  end
+  it "should sum_of" do
+    Sale.respond_to?(:sum_of).should eql true
+  end
+  it "returns a cursor when asked to count" do
+    values = Sale.count_by(:tags)
+    values.class.should eql(Mongo::Cursor)
+  end
+  it "should be able to count" do 
+    values = Sale.count_by(:tags).to_a
+    answers = count_answers
+    values.each do |value|
+      id = value['_id']
+      answer = answers[id]
+      answer.should eql(value['value']), "Number of #{id}s calculated. Expected #{value['value']} got #{answer}"
+    end  
+  end
+  it "should be able to add" do
+    values = Sale.sum_of(:sale_amount,:tags).to_a
+    answers = sum_answers
+    values.each do |value|
+      id = value['_id']
+      answer = answers[id]
+      answer.should eql(value['value']), "Number of #{id}s calculated. Expected #{value['value']} got #{answer}"
+    end  
+  end
+end

metadata

@@ -0,0 +1,77 @@
+--- !ruby/object:Gem::Specification
+name: reduceable
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Leonard Garvey
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-05-16 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &2156485540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *2156485540
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2156484980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *2156484980
+description: ''
+email:
+- lengarvey@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.markdown
+- Rakefile
+- example.rb
+- lib/reduceable.rb
+- lib/reduceable/version.rb
+- reduceable.gemspec
+- spec/helpers.rb
+- spec/reduceable_spec.rb
+homepage: http://github.com/lengarvey/reduceable
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Reduceable makes map reduce in mongo easy
+test_files: []