modesty 0.1.0

data/Gemfile

@@ -0,0 +1,13 @@
+source 'http://rubygems.org'
+
+
+gem 'redis'
+
+group :development do
+  gem 'rake'
+  gem 'jeweler'
+end
+
+group :test do
+  gem 'rspec', '~> 1'
+end

data/Gemfile.lock

@@ -0,0 +1,18 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.8.7)
+    rspec (1.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jeweler
+  rake
+  rspec (~> 1)

data/LICENSE

@@ -0,0 +1,21 @@
+#MIT license.  See http://www.opensource.org/licenses/mit-license.php
+
+Copyright (c) 2010 Philotic, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,121 @@
+# Modesty
+
+Modesty is a really simple metrics and a/b testing framework that doesn't really do all that much.  It was inspired by assaf's Vanity (github.com/assaf/vanity).
+
+## Metrics
+A metric keeps track of things that happen, and ties in and disaggregates many different types of data.  Define a metric with:
+
+    Modesty.new_metric :foo
+
+Your metric will now be available as `Modesty.metrics[:foo]`.
+You can track it with
+
+    Modesty.track! :foo`,
+
+You can get a raw count with
+
+    Modesty.metrics[:foo].count`.
+
+You can track multiple counts with `Modesty.track! :foo, 7`,
+or, if you prefer, `Modesty.track! :foo, :count => 7`.
+
+Simple, huh?
+
+You can also pass in any sort of data when you track your metric.  For example,
+
+    Modesty.track! :product_page_viewed, :with => {:product_id => 500, :seller_id => 278}
+
+This provides you with a few more granular methods.
+
+    m = Modesty.metrics[:product_page_viewed]
+    m.unique :product_ids # => number of unique product_ids that were tracked
+    m.all :product_ids    # => the actual ids that were tracked
+    m.aggregate_by :product_ids # => a hash of {product_id => tracks for this product id}
+    m.distribution_by :product_ids # => equivalent to aggregate_by(:product_ids).values.histogram
+
+TODO: submetrics
+
+## Identity
+To save you some hassle, Modesty keeps around a global identity in `Modesty.identity`.
+You can set the identity with `Modesty.identify! id`,
+where `id` is either an integer (i.e. the id of the current user) or `nil` for guests.
+When the identity is present, all metrics tracked will get a `:user` parameter passed in.
+This makes it really easy to call `m.unique :users` and such,
+without having to pass it in every time.
+To override this, just call `Modesty.track! :metric, :with => {:user => other_user}`.
+
+If you're using Rails, I recommend putting a `before_filter` on `ApplicationController` that does something akin to `Modesty.identify! viewer.id`.
+
+## Experiments
+Experiments are really the point of Modesty.
+With an experiment, you separate your users into experiment groups
+and track how each group performs on a given set of metrics.
+Modesty will ensure that
+  * each group contains roughly the same number of users
+  * Each user has a consistent experience
+  * All specified metrics can be disaggregated by experiment group.
+
+Here's how you make an experiment:
+
+    Modesty.new_experiment :button_size do |e|
+      e.metrics :conversion, :view
+      e.alternatives :huge, :medium, :small
+    end
+
+Then, you can do something like this (in a controller, say)
+
+    button_size = Modesty.experiment :button_size do |e|
+      e.group :huge do
+        9001
+      end
+      e.group :medium do
+        1337
+      end
+      e.group :small do
+        2
+      end
+    end
+
+This code will use `Modesty.identity` to determine the appropriate experiment group,
+and run the corresponding block.
+
+All your tracking data will automagically be disaggregated into
+
+    Modesty.metrics[:conversion/:button_size/:huge]
+    Modesty.metrics[:conversion/:button_size/:medium]
+    Modesty.metrics[:conversion/:button_size/:small]
+
+## Statistics and reporting
+
+TODO.
+
+##Datastores and config
+
+Right now there are two datastores available: Redis and a sweet mock Redis for testing.  To switch between them, use
+
+    Modesty.data = :redis
+    Modesty.data = :mock
+
+If you need to pass in more options, use
+
+    Modesty.set_store :redis, :port => 8888, :host => '123.123.123.1'
+
+## Rails
+
+By default, Modesty looks in configy/modesty.yml for something like:
+
+    datastore:
+      type: redis
+      port: 6739
+      host: localhost
+
+    paths:
+      experiments: modesty/experiments
+      metrics: modesty/metrics
+
+In this, the default setup, Modesty will look for experiments
+in #{Rails.root}/modesty/experiments, and metrics in #{Rails.root}/modesty/metrics.
+If no config file is found, or you omit something, Modesty will use these settings.
+Everything in the `datastore:` stanza (sans type: redis) will be passed as options to Redis.new.
+
+Have fun!

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'rake'
+require 'spec/rake/spectask'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "modesty"
+    gem.summary = %Q{Modesty is simple and scalable split testing and event tracking framework.}
+    gem.description = %Q{Modesty is simple and scalable split testing and event tracking framework. It was inspired by assaf's Vanity (github.com/assaf/vanity).}
+    gem.email = "jay@causes.com"
+    gem.homepage = "http://github.com/causes/modesty"
+    gem.authors = ["Jay Adkisson", "Kevin Ball", "Kristján Pétursson"]
+    gem.add_development_dependency "rspec"
+    gem.add_runtime_dependency "redis"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+Jeweler::RubygemsDotOrgTasks.new
+
+
+Spec::Rake::SpecTask.new('spec') do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.ruby_opts = ['-Ilib']
+  t.spec_opts = ['-O spec/spec.opts']
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/init.rb

@@ -0,0 +1 @@
+require File.join(File.dirname(__FILE__), 'lib', 'modesty.rb')

data/lib/modesty.rb

@@ -0,0 +1,26 @@
+require 'yaml'
+require 'rubygems'
+require 'active_support'
+
+
+module Modesty
+  LIB = File.dirname(__FILE__)
+  ROOT = File.expand_path(File.join(LIB, '..'))
+  VENDOR = File.expand_path(File.join(ROOT, 'vendor'))
+  TEST = File.expand_path(File.join(ROOT, 'test'))
+end
+
+$:.unshift Modesty::LIB
+require 'modesty/core_ext.rb'
+require 'modesty/api.rb'
+require 'modesty/datastore.rb'
+require 'modesty/identity.rb'
+require 'modesty/metric.rb'
+require 'modesty/experiment.rb'
+require 'modesty/load.rb'
+
+if defined? Rails
+  require 'modesty/frameworks/rails'
+end
+
+$:.shift

data/lib/modesty/api.rb

@@ -0,0 +1,14 @@
+module Modesty
+  class API
+  end
+
+  class << self
+    def api
+      @api ||= API.new
+    end
+
+    def method_missing(meth, *args, &blk)
+      self.api.send(meth, *args, &blk)
+    end
+  end
+end

data/lib/modesty/core_ext.rb

@@ -0,0 +1,5 @@
+require 'modesty/core_ext/array.rb'
+require 'modesty/core_ext/fixnum.rb'
+require 'modesty/core_ext/hash.rb'
+require 'modesty/core_ext/symbol.rb'
+require 'modesty/core_ext/string.rb'

data/lib/modesty/core_ext/array.rb

@@ -0,0 +1,21 @@
+class Array
+  unless method_defined? :histogram
+    def histogram
+      hsh = Hash.new(0)
+      self.each do |e|
+        hsh[e] += 1
+      end
+      hsh
+    end
+  end
+
+  def to_h
+    Hash[self]
+  end
+  
+  def hashmap
+    self.map do |e|
+      [e, yield(e)]
+    end.to_h
+  end
+end

data/lib/modesty/core_ext/fixnum.rb

@@ -0,0 +1,5 @@
+class Fixnum
+  def to_i?
+    self
+  end
+end

data/lib/modesty/core_ext/hash.rb

@@ -0,0 +1,39 @@
+class Hash
+  # addition for histogram aggregation
+  # >> {:a=>1, :b=>2} + {:a=>3, :c=>4}
+  # => {:a=>4, :b=>2, :c=>4}
+  def +(other)
+    hash = self.dup
+    other.each do |k, v|
+      if hash.include? k
+        hash[k] += v
+      else
+        hash[k] = v
+      end
+    end
+    hash
+  end
+
+  def map_values!(&blk)
+    self.each do |k,v|
+      self[k] = yield(v)
+    end
+  end
+
+  def map_values(&blk)
+    self.dup.map_values!(&blk)
+  end
+
+  def map!
+    self.keys.each do |k|
+      v = self.delete(k)
+      new_k, new_v = yield [k,v]
+      self[new_k] = new_v
+    end
+    self
+  end
+
+  def to_h
+    self
+  end
+end

data/lib/modesty/core_ext/string.rb

@@ -0,0 +1,9 @@
+class String
+  # casts to an integer if applicable,
+  # else leaves it alone.
+  def to_i?
+    Integer(self)
+  rescue
+    self
+  end
+end

data/lib/modesty/core_ext/symbol.rb

@@ -0,0 +1,33 @@
+class Symbol
+  unless method_defined? :/
+    def /(other)
+      :"#{self}/#{other}"
+    end
+  end
+
+  alias __old_inspect inspect
+  def inspect
+    s = self.to_s
+
+    #some things should not use this.
+    if (
+      s[0..0] == '/' ||
+      s[-1..-1] == '/' ||
+      s.include?("//") ||
+      s.include?(":")
+    )
+      return self.__old_inspect
+    end
+
+    begin
+      inspected = self.to_s.split(/\//).map { |s| ":#{s}"}.join('/')
+      return inspected
+    rescue
+      return self.__old_inspect
+    end
+  end
+
+  def <=>(other)
+    self.to_s <=> other.to_s
+  end
+end

data/lib/modesty/datastore.rb

@@ -0,0 +1,51 @@
+module Modesty
+  class Datastore
+    class ConnectionError < StandardError; end
+    def connected?
+      self.ping!
+      true
+    rescue ConnectionError
+      false
+    end
+
+    attr_reader :store
+
+    class MetricData
+      def initialize(metric)
+        @metric = metric
+      end
+    end
+
+    class ExperimentData
+      def initialize(exp)
+        @experiment = exp
+      end
+    end
+  end
+
+  module DatastoreMethods
+    def set_store(type, opts={})
+      @data = case type.to_s
+      when 'redis'
+        require File.join(Modesty::LIB, 'modesty', 'datastore', 'redis')
+        RedisData.new(opts)
+      else
+        puts "Unrecognized datastore #{type}.  Defaulting to MockRedis."
+        self.set_store :redis, :mock => true
+      end
+    end
+    alias data= set_store
+
+    def data
+      @data || set_store(:redis, :mock => true)
+    end
+
+    def handle_error(e)
+      raise e
+    end
+  end
+
+  class API
+    include DatastoreMethods
+  end
+end

data/lib/modesty/datastore/redis.rb

@@ -0,0 +1,180 @@
+module Modesty
+  class RedisData < Datastore
+    def self.date_key(date)
+      "%04d-%02d-%02d" % [date.year, date.month, date.day]
+    end
+
+    def initialize(options={})
+      if options[:redis]
+        @store = options[:redis]
+      elsif options['mock'] || options[:mock]
+        require File.join(
+          Modesty::VENDOR, 'mock_redis', 'lib', 'mock_redis.rb'
+        )
+        @store = MockRedis.new
+      else
+        $:.push(File.join(Modesty::VENDOR, 'redis-rb', 'lib'))
+          require 'redis'
+        $:.pop
+
+        @store = Redis.new(options)
+      end
+    end
+
+    def ping!
+      self.ping
+    end
+
+    def method_missing(name, *args, &blk)
+      @store.send(name, *args, &blk)
+    rescue Exception => e
+      raise ConnectionError, e.to_s
+    end
+
+    def self.keyify(*args)
+      args.map {|a| a.to_s}.map do |a|
+        (a.is_a?(Date)) ? date_key(a) : a
+      end.map do |k|
+        k.gsub(/[^\w\-\/]/,'_')
+      end.unshift('modesty').join(':')
+    end
+
+    class MetricData < Datastore::MetricData
+      def key(*args)
+        RedisData.keyify('metrics', @metric.slug, *args)
+      end
+
+      def data
+        Modesty.data
+      end
+
+      def key_for_with(*args)
+        key(:with, *args)
+      end
+
+    # -*- raw counts -*- #
+      def count_key(date)
+        key(date, :count)
+      end
+
+      def add_counts(date, count)
+        data.incrby(count_key(date),  count)
+      end
+
+      def count(date)
+        data.get(count_key(date)).to_i
+      end
+
+      def count_range(range)
+        keys = range.map { |d| count_key(d) }
+        data.mget(keys).map(&:to_i?)
+      end
+
+    # -*- unidentified users -*- #
+      def count_unidentified_user(date)
+        data.incr(unidentified_users_key(date))
+      end
+
+      def unidentified_users_key(date)
+        key_for_with(:users, date, :unidentified)
+      end
+
+      def unidentified_users(date = :all)
+        data.get(unidentified_users_key(date)).to_i
+      end
+
+      def unidentified_users_range(range)
+        keys = range.map { |d| unidentified_users_key(d) }
+        data.mget(keys).map(&:to_i?)
+      end
+
+    # -*- :with => params -*- #
+      def add_param_counts(date, count, param, id)
+      #puts "data.hincrby(#{key_for_with(param, date).inspect}, #{id.inspect}, #{count.inspect})"
+        data.hincrby(key_for_with(param, date), id, count)
+      end
+
+      def unique(param, date)
+        data.hlen(key_for_with(param, date))
+      end
+
+      def all(param, date)
+        data.hkeys(key_for_with(param, date)).map(&:to_i?)
+      end
+
+      def distribution_by(param, date)
+        dist = data.hvals(key_for_with(param, date)).histogram
+        dist.map! { |k,v| [k,v].map(&:to_i?) }
+        dist
+      end
+
+      def aggregate_by(param, date)
+        agg = data.hgetall(key_for_with(param, date))
+        agg.map! { |k,v| [k,v].map(&:to_i?) }
+        agg
+      end
+
+      def track!(count, with_args)
+        data.pipelined do
+          [:all, Date.today].each do |date|
+            self.add_counts(date, count)
+
+            self.count_unidentified_user(date) unless with_args[:user]
+
+            with_args.each do |param, id|
+              self.add_param_counts(date, count, param, id)
+            end
+          end
+        end
+      end
+
+    end
+
+    class ExperimentData < Datastore::ExperimentData
+      def data
+        Modesty.data
+      end
+
+      def key(*args)
+        RedisData.keyify(:experiments, @experiment.slug, *args)
+      end
+
+      def register!(alt, identity)
+        old_alt = self.get_cached_alternative(identity)
+        if old_alt
+          data.srem(self.key(old_alt), identity)
+        end
+        data.sadd(self.key(alt), identity)
+        return alt
+      end
+
+      def get_cached_alternative(identity)
+        @experiment.alternatives.each do |alt|
+          if data.sismember(self.key(alt), identity)
+            return alt
+          end
+        end
+        return nil
+      end
+
+      def users(alt=nil)
+        if alt.nil? #return the union
+          data.sunion(*@experiment.alternatives.map {|a| self.key(a) })
+        else
+          data.smembers(self.key(alt))
+        end.map(&:to_i)
+      end
+
+      def num_users(alt=nil)
+        if alt.nil?
+          @experiment.alternatives.map do |alt|
+            data.scard(self.key(alt)).to_i
+          end.sum
+        else
+          data.scard(self.key(alt)).to_i
+        end
+      end
+
+    end
+  end
+end