modesty 0.1.0

data/lib/modesty/metric/data.rb

@@ -0,0 +1,133 @@
+module Modesty
+  class Metric
+
+    def data
+      @data ||= (Modesty.data.class)::MetricData.new(self)
+    end
+
+    def parse_date(date)
+      if date.is_a? Symbol
+        return date if date == :all
+        Date.send(date)
+      elsif date.nil?
+        Date.today
+      else
+        date.to_date
+      end
+    end
+
+    def parse_date_or_range(start=nil,fin=nil)
+      if fin
+        parse_date(start)..parse_date(fin)
+      elsif start.is_a?(Range)
+        parse_date(start.first)..parse_date(start.last)
+      else
+        parse_date(start)
+      end
+    end
+
+
+    def count(*dates)
+      date_or_range = parse_date_or_range(*dates)
+
+      case date_or_range
+      when Range
+        if self.data.respond_to? :count_by_range
+          self.data.count_range(date_or_range)
+        else
+          date_or_range.map do |date|
+            self.data.count(date)
+          end
+        end
+      when Date, :all
+        self.data.count(date_or_range)
+      end
+    end
+
+    # for grep:
+    # def all
+    # def unique
+    # def distribution_by
+    # def aggregate_by
+    [
+      :all,
+      :unique,
+      :distribution_by,
+      :aggregate_by
+    ].each do |data_type|
+      by_range = :"#{data_type}_range"
+      define_method(data_type) do |sym, *dates|
+        sym = sym.to_s.singularize.to_sym
+        date_or_range = (dates.empty?) ? :all : parse_date_or_range(*dates)
+        if date_or_range.is_a? Range
+          if self.data.respond_to?(by_range)
+            return self.data.send(by_range, sym, date_or_range)
+          else
+            return date_or_range.map do |date|
+              self.data.send(data_type, sym, date)
+            end
+          end
+        elsif date_or_range.is_a?(Date) || date_or_range == :all
+          return self.data.send(data_type, sym, date_or_range)
+        end
+      end
+    end
+
+    # def distribution
+    # def aggregate
+    # helpers to use the default context if it's an experiment-generated metric
+    # otherwise default to users
+    %w(
+      distribution
+      aggregate
+    ).each do |data_type|
+      define_method data_type do |*dates|
+        context = if @experiment
+          @experiment.identity_for(self.parent)
+        else
+          :user
+        end
+        send("#{data_type}_by", context, *dates)
+      end
+    end
+
+    def track!(count=1, options={})
+      if count.is_a? Hash
+        options = count
+        count = options[:count] || 1
+      end
+
+      with = options[:with] || {}
+
+      with[:user] ||= Modesty.identity if Modesty.identity
+
+      self.experiments.each do |exp|
+        # only track the for the experiment group if
+        # the user has previously hit the experiment
+        identity_slug = exp.identity_for(self)
+        identity = with[identity_slug]
+
+        if !identity && exp.metric_contexts.has_key?(self.slug)
+          raise IdentityError, <<-msg.squish
+            #{exp.inspect} requires #{self.inspect} to be tracked
+            with #{identity_slug.to_s.singularize.to_sym.inspect}.
+
+            It was tracked :with => #{with.inspect}
+          msg
+        end
+
+        if identity
+          alt = exp.data.get_cached_alternative(identity)
+          if alt
+            (self/(exp.slug/alt)).data.track!(count, with)
+          end
+        end
+      end
+
+      self.data.track!(count, with)
+      @parent.track!(count, :with => with) if @parent
+    rescue Datastore::ConnectionError => e
+      Modesty.handle_error(e)
+    end
+  end
+end

data/modesty.gemspec

@@ -0,0 +1,192 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{modesty}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jay Adkisson", "Kevin Ball", "Kristj\303\241n P\303\251tursson"]
+  s.date = %q{2011-04-26}
+  s.description = %q{Modesty is simple and scalable split testing and event tracking framework. It was inspired by assaf's Vanity (github.com/assaf/vanity).}
+  s.email = %q{jay@causes.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "init.rb",
+    "lib/modesty.rb",
+    "lib/modesty/api.rb",
+    "lib/modesty/core_ext.rb",
+    "lib/modesty/core_ext/array.rb",
+    "lib/modesty/core_ext/fixnum.rb",
+    "lib/modesty/core_ext/hash.rb",
+    "lib/modesty/core_ext/string.rb",
+    "lib/modesty/core_ext/symbol.rb",
+    "lib/modesty/datastore.rb",
+    "lib/modesty/datastore/redis.rb",
+    "lib/modesty/experiment.rb",
+    "lib/modesty/experiment/base.rb",
+    "lib/modesty/experiment/builder.rb",
+    "lib/modesty/experiment/console.rb",
+    "lib/modesty/experiment/data.rb",
+    "lib/modesty/experiment/interface.rb",
+    "lib/modesty/experiment/significance.rb",
+    "lib/modesty/experiment/stats.rb",
+    "lib/modesty/frameworks/rails.rb",
+    "lib/modesty/identity.rb",
+    "lib/modesty/load.rb",
+    "lib/modesty/load/load_experiments.rb",
+    "lib/modesty/load/load_metrics.rb",
+    "lib/modesty/metric.rb",
+    "lib/modesty/metric/base.rb",
+    "lib/modesty/metric/builder.rb",
+    "lib/modesty/metric/data.rb",
+    "modesty.gemspec",
+    "spec/core_ext_spec.rb",
+    "spec/experiment_spec.rb",
+    "spec/identity_spec.rb",
+    "spec/load_spec.rb",
+    "spec/metric_spec.rb",
+    "spec/rails_spec.rb",
+    "spec/redis_spec.rb",
+    "spec/significance_spec.rb",
+    "spec/spec.opts",
+    "test/myapp/config/modesty.yml",
+    "test/myapp/modesty/experiments/cookbook.rb",
+    "test/myapp/modesty/metrics/kitchen_metrics.rb",
+    "test/myapp/modesty/metrics/stove/burner_metrics.rb",
+    "vendor/.piston.yml",
+    "vendor/mock_redis/.gitignore",
+    "vendor/mock_redis/README",
+    "vendor/mock_redis/lib/mock_redis.rb",
+    "vendor/mock_redis/lib/mock_redis/hash.rb",
+    "vendor/mock_redis/lib/mock_redis/list.rb",
+    "vendor/mock_redis/lib/mock_redis/misc.rb",
+    "vendor/mock_redis/lib/mock_redis/set.rb",
+    "vendor/mock_redis/lib/mock_redis/string.rb",
+    "vendor/redis-rb/.gitignore",
+    "vendor/redis-rb/LICENSE",
+    "vendor/redis-rb/README.markdown",
+    "vendor/redis-rb/Rakefile",
+    "vendor/redis-rb/benchmarking/logging.rb",
+    "vendor/redis-rb/benchmarking/pipeline.rb",
+    "vendor/redis-rb/benchmarking/speed.rb",
+    "vendor/redis-rb/benchmarking/suite.rb",
+    "vendor/redis-rb/benchmarking/thread_safety.rb",
+    "vendor/redis-rb/benchmarking/worker.rb",
+    "vendor/redis-rb/examples/basic.rb",
+    "vendor/redis-rb/examples/dist_redis.rb",
+    "vendor/redis-rb/examples/incr-decr.rb",
+    "vendor/redis-rb/examples/list.rb",
+    "vendor/redis-rb/examples/pubsub.rb",
+    "vendor/redis-rb/examples/sets.rb",
+    "vendor/redis-rb/examples/unicorn/config.ru",
+    "vendor/redis-rb/examples/unicorn/unicorn.rb",
+    "vendor/redis-rb/lib/redis.rb",
+    "vendor/redis-rb/lib/redis/client.rb",
+    "vendor/redis-rb/lib/redis/compat.rb",
+    "vendor/redis-rb/lib/redis/connection.rb",
+    "vendor/redis-rb/lib/redis/distributed.rb",
+    "vendor/redis-rb/lib/redis/hash_ring.rb",
+    "vendor/redis-rb/lib/redis/pipeline.rb",
+    "vendor/redis-rb/lib/redis/subscribe.rb",
+    "vendor/redis-rb/redis.gemspec",
+    "vendor/redis-rb/test/commands_on_hashes_test.rb",
+    "vendor/redis-rb/test/commands_on_lists_test.rb",
+    "vendor/redis-rb/test/commands_on_sets_test.rb",
+    "vendor/redis-rb/test/commands_on_sorted_sets_test.rb",
+    "vendor/redis-rb/test/commands_on_strings_test.rb",
+    "vendor/redis-rb/test/commands_on_value_types_test.rb",
+    "vendor/redis-rb/test/connection_handling_test.rb",
+    "vendor/redis-rb/test/db/.gitignore",
+    "vendor/redis-rb/test/distributd_key_tags_test.rb",
+    "vendor/redis-rb/test/distributed_blocking_commands_test.rb",
+    "vendor/redis-rb/test/distributed_commands_on_hashes_test.rb",
+    "vendor/redis-rb/test/distributed_commands_on_lists_test.rb",
+    "vendor/redis-rb/test/distributed_commands_on_sets_test.rb",
+    "vendor/redis-rb/test/distributed_commands_on_strings_test.rb",
+    "vendor/redis-rb/test/distributed_commands_on_value_types_test.rb",
+    "vendor/redis-rb/test/distributed_commands_requiring_clustering_test.rb",
+    "vendor/redis-rb/test/distributed_connection_handling_test.rb",
+    "vendor/redis-rb/test/distributed_internals_test.rb",
+    "vendor/redis-rb/test/distributed_persistence_control_commands_test.rb",
+    "vendor/redis-rb/test/distributed_publish_subscribe_test.rb",
+    "vendor/redis-rb/test/distributed_remote_server_control_commands_test.rb",
+    "vendor/redis-rb/test/distributed_sorting_test.rb",
+    "vendor/redis-rb/test/distributed_test.rb",
+    "vendor/redis-rb/test/distributed_transactions_test.rb",
+    "vendor/redis-rb/test/encoding_test.rb",
+    "vendor/redis-rb/test/helper.rb",
+    "vendor/redis-rb/test/internals_test.rb",
+    "vendor/redis-rb/test/lint/hashes.rb",
+    "vendor/redis-rb/test/lint/internals.rb",
+    "vendor/redis-rb/test/lint/lists.rb",
+    "vendor/redis-rb/test/lint/sets.rb",
+    "vendor/redis-rb/test/lint/sorted_sets.rb",
+    "vendor/redis-rb/test/lint/strings.rb",
+    "vendor/redis-rb/test/lint/value_types.rb",
+    "vendor/redis-rb/test/persistence_control_commands_test.rb",
+    "vendor/redis-rb/test/pipelining_commands_test.rb",
+    "vendor/redis-rb/test/publish_subscribe_test.rb",
+    "vendor/redis-rb/test/redis_mock.rb",
+    "vendor/redis-rb/test/remote_server_control_commands_test.rb",
+    "vendor/redis-rb/test/sorting_test.rb",
+    "vendor/redis-rb/test/test.conf",
+    "vendor/redis-rb/test/thread_safety_test.rb",
+    "vendor/redis-rb/test/transactions_test.rb",
+    "vendor/redis-rb/test/unknown_commands_test.rb",
+    "vendor/redis-rb/test/url_param_test.rb"
+  ]
+  s.homepage = %q{http://github.com/causes/modesty}
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.6.2}
+  s.summary = %q{Modesty is simple and scalable split testing and event tracking framework.}
+  s.test_files = [
+    "spec/core_ext_spec.rb",
+    "spec/experiment_spec.rb",
+    "spec/identity_spec.rb",
+    "spec/load_spec.rb",
+    "spec/metric_spec.rb",
+    "spec/rails_spec.rb",
+    "spec/redis_spec.rb",
+    "spec/significance_spec.rb",
+    "test/myapp/modesty/experiments/cookbook.rb",
+    "test/myapp/modesty/metrics/kitchen_metrics.rb",
+    "test/myapp/modesty/metrics/stove/burner_metrics.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<redis>, [">= 0"])
+      s.add_development_dependency(%q<rake>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_runtime_dependency(%q<redis>, [">= 0"])
+    else
+      s.add_dependency(%q<redis>, [">= 0"])
+      s.add_dependency(%q<rake>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<redis>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<redis>, [">= 0"])
+    s.add_dependency(%q<rake>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<redis>, [">= 0"])
+  end
+end
+

data/spec/core_ext_spec.rb

@@ -0,0 +1,17 @@
+require 'modesty'
+
+describe Symbol do
+  [
+    :abc,
+    :abc=,
+    :a/:bc/:def,
+    :/,
+    :"////",
+    :':',
+    :"http://www.google.com/",
+  ].each do |sym|
+    it "eval(:\"#{sym}\".inspect) == :\"#{sym}\"" do
+      eval(sym.inspect).should == sym
+    end
+  end
+end

data/spec/experiment_spec.rb

@@ -0,0 +1,239 @@
+require 'modesty'
+
+describe Modesty::Experiment, "creating an experiment" do
+  before :each do
+    Modesty.metrics.clear
+    Modesty.experiments.clear
+    Modesty.new_metric :foo do |m|
+      m.description "Foo"
+      m.submetric :bar do |m|
+        m.description "Bar"
+      end
+    end
+
+    Modesty.new_metric :baz do |m|
+      m.description "Baz"
+    end
+
+    @e = Modesty.new_experiment(:creation_page) do |m|
+      m.description "Three versions of the creation page"
+      m.alternatives :heavyweight, :lightweight
+      m.metrics :foo/:bar, :baz
+    end 
+
+    @f = Modesty.new_experiment :ab_test do |e|
+      e.description "only two groups"
+      e.metrics :baz
+    end
+  end
+
+  it "can create an experiment with a block" do
+    Modesty.experiments.should include :creation_page
+    Modesty.experiments[:creation_page].should == @e
+
+    @e.metrics.should include Modesty.metrics[:foo/:bar]
+    @e.metrics.should include Modesty.metrics[:baz]
+    @e.alternatives.should == [:control, :heavyweight, :lightweight]
+    @e.description.should == "Three versions of the creation page"
+  end
+
+  it "uses [:control, :experiment] as the default experiment groups" do
+    @f.alternatives.should == [:control, :experiment]
+  end
+
+  it "auto-creates metrics" do
+    Modesty.metrics.keys.should include :foo/:bar/:creation_page/:control
+    Modesty.metrics.keys.should include :foo/:bar/:creation_page/:heavyweight
+    Modesty.metrics.keys.should include :foo/:bar/:creation_page/:lightweight
+    Modesty.metrics.keys.should include :baz/:creation_page/:control
+    Modesty.metrics.keys.should include :baz/:creation_page/:heavyweight
+    Modesty.metrics.keys.should include :baz/:creation_page/:lightweight
+
+    Modesty.metrics.keys.should include :baz/:ab_test/:control
+    Modesty.metrics.keys.should include :baz/:ab_test/:experiment
+  end
+
+  it "grabs metrics nicely with e.metrics(alt)" do
+    @e.alternatives.each do |alt|
+      @e.metrics(alt).should == {
+        :foo/:bar => Modesty.metrics[:foo/:bar/:creation_page/alt],
+        :baz      => Modesty.metrics[:baz/:creation_page/alt],
+      }
+    end
+
+    lambda do
+      @e.metrics(:i_dont_exist)
+    end.should raise_error(Modesty::Experiment::Error)
+  end
+end
+
+describe "A/B testing" do
+  before :all do
+    Modesty.set_store :redis, :mock => true
+    Modesty.data.flushdb
+  end
+
+  before :each do
+    @e = Modesty.experiments[:creation_page]
+  end
+
+  it "Selects evenly between alternatives" do
+    (0..(3*100-1)).each do |i|
+      Modesty.identify! i
+      Modesty.experiment :creation_page do |exp|
+        [:control, :lightweight, :heavyweight].each do |alt|
+          exp.group alt do
+            Modesty.track! :baz
+          end
+        end
+      end
+      Modesty.metrics[:baz].count.should == 1+i
+    end
+  end
+
+  it "raises Modesty::Experiment::MissingError if exp is missing" do
+    lambda do
+      Modesty.group(:idontexist)
+    end.should raise_error(Modesty::Experiment::Error)
+  end
+
+  it "assigns guests to :control" do
+    Modesty.identify! nil
+    Modesty.group(:ab_test).should == :control
+    Modesty.group?(:ab_test/:control).should == true
+
+    Modesty.identify! 200
+  end
+
+  it "can use a passed-in identity" do
+    leet = Modesty.experiment :ab_test, :on => 1337 do |exp|
+      [:control, :experiment].each do |alt|
+        exp.group alt do
+          Modesty.identity
+        end
+      end
+    end
+
+    leet.should == 1337
+  end
+
+  it "can used a passed-in nil identity" do
+    test = Modesty.experiment :ab_test, :on => nil do |exp|
+      exp.group :control do
+        Modesty.identity
+      end
+      exp.group :experiment do
+        "fail"
+      end
+    end
+
+    test.should be_nil
+  end
+
+  it "tracks the users in each experimental group" do
+    e = Modesty.experiments[:creation_page]
+    lambda { e.users }.should_not raise_error
+    u = e.users
+    u.should be_a Array
+    (1..(3*100-1)).each do |i|
+      u.should include i
+    end
+  end
+
+  it "tracks the number of users in each experimental group" do
+    e = Modesty.experiments[:creation_page]
+    e.num_users.should == 3*100
+    [:lightweight, :control, :heavyweight].each do |alt|
+      e.num_users(alt).should be_close 3*100/4, 2 + 3*100/6
+    end
+  end
+
+  it "uses cached alternative" do
+    # should ask Redis for the correct group
+    # instead of running generate_group
+    (1..10).each do |i|
+      Modesty.identify! i
+      Modesty.group :creation_page
+    end
+
+    @e.should_not_receive(:generate_group)
+    (1..10).each do |i|
+      Modesty.identify! i
+      Modesty.group :creation_page
+    end
+  end
+
+  it "tracks the experiment group if you've hit the experiment" do
+    Modesty.identify! 500 #not in the experiment group yet
+    [:control, :lightweight, :heavyweight].each do |alt|
+      lambda do
+        Modesty.track! :baz
+      end.should_not change(Modesty.metrics[:baz/:creation_page/alt], :count)
+    end
+
+    alt = Modesty.group :creation_page
+    lambda do
+      Modesty.track! :baz, 5
+    end.should change(Modesty.metrics[:baz/:creation_page/alt], :count).by(5)
+  end
+
+  it "allows for manually setting your experiment group" do
+    Modesty.identify! 50
+    e = Modesty.experiments[:creation_page]
+    2.times do
+      e.alternatives.each do |alt|
+        lambda do
+          e.chooses alt
+          e.group.should == alt
+        end.should_not change(e, :num_users)
+      end
+    end
+  end
+end
+
+describe "Datastore failing" do
+  before :each do
+    Modesty.data.flushdb
+    Modesty.experiments.clear
+    Modesty.metrics.clear
+
+    Modesty.new_metric :foo
+
+    Modesty.new_experiment :my_exp do |e|
+      e.metric :foo
+    end
+
+    Modesty.data.stub!(:get_cached_alternative).and_raise(
+      Modesty::Datastore::ConnectionError
+    )
+
+    Modesty.data.stub!(:register!).and_raise(
+      Modesty::Datastore::ConnectionError
+    )
+  end
+
+  it "fails gracefully for choose_group" do
+    Modesty.identify! 9876
+    first_group = second_group = nil
+    lambda do
+      first_group = Modesty.group :my_exp
+    end.should_not raise_error(Modesty::Datastore::ConnectionError)
+
+    lambda do
+      second_group = Modesty.group :my_exp
+    end.should_not raise_error(Modesty::Datastore::ConnectionError)
+
+    #should be consistent
+    first_group.should == second_group
+  end
+end
+
+describe "regression for 740de0b1f" do
+  it "doesn't blow up on startup" do
+    Modesty.api.instance_variable_set("@experiments", nil)
+    lambda do
+      Modesty.new_experiment :foo do
+      end
+    end.should_not raise_error
+  end
+end