edhd 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Kyle Maxwell
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,40 @@
+= edhd
+
+Experiment A/B framework for Ruby.  Thread-safety is left as 
+an exercise to the user.  I use globals.  Shoot me.  Supports 
+nested experiments.
+
+== Overview
+
+Experiments are defined as follows (look at the specs for more details):
+
+* Exactly one of the hypotheses blocks in an experiment will run.
+* A hypothesis will be chosen with probability proportional to its weight.
+* You can pass in a seed for the random number generator to ensure that, for example, a given user always sees the same hypothesis.
+* An experiment block will log which hypothesis actually runs by calling $observer.hypothesized(experiment_name, hypothesis_name, seed)
+* Calling observation(name, value, seed) will log to $observer.observed(name, value, seed).
+* $observer = IOObserver(STDOUT) will give you a sample observer, but you'll probably want to log to a database or something.  Exercise left to the user.
+* Statistical analysis is an exercise left to the user.
+    
+    # Method signatures
+    # def experiment(name = "default", seed = rand, &block)
+    # def hypothesis(name, weight = 1, &block)
+    # def observation(name, value, seed)
+    
+    # Example
+    experiment "hello world", current_user.login do
+      hypothesis "a1", 3 do
+        @a = 1
+      end
+    
+      hypothesis "a2" do
+        @a = 2
+      end
+    end
+
+    observation "value of a", @a, current_user.login
+
+
+== Copyright
+
+Copyright (c) 2009 Kyle Maxwell. MIT Licensed

data/Rakefile

@@ -0,0 +1,45 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "edhd"
+    gem.summary = "Simple AB testing"
+    gem.description = "Simple AB testing"
+    gem.email = "kyle@kylemaxwell.com"
+    gem.homepage = "http://github.com/fizx/edhd"
+    gem.authors = ["Kyle Maxwell"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "edhd #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/edhd.rb

@@ -0,0 +1,24 @@
+require File.dirname(__FILE__) + "/io_observer"
+module Kernel
+  $observer = IOObserver.new(STDOUT)
+
+  def experiment(name = "default", seed = rand)
+    $hypothesis_sets ||= []
+    $hypothesis_sets << []
+    yield
+    hypothesis_set = $hypothesis_sets.pop
+    hypothesis_name, block = hypothesis_set[seed.to_s.hash % hypothesis_set.length]
+    block.call
+    $observer.hypothesized(name, hypothesis_name, seed)
+  end
+  
+  def hypothesis(name, weight = 1, &block)
+    weight.times do
+      $hypothesis_sets.last << [name, block]
+    end
+  end
+  
+  def observation(name, value, seed)
+    $observer.observed(name, value, seed)
+  end
+end

data/lib/io_observer.rb

@@ -0,0 +1,13 @@
+class IOObserver
+  def initialize(io)
+    @io = io
+  end
+  
+  def hypothesized(experiment, hypothesis, seed)
+    @io.puts "#{Time.now.to_i}\thypothesis\t#{experiment}\t#{hypothesis}\t#{seed}"
+  end
+  
+  def observed(name, value, seed)
+    @io.puts "#{Time.now.to_i}\tobservation\t#{name}\t#{value}\t#{seed}"
+  end
+end

data/spec/edhd_spec.rb

@@ -0,0 +1,160 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+class QuietObserver
+  def method_missing(*args);end
+end
+
+describe "Edhd" do
+  before do
+    @a = 0
+    @b = 0
+    $observer = QuietObserver.new
+  end
+  
+  it "should define #experiment" do
+    Kernel.should respond_to(:experiment)
+  end
+  
+  it "should define #hypothesis" do
+    Kernel.should respond_to(:hypothesis)
+  end
+
+  it "should define #observation" do
+    Kernel.should respond_to(:observation)
+  end
+  
+  context "one hypothesis" do
+    it "should execute the hypothesis" do 
+      experiment do
+        hypothesis "a" do
+          @a = 1
+        end
+      end
+      @a.should == 1
+    end
+  end
+  
+  context "multiple hypotheses" do
+    it "should execute only one" do
+      experiment do
+        hypothesis "a" do
+          @a += 1
+        end
+        hypothesis "b" do
+          @a += 1
+        end
+      end
+      @a.should == 1
+    end
+    
+    it "should execute randomly if no seed provided" do
+      100.times do
+        experiment do
+          hypothesis "a" do
+            @a += 1
+          end
+          hypothesis "b" do
+            @b += 1
+          end
+        end
+      end
+      @a.should > 0
+      @b.should > 0
+      (@a + @b).should == 100
+    end
+    
+    it "should execute deterministically if a seed is provided" do
+      100.times do
+        experiment "seeded experiment", "ADSF" do
+          hypothesis "a" do
+            @a += 1
+          end
+          hypothesis "b" do
+            @b += 1
+          end
+        end
+      end
+      @a.should == 0
+      @b.should == 100
+    end
+    
+    it "should call hypotheses more often if weighted higher" do
+      5.times do 
+        @a = 0
+        @b = 0
+        100.times do
+          experiment do
+            hypothesis "a", 3 do
+              @a += 1
+            end
+            hypothesis "b" do
+              @b += 1
+            end
+          end
+        end
+        @a.should > @b
+      end
+    end
+  end
+  
+  context "with an observer" do
+    class TestObserver
+      def initialize(events = [])
+        @events = events
+      end
+      def hypothesized(*args)
+        @events << [:hypothesis] + args
+      end
+      def observed(*args)
+        @events << [:observation] + args
+      end
+    end
+    
+    before do
+      $observer = TestObserver.new(@events = [])
+    end
+    
+    it "should record the hypothesis chosen" do
+      experiment "name", "seed" do
+        hypothesis "a", 3 do
+          @a += 1
+        end
+        hypothesis "b" do
+          @b += 1
+        end
+      end
+      @events.length.should == 1
+      type, exp, hypo, seed = @events.first
+      type.should == :hypothesis
+      exp.should == "name"
+      hypo.should == "a"
+      seed.should == "seed"
+    end
+    
+    it "should record observations" do
+      experiment "name", "seed" do
+        hypothesis "a", 3 do
+          @a += 1
+        end
+        hypothesis "b" do
+          @b += 1
+        end
+      end
+      observation "value of a", @a, "seed"
+      
+      @events.length.should == 2
+      type, exp, hypo, seed = @events.first
+      type.should == :hypothesis
+      exp.should == "name"
+      hypo.should == "a"
+      seed.should == "seed"
+      
+      type, name, value, seed = @events.last
+      type.should == :observation
+      name.should == "value of a"
+      value.should == 1
+      seed.should == "seed"
+    end
+    
+  end
+end

data/spec/io_observer_spec.rb

@@ -0,0 +1,18 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "IOObserver" do
+  
+  before do 
+    @observer = IOObserver.new(StringIO.new(@string = ""))
+  end
+  
+  it "should log to the io stream" do
+    @observer.hypothesized "experiment", "hypothesis", "seed"
+    @string.length.should > 0
+  end
+  
+  it "should log to the io stream" do
+    @observer.observed "event", "value", "seed"
+    @string.length.should > 0
+  end
+end

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'edhd'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: edhd
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Kyle Maxwell
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-30 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.2.9
+    version: 
+description: Simple AB testing
+email: kyle@kylemaxwell.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/edhd.rb
+- lib/io_observer.rb
+- spec/edhd_spec.rb
+- spec/io_observer_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/fizx/edhd
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Simple AB testing
+test_files: 
+- spec/edhd_spec.rb
+- spec/io_observer_spec.rb
+- spec/spec_helper.rb