verdict 0.1.0

data/lib/verdict/group.rb

@@ -0,0 +1,36 @@
+class Verdict::Group
+  include Verdict::Metadata
+
+  attr_reader :experiment, :handle
+
+  def initialize(experiment, handle)
+    @experiment, @handle = experiment, handle.to_s
+  end
+
+  def to_s
+    handle
+  end
+
+  def to_sym
+    handle.to_sym
+  end
+
+  def ===(other)
+    case other
+      when Verdict::Group; experiment == other.experiment && other.handle == handle
+      when Symbol, String; handle == other.to_s
+      else false
+    end
+  end
+
+  def as_json(options = {})
+    {
+      handle: handle,
+      metadata: metadata
+    }
+  end
+
+  def to_json(options = {})
+    as_json(options).to_json
+  end
+end

data/lib/verdict/metadata.rb

@@ -0,0 +1,30 @@
+module Verdict::Metadata
+
+  def self.included(klass)
+    klass.send(:attr_reader, :metadata)
+  end
+
+  def name(new_name = nil)
+    @metadata ||= {}
+    return @metadata[:name] if new_name.nil?
+    @metadata[:name] = new_name
+  end
+
+  def description(new_description = nil)
+    @metadata ||= {}
+    return @metadata[:description] if new_description.nil?
+    @metadata[:description] = new_description
+  end
+
+  def screenshot(new_screenshot = nil)
+    @metadata ||= {}
+    return @metadata[:screenshot] if new_screenshot.nil?
+    @metadata[:screenshot] = new_screenshot
+  end
+
+  def owner(new_owner = nil)
+    @metadata ||= {}
+    return @metadata[:owner] if new_owner.nil?
+    @metadata[:owner] = new_owner
+  end
+end

data/lib/verdict/railtie.rb

@@ -0,0 +1,12 @@
+class Verdict::Railtie < Rails::Railtie
+  initializer "experiments.configure_rails_initialization" do |app|
+    Verdict.default_logger = Rails.logger
+    Verdict.directory = Rails.root.join('app', 'experiments')
+
+    app.config.eager_load_paths -= [Verdict.directory.to_s]
+  end
+
+  rake_tasks do
+    load File.expand_path("./tasks.rake", File.dirname(__FILE__))
+  end
+end

data/lib/verdict/segmenter.rb

@@ -0,0 +1,78 @@
+require 'digest/md5'
+
+module Verdict::Segmenter
+
+  class Base
+
+    attr_reader :experiment, :groups
+
+    def initialize(experiment)
+      @experiment = experiment
+      @groups = {}
+    end
+
+    def verify!
+    end
+
+    def group(identifier, subject, context)
+      raise NotImplementedError
+    end
+  end
+
+  class StaticPercentage < Base
+
+    class Group < Verdict::Group
+
+      attr_reader :percentile_range
+
+      def initialize(experiment, handle, percentile_range)
+        super(experiment, handle)
+        @percentile_range = percentile_range
+      end
+
+      def percentage_size
+        percentile_range.end - percentile_range.begin
+      end
+
+      def to_s
+        "#{handle} (#{percentage_size}%)"
+      end
+
+      def as_json(options = {})
+        super(options).merge(percentage: percentage_size)
+      end
+    end
+
+    def initialize(experiment)
+      super
+      @total_percentage_segmented = 0
+    end
+
+    def verify!
+      raise Verdict::SegmentationError, "Should segment exactly 100% of the cases, but segments add up to #{@total_percentage_segmented}%." if @total_percentage_segmented != 100
+    end
+
+    def group(handle, size, &block)
+      percentage = size.kind_of?(Hash) && size[:percentage] ? size[:percentage] : size
+      n = case percentage
+        when :rest; 100 - @total_percentage_segmented
+        when :half; 50
+        when Integer; percentage
+        else Integer(percentage)
+      end
+
+      group = Group.new(experiment, handle, @total_percentage_segmented ... (@total_percentage_segmented + n))
+      @groups[group.handle] = group
+      @total_percentage_segmented += n
+      group.instance_eval(&block) if block_given?
+      return group
+    end
+
+    def assign(identifier, subject, context)
+      percentile = Digest::MD5.hexdigest("#{@experiment.handle}#{identifier}").to_i(16) % 100
+      _, group = groups.find { |_, group| group.percentile_range.include?(percentile) }
+      raise Verdict::SegmentationError, "Could not get segment for subject #{identifier.inspect}!" unless group
+      group
+    end
+  end
+end

data/lib/verdict/storage.rb

@@ -0,0 +1,138 @@
+require 'json'
+
+module Verdict::Storage
+
+  class MockStorage
+
+    # Should store the assignments to allow quick lookups.
+    # - Assignments should be unique on the combination of 
+    #   `assignment.experiment.handle` and `assignment.subject_identifier`.
+    # - The main property to store is `group.handle`
+    # - Should return true if stored successfully.
+    def store_assignment(assignment)
+      false
+    end
+
+    # Should do a fast lookup of an assignment of the subject for the given experiment.
+    # - Should return nil if not found in store
+    # - Should return an Assignment instance otherwise.
+    def retrieve_assignment(experiment, subject_identifier)
+      nil
+    end
+
+    # Should remove the subject from storage, so it will be reassigned later.
+    def remove_assignment(experiment, subject_identifier)
+    end
+
+    # Should clear out the storage used for this experiment
+    def clear_experiment(experiment)
+    end
+
+    # Retrieves the start timestamp of the experiment
+    def retrieve_start_timestamp(experiment)
+      nil
+    end
+
+    # Stores the timestamp on which the experiment was started
+    def store_start_timestamp(experiment, timestamp)
+    end
+  end
+
+  class MemoryStorage
+
+    attr_reader :assignments, :start_timestamps
+
+    def initialize
+      @assignments = {}
+      @start_timestamps = {}
+    end
+
+    def store_assignment(assignment)
+      @assignments[assignment.experiment.handle] ||= {}
+      @assignments[assignment.experiment.handle][assignment.subject_identifier] = assignment.returning
+      true
+    end
+
+    def retrieve_assignment(experiment, subject_identifier)
+      experiment_store = @assignments[experiment.handle] || {}
+      experiment_store[subject_identifier]
+    end
+
+    def remove_assignment(experiment, subject_identifier)
+      @assignments[assignment.experiment.handle] ||= {}
+      @assignments[assignment.experiment.handle].delete(subject_identifier)
+    end
+
+    def clear_experiment(experiment)
+      @assignments.delete(experiment.handle)
+    end
+
+    def retrieve_start_timestamp(experiment)
+      @start_timestamps[experiment.handle]
+    end
+
+    def store_start_timestamp(experiment, timestamp)
+      @start_timestamps[experiment.handle] = timestamp
+    end
+  end
+
+  class RedisStorage
+
+    attr_accessor :redis, :key_prefix
+
+    def initialize(redis = nil, options = {})
+      @redis = redis
+      @key_prefix = options[:key_prefix] || 'experiments/'
+    end
+
+    def retrieve_assignment(experiment, subject_identifier)
+      if value = redis.hget(generate_experiment_key(experiment), subject_identifier)
+        hash = JSON.parse(value)
+        experiment.subject_assignment(
+          subject_identifier, 
+          experiment.group(hash['group']),
+          DateTime.parse(hash['created_at']).to_time
+        )
+      end
+    rescue ::Redis::BaseError => e
+      raise Verdict::StorageError, "Redis error: #{e.message}"
+    end
+
+    def store_assignment(assignment)
+      hash = { group: assignment.handle, created_at: assignment.created_at }
+      redis.hset(generate_experiment_key(assignment.experiment), assignment.subject_identifier, JSON.dump(hash))
+    rescue ::Redis::BaseError => e
+      raise Verdict::StorageError, "Redis error: #{e.message}"
+    end
+
+    def remove_assignment(experiment, subject_identifier)
+      redis.hdel(generate_experiment_key(experiment), subject_identifier)
+    end
+
+    def clear_experiment(experiment)
+      redis.del(generate_experiment_key(experiment))
+      redis.del(generate_experiment_start_timestamp_key(experiment))
+    end
+
+    def retrieve_start_timestamp(experiment)
+      if started_at = redis.get(generate_experiment_start_timestamp_key(experiment))
+        DateTime.parse(started_at).to_time
+      end
+    end
+
+    def store_start_timestamp(experiment, timestamp)
+      redis.setnx(generate_experiment_start_timestamp_key(experiment), timestamp.to_s)
+    end
+
+
+    private
+
+    def generate_experiment_key(experiment)
+      "#{@key_prefix}#{experiment.handle}"
+    end
+
+    def generate_experiment_start_timestamp_key(experiment)
+      "#{@key_prefix}#{experiment.handle}/started_at"
+    end
+  end
+end

data/lib/verdict/tasks.rake

@@ -0,0 +1,59 @@
+def require_env(key)
+  value = ENV[key.downcase].presence || ENV[key.upcase].presence
+  raise "Provide #{key} as environment variable" if value.blank?
+  value
+end
+
+namespace :experiments do
+  
+  desc "List all defined experiments"
+  task :list => 'environment' do
+    length = Experiments.repository.keys.map(&:length).max
+    Experiments.repository.each do |_, experiment|
+      print "#{experiment.handle.ljust(length)} | "
+      print "Groups: #{experiment.groups.values.map(&:to_s).join(', ')}"
+      puts
+    end
+  end
+
+  desc "Looks up the assignment for a given experiment and subject"
+  task :lookup_assignment => 'environment' do
+    experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
+    subject_identifier = require_env('subject')
+    assignment = experiment.lookup_assignment_for_identifier(subject_identifier)
+    if assignment.nil?
+      puts "Subject #{ENV['subject']} is not assigned to experiment #{experiment.handle} yet."
+    elsif assignment.qualified?
+      puts "Subject #{ENV['subject']} is assigned to group `#{assignment.group.handle}`"
+    else
+      puts "Subject #{ENV['subject']} is unqualified for experiment #{experiment.handle}"
+    end
+  end
+
+  desc "Manually assign a subject to a given group in an experiment"
+  task :assign_manually => 'environment' do
+    experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
+    group = experiment.group(require_env('group')) or raise "Group not found"
+    assignment = experiment.subject_assignment(require_env('subject'), group, false)
+    experiment.store_assignment(assignment)
+  end
+
+  desc "Disqualify a subject from an experiment"
+  task :disqualify => 'environment' do
+    experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
+    assignment = experiment.subject_assignment(require_env('subject'), nil, false)
+    experiment.store_assignment(assignment)
+  end
+
+  desc "Removes the assignment for a subject so it will be reassigned to the experiment."
+  task :remove_assignment => 'environment' do
+    experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
+    experiment.remove_subject_identifier(require_env('subject'))
+  end
+
+  desc "Runs the cleanup tasks for an experiment"
+  task :wrapup => 'environment' do
+    experiment = Experiments[require_env('experiment')] or raise "Experiment not found"
+    experiment.wrapup
+  end
+end

data/lib/verdict/version.rb

@@ -0,0 +1,3 @@
+module Verdict
+  VERSION = "0.1.0"
+end

data/test/assignment_test.rb

@@ -0,0 +1,76 @@
+require 'test_helper'
+require 'json'
+
+class AssignmentTest < MiniTest::Unit::TestCase
+
+  def setup
+    @experiment = Verdict::Experiment.new('assignment test')
+    @group = Verdict::Group.new(@experiment, :control)
+  end
+
+  def test_basic_properties
+    assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', @group, Time.now.utc)
+    assert_equal 'test_subject_id', assignment.subject_identifier
+    assert_equal @experiment, assignment.experiment 
+    assert_equal @group, assignment.group 
+    assert assignment.returning?
+    assert assignment.qualified?
+    assert_equal :control, assignment.to_sym
+    assert_equal 'control', assignment.handle
+    assert_kind_of Time, assignment.created_at
+
+    non_assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, nil)
+    assert_equal nil, non_assignment.group 
+    assert !non_assignment.returning?
+    assert !non_assignment.qualified?
+    assert_equal nil, non_assignment.to_sym
+    assert_equal nil, non_assignment.handle
+    assert_kind_of Time, assignment.created_at
+  end
+
+  def test_subject_lookup
+    assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, Time.now.utc)
+    assert_raises(NotImplementedError) { assignment.subject }
+
+    @experiment.expects(:fetch_subject).with('test_subject_id').returns(subject = mock('subject'))
+    assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, Time.now.utc)
+    assert_equal subject, assignment.subject
+  end
+
+  def test_triple_equals
+    assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', @group, Time.now.utc)
+    assert !(assignment === nil)
+    assert assignment === @group
+    assert assignment === 'control'
+    assert assignment === :control
+
+    non_assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, Time.now.utc)
+    assert non_assignment === nil
+    assert !(non_assignment === @group)
+    assert !(non_assignment === 'control')
+    assert !(non_assignment === :control)
+  end
+
+  def test_json_representation
+    assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', @group, Time.new(2013, 1, 1, 0, 0, 0, '+00:00'))
+    json = JSON.parse(assignment.to_json)
+
+    assert_equal 'assignment test', json['experiment']
+    assert_equal 'test_subject_id', json['subject']
+    assert_equal true, json['qualified']
+    assert_equal true, json['returning']
+    assert_equal 'control', json['group']
+    assert_equal '2013-01-01T00:00:00Z', json['created_at']
+
+    Timecop.freeze(Time.new(2012, 1, 1, 0, 0, 0, '+00:00')) do
+      non_assignment = Verdict::Assignment.new(@experiment, 'test_subject_id', nil, nil)
+      json = JSON.parse(non_assignment.to_json)
+      assert_equal 'assignment test', json['experiment']
+      assert_equal 'test_subject_id', json['subject']
+      assert_equal false, json['qualified']
+      assert_equal false, json['returning']
+      assert_equal nil, json['group']
+      assert_equal '2012-01-01T00:00:00Z', json['created_at']
+    end
+  end
+end

data/test/conversion_test.rb

@@ -0,0 +1,37 @@
+require 'test_helper'
+require 'json'
+
+class ConversionTest < MiniTest::Unit::TestCase
+
+  def setup
+    @experiment = Verdict::Experiment.new('conversion test') do
+      groups { group :all, 100 }
+    end
+  end
+
+  def test_subject_lookup
+    conversion = Verdict::Conversion.new(@experiment, 'test_subject_id', :test_goal)
+    assert_raises(NotImplementedError) { conversion.subject }
+
+    @experiment.expects(:fetch_subject).with('test_subject_id').returns(subject = mock('subject'))
+    conversion = Verdict::Conversion.new(@experiment, 'test_subject_id', :test_goal)
+    assert_equal subject, conversion.subject
+  end
+
+  def test_assignment_lookup
+    @experiment.subject_storage.expects(:retrieve_assignment).with(@experiment, 'test_subject_id')
+    conversion = Verdict::Conversion.new(@experiment, 'test_subject_id', :test_goal)
+    conversion.assignment
+  end
+
+  def test_json_representation
+    conversion = Verdict::Conversion.new(@experiment, 'test_subject_id', :test_goal, Time.new(2013, 1, 1, 4, 5, 6, '+00:00'))
+    json = JSON.parse(conversion.to_json)
+
+    assert_equal 'conversion test',      json['experiment']
+    assert_equal 'test_subject_id',      json['subject']
+    assert_equal 'test_goal',            json['goal']
+    assert_equal 'test_goal',            json['goal']
+    assert_equal '2013-01-01T04:05:06Z', json['created_at']
+  end
+end