verdict 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MDEzN2VlY2MyNGUyZmQ0MzY2YWViYTQ5NmRjMmVhZjUzMmFjZDNlYg==
+  data.tar.gz: !binary |-
+    YmVlMzgxMDAxZDRjOGI2NmI0MzY2OTNkYzkxOGY2Y2VmNjFkM2EyMg==
+SHA512:
+  metadata.gz: !binary |-
+    MzJhMDlmYTRiMjI4NTRiZDI5ZGM4YWY2MTNkNTQxNzFiYjZkZDdkNGE1MmJm
+    NDdjZTEzMWU1NTBmNDllNDJhODRhYTcxYTNiYTgwOTI2NjA5ZmIyYTVlN2E1
+    ZjZjNzVkZGQyNTBlNGFiNDYzYmI2YTFjOWU4MDRmZWQzMjk2MjE=
+  data.tar.gz: !binary |-
+    MGZmYTI3ODA4ZGMzMzhlMTM4ZThlYWZkMGU3NWE3YWYzZmY4NzY2ZDk0OGQ2
+    ZDEyYTUzN2VhYmU4MTdlNjRlY2M4OWI1MmJlOWMwNmZhMDRkMzg0OWQ2NGE1
+    ZGEwMmY3ODFjNDUwNjdlNGZmMDM4NmUyOWRjMmRlZGFiMjM4NTQ=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+script: bundle exec rake
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - ruby-head
+  - rbx
+  - jruby-19mode
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+services:
+  - redis-server

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in experiments.gemspec
+gemspec
+gem "rubysl", platform: :rbx
+gem "json", platform: :rbx

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012-2013 Shopify Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,89 @@
+# Verdict
+
+This library allows you to define and use experiments in your application.
+
+- This library can be used in any Ruby application, and comes with a `Railtie` to
+  make integrating it with a Rails app easy.
+- This library only handles consistently assigning subjects to experiment groups, 
+  and storing/logging these assignments for analysis. It doesn't do any analysis
+  of results. That should happen elsewhere, e.g. in a data warehouse environment.
+
+
+## Installation
+
+Add this line to your application's Gemfile, and run `bundle install`:
+
+    gem 'verdict'
+
+## Usage
+
+This gem contains the `Experiments::Experiment` model used create the experiment instance,
+in order consistently modify application behaviour based on an object's unique key. 
+
+Define an experiment like so:
+
+``` ruby
+Verdict::Experiment.define :my_experiment do
+
+  # This block should return true if the subject is qualified to participate
+  qualify { |subject, context|  ... }
+
+  # Specify the groups and the percentages
+  groups do
+    group :test, :half
+    group :control, :rest
+  end
+
+  # Specify how assignments will be stored.
+  storage Verdict::Storage::MemoryStorage.new
+end
+```
+
+Usually you want to place this in a file called **my_experiment.rb** in the 
+**/app/experiments** folder. Also, usually you want to subclass `Experiments::Experiment` 
+to set some default options for your app's environment, and call `define` on that class
+instead.
+
+Refer to the experiment elsewhere in your codebase like this:
+
+``` ruby
+context = { ... } # anything you want to pass along to the qualify block. 
+case Experiments['my experiment'].switch(shop, context)
+when :test
+  # Handle test group
+when :control
+  # Handle control group
+else 
+  # Handle unqualified subjects. 
+end
+```
+
+## Storage & logging
+
+The library uses a basic interface to store experiment assignments. Except for
+a development-only memory store, it doesn't include any storage models.
+
+You can set up storage by calling the `storage` method of your experiment, with
+an object that responds to the following tho methods:
+
+- `def retrieve_assignment(experiment, subject_identifier)`
+- `def store_assignment(assignment)`
+
+In which `experiment` is the Experiment instance, `subject_identifier` is a  
+string that uniquely identifies the subject, and `assignment` is an
+`Experiment::Assignment` instance. By default it will use `subject.id.to_s` as
+`subject_identifier`, but you can change that by overriding the 
+`def subject_identifier(subject)` method on the experiment.
+
+The library will also log every assignment to `Experiments.logger`. The Railtie
+sets `Experiment.logger` to `Rails.logger`, so experiment assignments will show
+up in your Rails log. You can override the logging by overriding the 
+`def log_assignment(assignment)` method on the experiment.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes, including tests (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request, and mention @wvanbergen.

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.test_files = Dir['test/**/*_test.rb']
+  t.libs << 'test'
+  t.verbose = true
+end
+
+task default: :test

data/lib/verdict.rb

@@ -0,0 +1,54 @@
+require 'logger'
+
+module Verdict
+  extend self
+
+  attr_accessor :default_logger, :directory
+
+  def [](handle)
+    Verdict.repository[handle.to_s]
+  end
+
+  def repository
+    if @repository.nil?
+      @repository = {}
+      discovery
+    end
+
+    @repository
+  end
+
+  def discovery
+    Dir[File.join(Verdict.directory, '**', '*.rb')].each { |f| require f } if @directory
+  end
+
+  class Error < StandardError; end
+  class SegmentationError < Verdict::Error; end
+  class InvalidSubject < Verdict::Error; end
+  class EmptySubjectIdentifier < Verdict::Error; end
+  class StorageError < Verdict::Error; end
+
+  class ExperimentHandleNotUnique < Verdict::Error
+    attr_reader :handle
+
+    def initialize(handle)
+      @handle = handle
+      super("Another experiment with handle #{handle.inspect} is already defined!")
+    end
+  end
+end
+
+require "verdict/version"
+require "verdict/railtie" if defined?(Rails)
+
+require "verdict/metadata"
+require "verdict/experiment"
+require "verdict/group"
+require "verdict/assignment"
+require "verdict/conversion"
+require "verdict/segmenter"
+require "verdict/storage"
+require "verdict/event_logger"
+
+Verdict.default_logger ||= Logger.new("/dev/null")
+Verdict.directory = nil

data/lib/verdict/assignment.rb

@@ -0,0 +1,61 @@
+class Verdict::Assignment
+
+  attr_reader :experiment, :subject_identifier, :group, :created_at
+
+  def initialize(experiment, subject_identifier, group, originally_created_at)
+    @experiment         = experiment
+    @subject_identifier = subject_identifier
+    @group              = group
+    @returning          = !originally_created_at.nil?
+    @created_at         = originally_created_at || Time.now.utc
+  end
+
+  def subject
+    @subject ||= experiment.fetch_subject(subject_identifier)
+  end
+
+  def qualified?
+    !group.nil?
+  end
+
+  def returning
+    self.class.new(@experiment, @subject_identifier, @group, @created_at)
+  end
+
+  def returning?
+    @returning
+  end
+
+  def handle
+    qualified? ? group.handle : nil
+  end
+
+  def to_sym
+    qualified? ? group.to_sym : nil
+  end  
+
+  def as_json(options = {})
+    {
+      experiment: experiment.handle,
+      subject:    subject_identifier,
+      qualified:  qualified?,
+      returning:  returning?,
+      group:      qualified? ? group.handle : nil,
+      created_at: created_at.utc.strftime('%FT%TZ')
+    }
+  end
+
+  def to_json(options = {})
+    as_json(options).to_json
+  end
+
+  def ===(other)
+    case other
+      when nil; !qualified?
+      when Verdict::Assignment; other.group === group
+      when Verdict::Group; other === group
+      when Symbol, String; qualified? ? group.handle == other.to_s : false
+      else false
+    end
+  end
+end

data/lib/verdict/conversion.rb

@@ -0,0 +1,32 @@
+class Verdict::Conversion
+
+  attr_reader :experiment, :subject_identifier, :goal, :created_at
+
+  def initialize(experiment, subject_identifier, goal, created_at = Time.now.utc)
+    @experiment = experiment
+    @subject_identifier = subject_identifier
+    @goal = goal
+    @created_at = created_at
+  end
+
+  def subject
+    experiment.fetch_subject(subject_identifier)
+  end
+
+  def assignment
+    experiment.fetch_assignment(subject_identifier)
+  end
+
+  def as_json(options = {})
+    {
+      experiment: experiment.handle,
+      subject:    subject_identifier,
+      goal:       goal,
+      created_at: created_at.utc.strftime('%FT%TZ')
+    }
+  end
+
+  def to_json(options = {})
+    as_json(options).to_json
+  end
+end

data/lib/verdict/event_logger.rb

@@ -0,0 +1,20 @@
+class Verdict::EventLogger
+  attr_reader :logger, :level
+
+  def initialize(logger, level = :info)
+    @logger, @level = logger, level
+  end
+
+  def log_assignment(assignment)
+    status = assignment.returning? ? 'returning' : 'new'
+    if assignment.qualified?
+      logger.send(level, "[Verdict::Assignment] experiment=#{assignment.experiment.handle} subject=#{assignment.subject_identifier} status=#{status} qualified=true group=#{assignment.group.handle}")
+    else
+      logger.send(level, "[Verdict::Assignment] experiment=#{assignment.experiment.handle} subject=#{assignment.subject_identifier} status=#{status} qualified=false")
+    end 
+  end
+
+  def log_conversion(conversion)
+    logger.send(level, "[Verdict::Conversion] experiment=#{conversion.experiment.handle} subject=#{conversion.subject_identifier} goal=#{conversion.goal}")
+  end
+end

data/lib/verdict/experiment.rb

@@ -0,0 +1,220 @@
+class Verdict::Experiment
+
+  include Verdict::Metadata
+
+  attr_reader :handle, :qualifier, :subject_storage, :event_logger
+
+  def self.define(handle, *args, &block)
+    experiment = self.new(handle, *args, &block)
+    raise Verdict::ExperimentHandleNotUnique.new(experiment.handle) if Verdict.repository.has_key?(experiment.handle)
+    Verdict.repository[experiment.handle] = experiment
+  end
+
+  def initialize(handle, options = {}, &block)
+    @handle = handle.to_s
+
+    options = default_options.merge(options)
+    @qualifier         = options[:qualifier]
+    @event_logger      = options[:event_logger] || Verdict::EventLogger.new(Verdict.default_logger)
+    @subject_storage   = options[:storage] || Verdict::Storage::MemoryStorage.new
+    @store_unqualified = options[:store_unqualified]
+    @segmenter         = options[:segmenter]
+    @subject_type      = options[:subject_type]
+    instance_eval(&block) if block_given?
+  end
+
+  def subject_type(type = nil)
+    return @subject_type if type.nil?
+    @subject_type = type
+  end
+
+  def store_unqualified?
+    @store_unqualified
+  end
+
+  def group(handle)
+    segmenter.groups[handle.to_s]
+  end
+
+  def groups(segmenter_class = Verdict::Segmenter::StaticPercentage, &block)
+    return @segmenter.groups unless block_given?
+    @segmenter ||= segmenter_class.new(self)
+    @segmenter.instance_eval(&block)
+    @segmenter.verify!
+    return self
+  end
+
+  def qualify(&block)
+    @qualifier = block
+  end
+
+  def storage(subject_storage, options = {})
+    @store_unqualified = options[:store_unqualified] if options.has_key?(:store_unqualified)
+    @subject_storage = subject_storage
+  end
+
+  def segmenter
+    raise Verdict::Error, "No groups defined for experiment #{@handle.inspect}." if @segmenter.nil?
+    @segmenter
+  end
+
+  def started_at
+    @started_at ||= @subject_storage.retrieve_start_timestamp(self)
+  end
+
+  def started?
+    !@started_at.nil?
+  end
+
+  def group_handles
+    segmenter.groups.keys
+  end
+
+  def subject_assignment(subject_identifier, group, originally_created_at = nil)
+    Verdict::Assignment.new(self, subject_identifier, group, originally_created_at)
+  end
+
+  def subject_conversion(subject_identifier, goal, created_at = Time.now.utc)
+    Verdict::Conversion.new(self, subject_identifier, goal, created_at)
+  end
+
+  def convert(subject, goal)
+    identifier = retrieve_subject_identifier(subject)
+    conversion = subject_conversion(identifier, goal)
+    event_logger.log_conversion(conversion)
+    conversion
+  end
+
+  def assign(subject, context = nil)
+    identifier = retrieve_subject_identifier(subject)
+    assignment = if store_unqualified?
+      assignment_with_unqualified_persistence(identifier, subject, context)
+    else
+      assignment_without_unqualified_persistence(identifier, subject, context)
+    end
+
+    store_assignment(assignment)
+  rescue Verdict::StorageError
+    subject_assignment(identifier, nil, nil)
+  end
+
+  def disqualify(subject)
+    identifier = retrieve_subject_identifier(subject)
+    store_assignment(subject_assignment(identifier, nil, nil))
+  end
+
+  def store_assignment(assignment)
+    @subject_storage.store_assignment(assignment) if should_store_assignment?(assignment)
+    event_logger.log_assignment(assignment)
+    assignment
+  end
+
+  def remove_subject(subject)
+    remove_subject_identifier(retrieve_subject_identifier(subject))
+  end
+
+  def remove_subject_identifier(subject_identifier)
+    @subject_storage.remove_assignment(self, subject_identifier)
+  end
+
+  def switch(subject, context = nil)
+    assign(subject, context).to_sym
+  end
+
+  def lookup(subject)
+    lookup_assignment_for_identifier(retrieve_subject_identifier(subject))
+  end
+
+  def lookup_assignment_for_identifier(subject_identifier)
+    fetch_assignment(subject_identifier)
+  end
+
+  def wrapup
+    @subject_storage.clear_experiment(self)
+  end
+
+  def retrieve_subject_identifier(subject)
+    identifier = subject_identifier(subject).to_s
+    raise Verdict::EmptySubjectIdentifier, "Subject resolved to an empty identifier!" if identifier.empty?
+    identifier
+  end
+
+  def has_qualifier?
+    !@qualifier.nil?
+  end
+
+  def everybody_qualifies?
+    !has_qualifier?
+  end
+
+  def as_json(options = {})
+    data = {
+      handle: handle,
+      has_qualifier: has_qualifier?,
+      groups: segmenter.groups.values.map { |g| g.as_json(options) },
+      metadata: metadata,
+      started_at: started_at.nil? ? nil : started_at.utc.strftime('%FT%TZ')
+    }
+
+    data.tap do |data|
+      data[:subject_type] = subject_type.to_s unless subject_type.nil?
+    end
+  end
+
+  def to_json(options = {})
+    as_json(options).to_json
+  end 
+
+  def fetch_subject(subject_identifier)
+    raise NotImplementedError, "Fetching subjects based in identifier is not implemented for eperiment @{handle.inspect}."
+  end
+
+  def fetch_assignment(subject_identifier)
+    @subject_storage.retrieve_assignment(self, subject_identifier)
+  end
+
+  protected
+
+  def default_options
+    {}
+  end
+
+  def should_store_assignment?(assignment)
+    !assignment.returning? && (store_unqualified? || assignment.qualified?)
+  end
+
+  def assignment_with_unqualified_persistence(subject_identifier, subject, context)
+    fetch_assignment(subject_identifier) || (
+      subject_qualifies?(subject, context) ? 
+        subject_assignment(subject_identifier, @segmenter.assign(subject_identifier, subject, context), nil) :
+        subject_assignment(subject_identifier, nil, nil)
+    )
+  end
+
+  def assignment_without_unqualified_persistence(subject_identifier, subject, context)
+    if subject_qualifies?(subject, context)
+      fetch_assignment(subject_identifier) ||
+        subject_assignment(subject_identifier, @segmenter.assign(subject_identifier, subject, context), nil)
+    else 
+      subject_assignment(subject_identifier, nil, nil)
+    end
+  end  
+
+  def subject_identifier(subject)
+    subject.respond_to?(:id) ? subject.id : subject.to_s
+  end
+
+  def subject_qualifies?(subject, context = nil)
+    ensure_experiment_has_started
+    everybody_qualifies? || @qualifier.call(subject, context)
+  end
+
+  def set_start_timestamp
+    @subject_storage.store_start_timestamp(self, started_now = Time.now.utc)
+    started_now
+  end
+
+  def ensure_experiment_has_started
+    @started_at ||= @subject_storage.retrieve_start_timestamp(self) || set_start_timestamp
+  end
+end