dat-science 0.0.0

data/.gitignore

@@ -0,0 +1,4 @@
+/*.gem
+/.bundle
+/.ruby-version
+/Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 GitHub, Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,106 @@
+# Dat Science!
+
+A Ruby library for carefully refactoring critical paths. Science isn't
+a feature flipper or an A/B testing tool, it's a pattern that helps
+measure and validate large code changes without altering behavior.
+
+## How do I do science?
+
+```ruby
+require "dat/science"
+
+include Dat::Science
+
+science "user-permissions" do |experiment|
+  experiment.control   { model.check_user(user).valid? }
+  experiment.candidate { user.can? :read, model }
+end
+```
+
+Wrap a `control` block around the code's original behavior, and wrap
+`candidate` around the new behavior. The `science` block will return
+whatever the `control` block returns, but it does a bunch of stuff
+behind the scenes:
+
+* Decides whether or not to run `candidate`,
+* Runs `candidate` before `control` 50% of the time,
+* Measures the duration of both behaviors,
+* Compares the results of both behaviors,
+* Swallows any exceptions raised by the candidate behavior, and
+* Publishes all this information for tracking and reporting.
+
+## Making Science Useful
+
+(Talk about subclassing `Dat::Science::Experiment` and setting
+`Dat::Science.experiment`)
+
+```ruby
+require "dat/science"
+
+module FooCorp
+  class Experiment < Dat::Science::Experiment
+    def enabled?
+      # See "Ramping up Experiments" below.
+    end
+
+    def publish(name, payload)
+      # See "Publishing Results" below.
+    end
+  end
+end
+```
+
+```ruby
+Dat::Science.experiment = FooCorp::Experiment
+```
+
+### Ramping up Experiments
+
+```ruby
+def enabled?
+  rand(100) < 10
+end
+```
+
+```ruby
+def enabled?
+  Flipper[name].enabled?
+end
+```
+
+### Publishing Results
+
+```ruby
+def publish(name, payload)
+  FooCorp.instrument "science.#{name}", payload
+end
+```
+
+```ruby
+{
+  :candidate => {
+    :duration  => 2.5,
+    :exception => nil,
+    :value     => 42
+  },
+
+  :control => {
+    :duration  => 25.0,
+    :exception => nil,
+    :value     => 24
+  },
+
+  :first => :control
+}
+```
+
+#### Adding Context
+
+(using `e.context`)
+
+## Hacking on Science
+
+Make sure a modern Bundler is available.`script/test` runs the unit
+tests. All development dependencies will be installed automatically if
+they're not available. Dat science happens primarily on Ruby 1.9.3 and
+1.8.7, but science should be universal.

data/dat-science.gemspec

@@ -0,0 +1,17 @@
+Gem::Specification.new do |gem|
+  gem.name          = "dat-science"
+  gem.version       = "0.0.0"
+  gem.authors       = ["John Barnette", "Rick Bradley"]
+  gem.email         = ["jbarnette@github.com"]
+  gem.description   = "Gradually test, measure, and track refactored code."
+  gem.summary       = "SO BRAVE WITH SCIENCE."
+  gem.homepage      = "https://github.com/github/dat-science"
+
+  gem.files         = `git ls-files`.split $/
+  gem.executables   = []
+  gem.test_files    = gem.files.grep /^test/
+  gem.require_paths = ["lib"]
+
+  gem.add_development_dependency "minitest"
+  gem.add_development_dependency "mocha"
+end

data/lib/dat/science.rb

@@ -0,0 +1,33 @@
+require "dat/science/experiment"
+
+module Dat
+
+  # Public: Include this module if you like science.
+  module Science
+
+    # Public: Do some science.
+    def science(name, &block)
+      Science.experiment.new(name, &block).run
+    end
+
+    # Public: The Class to use for all `science` experiments. Default is
+    # `Dat::Science::Experiment`.
+    def self.experiment
+      @experiment ||= Dat::Science::Experiment
+    end
+
+    # Public: Set the Class to use for all `science` experiments.
+    # Returns `klass`.
+    def self.experiment=(klass)
+      @experiment = klass
+    end
+
+    # Internal: Reset any static configuration (primarily
+    # `Dat::Science.experiment`. Returns `self`.
+    def self.reset
+      @experiment = nil
+
+      self
+    end
+  end
+end

data/lib/dat/science/experiment.rb

@@ -0,0 +1,135 @@
+require "dat/science/result"
+
+module Dat
+  module Science
+
+    # Public: Try things in code.
+    class Experiment
+
+      # Public: The name of this experiment.
+      attr_reader :name
+
+      # Public: Create a new experiment instance. `self` is yielded to
+      # an optional `block` if it's provided.
+      def initialize(name, &block)
+        @candidate  = nil
+        @context    = { :experiment => name }
+        @control    = nil
+        @name       = name
+
+        yield self if block_given?
+      end
+
+      # Public: Add a Hash of `payload` data to be included when events
+      # are published.
+      def context(payload)
+        @context.merge! payload
+      end
+
+      # Public: Declare the control behavior `block` for this
+      # experiment. Returns `block`.
+      def control(&block)
+        @control = block
+      end
+
+      # Public: Declare the candidate behavior `block` for this
+      # experiment. Returns `block`.
+      def candidate(&block)
+        @candidate = block
+      end
+
+      # Public: Run the control and candidate behaviors, timing each and
+      # comparing the results. The run order is randomized. Returns the
+      # control behavior's result.
+      #
+      # If the experiment is disabled or candidate behavior isn't
+      # provided the control behavior's result will be returned
+      # immediately.
+      def run
+        return run_control unless candidate? && enabled?
+
+        control_goes_first = rand(2) == 0
+
+        if control_goes_first
+          control   = observe_control
+          candidate = observe_candidate
+        else
+          candidate = observe_candidate
+          control   = observe_control
+        end
+
+        payload = {
+          :candidate => candidate.payload,
+          :control   => control.payload,
+          :first     => control_goes_first ? :control : :candidate
+        }
+
+        kind = control == candidate ? "match" : "mismatch"
+        publish_with_context kind, payload
+
+        raise control.exception if control.raised?
+
+        control.value
+      end
+
+      protected
+
+      # Internal: Does this experiment have candidate behavior?
+      def candidate?
+        !!@candidate
+      end
+
+      # Internal: Is this experiment enabled? More specifically, should
+      # the candidate behavior be run and compared to the control
+      # behavior? The default implementation returns `true`.
+      def enabled?
+        true
+      end
+
+      # Internal: Run `block`, measuring the duration and rescuing any
+      # raised exceptions. Returns a Dat::Science::Result.
+      def observe(&block)
+        start = Time.now
+
+        begin
+          value = block.call
+        rescue => ex
+          raised = ex
+        end
+
+        duration = (Time.now - start) * 1000
+        Science::Result.new value, duration, raised
+      end
+
+
+      # Internal. Returns a Dat::Science::Result for `candidate`.
+      def observe_candidate
+        observe { run_candidate }
+      end
+
+      # Internal. Returns a Dat::Science::Result for `control`.
+      def observe_control
+        observe { run_control }
+      end
+
+      # Internal: Broadcast an event `name` and `payload` Hash. The
+      # default implementation is a no-op. Returns nothing.
+      def publish(name, payload)
+      end
+
+      # Internal: Call `publish`, merging the `payload` with `context`.
+      def publish_with_context(name, payload)
+        publish name, @context.merge(payload)
+      end
+      # Internal: Run the candidate behavior and return its result.
+      def run_candidate
+        @candidate.call
+      end
+
+      # Internal: Run the control behavior and return its result.
+      def run_control
+        @control.call
+      end
+    end
+  end
+end

data/lib/dat/science/result.rb

@@ -0,0 +1,44 @@
+module Dat
+
+  # Internal. The value of a watched behavior.
+  module Science
+    class Result
+      attr_reader :duration
+      attr_reader :exception
+      attr_reader :value
+
+      def initialize(value, duration, exception)
+        @duration  = duration
+        @exception = exception
+        @value     = value
+      end
+
+      def ==(other)
+        return false unless other.is_a? Science::Result
+
+        values_are_equal = other.value == value
+        both_raised      = other.raised? && raised?
+        neither_raised   = !other.raised? && !raised?
+
+        exceptions_are_equivalent =
+          both_raised && other.exception.class == self.exception.class &&
+          other.exception.message == self.exception.message
+
+        (values_are_equal && neither_raised) ||
+          (both_raised && exceptions_are_equivalent)
+      end
+
+      def hash
+        exception ^ value
+      end
+
+      def payload
+        { :duration => duration, :exception => exception, :value => value }
+      end
+
+      def raised?
+        !!exception
+      end
+    end
+  end
+end

data/script/bootstrap

@@ -0,0 +1,9 @@
+#!/bin/sh
+# Ensure local dependencies are available.
+
+set -e
+
+cd $(dirname "$0")/..
+rm -rf .bundle/{binstubs,config}
+
+bundle install --binstubs .bundle/binstubs --path .bundle --quiet "$@"

data/script/test

@@ -0,0 +1,9 @@
+#!/bin/sh
+# Run the unit tests.
+
+set -e
+
+cd $(dirname "$0")/..
+  script/bootstrap && ruby -I lib -r rubygems \
+    -e 'require "bundler/setup"' \
+    -e '(ARGV.empty? ? Dir["test/**/*_test.rb"] : ARGV).each { |f| load f }' -- "$@"

data/test/dat_science_experiment_test.rb

@@ -0,0 +1,8 @@
+require "minitest/autorun"
+require "dat/science/experiment"
+
+class DatScienceExperimentTest < MiniTest::Unit::TestCase
+  def test_sanity
+    assert Dat::Science::Experiment
+  end
+end

data/test/dat_science_test.rb

@@ -0,0 +1,32 @@
+require "minitest/autorun"
+require "mocha/setup"
+require "dat/science"
+
+class DatScienceTest < MiniTest::Unit::TestCase
+  def teardown
+    Dat::Science.reset
+  end
+
+  def test_experiment_default
+    assert_equal Dat::Science::Experiment, Dat::Science.experiment
+  end
+
+  def test_experiment
+    Dat::Science.experiment = :foo
+    assert_equal :foo, Dat::Science.experiment
+  end
+
+  def test_science
+    experiment = mock do
+      expects(:run).returns 42
+    end
+
+    Dat::Science.experiment.expects(:new).with("foo").returns experiment
+
+    obj = Object.new
+    obj.extend Dat::Science
+
+    ret = obj.science "foo"
+    assert_equal 42, ret
+  end
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: dat-science
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+  prerelease: 
+platform: ruby
+authors:
+- John Barnette
+- Rick Bradley
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gradually test, measure, and track refactored code.
+email:
+- jbarnette@github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- dat-science.gemspec
+- lib/dat/science.rb
+- lib/dat/science/experiment.rb
+- lib/dat/science/result.rb
+- script/bootstrap
+- script/test
+- test/dat_science_experiment_test.rb
+- test/dat_science_test.rb
+homepage: https://github.com/github/dat-science
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: SO BRAVE WITH SCIENCE.
+test_files:
+- test/dat_science_experiment_test.rb
+- test/dat_science_test.rb