modesty 0.1.0

data/lib/modesty/experiment/stats.rb

@@ -0,0 +1,163 @@
+module Modesty
+  class Experiment
+
+    def stats
+      @stats ||= Hash.new do |hash, key|
+        raise Error, <<-msg.squish
+          Unrecognized stat #{key.inspect}
+        msg
+      end
+    end
+
+    def reports(*args)
+      self.stats.values.map { |s| s.report(*args) }
+    end
+
+    def aggregates(metric, *args)
+      metric = metric.slug if metric.is_a? Metric
+      context = self.identity_for(metric)
+      self.alternatives.hashmap do |a|
+        agg = self.metrics(a)[metric].aggregate_by(context, *args)
+        agg = agg.sum if agg.is_a?(Array)
+        self.users(a).hashmap { 0 }.merge!(agg)
+      end
+    end
+
+    def distributions(metric, *args)
+      aggregates(metric, *args).map_values! do |agg|
+        agg.values.histogram
+      end
+    end
+
+    def dist_analysis(metric, *args)
+      Significance.dist_significance(
+        distributions(metric, *args)
+      )
+    end
+
+    class Builder
+      def distribution(name, options={}, &blk)
+        @exp.stats[name] = DistributionStat.new(@exp, name, options, &blk)
+      end
+
+      def conversion(name, options={}, &blk)
+        @exp.stats[name] = ConversionStat.new(@exp, name, options, &blk)
+      end
+    end
+
+    class ArgumentProxy
+      def initialize(obj, *args)
+        @obj = obj
+        @args = args
+      end
+
+      def inspect
+        "#<ArgumentProxy[ #{@obj.inspect} ]>"
+      end
+
+      def method_missing(meth, *args)
+        data = @obj.send(meth, *(args + @args))
+        # [Jay] #TODO: Hack alert!
+        # this doesn't take into account Metric#all,
+        # which returns an Array for either a date range
+        # or a single day
+        data = data.sum if data.is_a?(Array)
+        data
+      end
+    end
+
+    class Stat
+      def initialize(exp, name, options={}, &blk)
+        @exp = exp
+        @name = name
+        @get_data = blk || default_get_data(options[:on])
+      end
+
+      def title
+        @name.to_s.split(/_/).map(&:capitalize).join(' ')
+      end
+
+      def report(*args)
+        sig = significance(*args)
+        sig = "not significant" if sig.nil?
+        return <<-report
+
+          === #{title} ===
+          #{analysis(*args).inspect}
+          Significance: #{sig}
+        report
+      end
+
+      def significant?(tolerance=0.01)
+        sig = self.significance
+        !sig.nil? && sig <= tolerance
+      end
+
+      private
+      def argument_proxy_hash(hsh, *args)
+        hsh.map_values do |v|
+          ArgumentProxy.new(v, *args)
+        end
+      end
+
+      def data_for(alt, *args)
+        data = @get_data.call(argument_proxy_hash(@exp.metrics(alt), *args))
+      end
+    end
+
+    class DistributionStat < Stat
+      def default_get_data(on_param)
+        lambda do |metrics|
+          metrics[on_param].distribution
+        end
+      end
+
+      def inspect
+        "#<Modesty::Experiment::DistributionStat[ (on #{@exp.slug}) (of #{@metric_sym.inspect}) ]>"
+      end
+
+      def data(*args)
+        @exp.alternatives.hashmap do |a|
+          data_for(a, *args)
+        end
+      end
+
+      def analysis(*args)
+        Significance.dist_significance(data(*args))
+      end
+
+      def significance(*args)
+        analysis(*args)[:significant]
+      end
+
+    end
+
+    class ConversionStat < Stat
+      def default_get_data(on_param)
+        lambda do |metrics|
+          num_count = metrics[on_param[0]].count
+          denom_count = metrics[on_param[1]].count
+          [num_count, denom_count - num_count]
+        end
+      end
+
+      def analysis(*args)
+        @exp.alternatives.hashmap do |a|
+          data_for(a, *args)
+        end
+      end
+
+      def data(*args)
+        analysis.values
+      end
+
+      def inspect
+        "#<Modesty::Experiment::ConversionStat[ #{@name} ]>"
+      end
+
+      def significance(*args)
+        Significance.significance(*data(*args))
+      end
+    end
+  end
+end

data/lib/modesty/frameworks/rails.rb

@@ -0,0 +1,27 @@
+if Rails.version.match('^3')
+  module Modesty
+    class Railtie < Rails::Railtie
+      initializer "modesty.initialize" do |app|
+        Modesty.root = File.join(Rails.root, 'modesty')
+        Modesty.config_path = File.join(Rails.root, 'config', 'modesty.yml')
+        Modesty.environment = Rails.env
+      end
+
+      config.after_initialize do
+        begin
+          Modesty.load_with_redis!(config.redis)
+        rescue NoMethodError
+          Modesty.load!
+        end
+      end
+
+    end
+  end
+else
+  Modesty.root = File.join(Rails.root, 'modesty')
+  Modesty.config_path = File.join(Rails.root, 'config', 'modesty.yml')
+  Modesty.environment = Rails.env
+  Rails.configuration.after_initialize do
+    Modesty.load!
+  end
+end

data/lib/modesty/identity.rb

@@ -0,0 +1,32 @@
+module Modesty
+  #TODO: cookies n stuff
+  
+  class IdentityError < RuntimeError; end
+
+  module IdentityMethods
+    attr_reader :identity
+
+    def identify!(id, opts={})
+      unless opts[:ignore]
+        raise(
+          IdentityError,
+          "Identity must be an integer or nil."
+        ) unless id.nil? || id.is_a?(Fixnum)
+
+        @identity = id
+      end
+    end
+
+    def with_identity(id)
+      old_identity = Modesty.identity
+      Modesty.identify! id
+      ret = yield
+      Modesty.identify! old_identity
+      ret
+    end
+  end
+
+  class API
+    include IdentityMethods
+  end
+end

data/lib/modesty/load.rb

@@ -0,0 +1,80 @@
+module Modesty
+  module LoadMethods
+    attr_writer :root
+    def root
+      @root ||= File.join(
+        File.dirname(__FILE__),
+        '..'
+      )
+      #TODO: is there a better default?
+    end
+
+    attr_writer :config_path
+    def config_path
+      @config_path ||= File.join(
+        Modesty.root,
+        '../config/modesty.yml'
+      )
+    end
+
+    attr_accessor :environment
+
+    def load_options(quiet = false)
+      options = begin
+        YAML.load(File.read(self.config_path))
+      rescue Errno::ENOENT
+        puts "No Modesty config file found" unless quiet
+        {}
+      end
+      options[self.environment] || options['default'] || options
+    end
+
+    def load_paths(options)
+      if options['paths']
+        options['paths'].each do |data, path|
+          Modesty.send("#{data}_dir=", File.join(Modesty.root, path))
+        end
+      end
+    end
+
+    def load_config!
+      options = load_options
+      load_paths(options)
+
+      if options['datastore'] && options['datastore']['type']
+        type = options['datastore'].delete('type')
+        data_options = Hash[
+          options['datastore'].map { |k,v| [k.to_sym, v] }
+        ]
+        self.set_store(type, data_options)
+      else
+        self.set_store :redis, :mock => true
+      end
+    end
+
+    def _load_with_redis(redis)
+      options = load_options(true)
+      load_paths(options)
+      self.set_store(:redis, :redis => redis)
+    end
+
+    def load!
+      load_config!
+      load_all_metrics!
+      load_all_experiments!
+    end
+
+    def load_with_redis!(redis)
+      _load_with_redis(redis)
+      load_all_metrics!
+      load_all_experiments!
+    end
+  end
+
+  class API
+    include LoadMethods
+  end
+end
+
+require 'modesty/load/load_experiments.rb'
+require 'modesty/load/load_metrics.rb'

data/lib/modesty/load/load_experiments.rb

@@ -0,0 +1,14 @@
+module Modesty
+  module LoadMethods
+    attr_writer :experiments_dir
+    def experiments_dir
+      @experiments_dir ||= File.join(Modesty.root, 'experiments')
+    end
+
+    def load_all_experiments!
+      Dir.glob(
+        File.join(self.experiments_dir, '*.rb')
+      ).each { |f| load f }
+    end
+  end
+end

data/lib/modesty/load/load_metrics.rb

@@ -0,0 +1,17 @@
+module Modesty
+  module LoadMethods
+    attr_writer :metrics_dir
+    def metrics_dir
+      @metrics_dir ||= File.join(
+        Modesty.root,
+        'metrics'
+      )
+    end
+
+    def load_all_metrics!
+      Dir.glob(
+        File.join(self.metrics_dir, '**', '*.rb')
+      ).each { |f| load f }
+    end
+  end
+end

data/lib/modesty/metric.rb

@@ -0,0 +1,56 @@
+module Modesty
+  class Metric
+    class Error < StandardError; end
+  end
+
+  module MetricMethods
+    attr_writer :metrics
+
+    def metrics
+      @metrics ||= Hash.new do |h, k|
+        raise Metric::Error, <<-msg.squish
+          Unrecognized metric #{k.inspect}
+        msg
+      end
+    end
+
+    def metrics_starting_with(name)
+      self.metrics.select{|k, v| k.to_s.starts_with?(name)}
+    end
+
+    def add_metric(metric)
+      raise Metric::Error, <<-msg if self.metrics.include? metric.slug
+        Metric #{metric.slug.inspect} already defined!
+      msg
+      self.metrics[metric.slug] = metric
+    end
+
+    def new_metric(slug, parent=nil, options={}, &block)
+      if parent.is_a? Hash
+        options=parent
+      else
+        options[:parent] = parent
+      end
+
+      metric = Metric.new(slug, options)
+      yield Metric::Builder.new(metric) if block_given?
+      add_metric(metric)
+      metric
+    end
+
+    # Tracking
+    def track!(name, *args)
+      self.metrics[name.to_sym].track! *args
+    rescue Modesty::Metric::Error
+      # Fail silently in the event that a metric is not found.
+    end
+  end
+
+  class API
+    include MetricMethods
+  end
+end
+
+require 'modesty/metric/base'
+require 'modesty/metric/builder'
+require 'modesty/metric/data'

data/lib/modesty/metric/base.rb

@@ -0,0 +1,38 @@
+module Modesty
+  class Metric
+
+    class << self
+      private
+      def data_type(name)
+      end
+    end
+
+    ATTRIBUTES = [
+      :description
+    ]
+    attr_reader *ATTRIBUTES
+    attr_reader :slug
+    attr_reader :parent
+    attr_reader :experiment
+
+    # metrics should know what experiments use them,
+    # to enable smart tracking.
+    def experiments
+      @experiments ||= []
+    end
+
+    def initialize(slug, options={})
+      @slug = slug
+      @parent = options[:parent]
+      @experiment = options[:experiment]
+    end
+
+    def inspect
+      "#<Modesty::Metric[ #{self.slug.inspect} ]>"
+    end
+
+    def /(sym)
+      Modesty.metrics[@slug/sym] || (raise NoMetricError, "Undefined metric :'#{@slug/sym}'")
+    end
+  end
+end

data/lib/modesty/metric/builder.rb

@@ -0,0 +1,23 @@
+module Modesty
+  class Metric
+    class Builder
+
+      def method_missing(name, *args)
+        if Metric::ATTRIBUTES.include? name
+          @metric.instance_variable_set("@#{name}", args[0])
+        else
+          super
+        end
+      end
+
+      def initialize(metric)
+        @metric = metric
+      end
+
+      def submetric(slug, &blk)
+        Modesty.new_metric(@metric.slug/slug, @metric, &blk)
+      end
+
+    end
+  end
+end