lab_tech 0.1.0

data/app/models/lab_tech/summary.rb

@@ -0,0 +1,183 @@
+module LabTech
+  class Summary
+    TAB  = " " * 4
+    LINE = "-" * 80
+    VAL = "█"
+    DOT = "·"
+
+    def initialize(experiment)
+      @experiment = experiment
+    end
+
+    def to_s
+      if experiment.results.count.zero?
+        return [ LINE, "No results for experiment #{@experiment.name.inspect}", LINE ].join("\n")
+      end
+
+      fetch_data
+
+      s = StringIO.new
+      s.puts LINE, "Experiment: #{@experiment.name}", LINE
+
+      add_time_span_to s
+      add_counts_to s
+
+      if @time_deltas.any?
+        add_time_deltas_to s
+        add_speedup_chart_to s
+      end
+
+      s.puts LINE
+      return s.string
+    end
+
+    private
+
+    attr_reader :experiment
+
+    def add_counts_to(s)
+      s.puts
+      summarize_count( s, :correct )
+      summarize_count( s, :mismatched )
+      summarize_count( s, :timeout, "timed out" )
+      summarize_count( s, :errored, "raised errors" )
+    end
+
+    def add_speedup_chart_to(s)
+      s.puts
+      s.puts "Speedups (by percentiles):"
+      speedup_magnitude = @speedup_factors.minmax.map(&:to_i).map(&:abs).max.ceil
+      speedup_magnitude = 25 if speedup_magnitude.zero?
+      (0..100).step(5) do |n|
+        s.puts TAB + speedup_summary_line(n, speedup_magnitude)
+      end
+    end
+
+    def add_time_deltas_to(s)
+      percentile = ->(n) { "%+.3fs" % LabTech::Percentile.call(n, @time_deltas) }
+      s.puts
+      s << "Median time delta: #{percentile.(50)}"
+      s << "  "
+      s << "(90% of observations between #{percentile.(5)} and #{percentile.(95)})"
+      s.puts
+    end
+
+    def add_time_span_to(s)
+      t0, t1 = @earliest_result, @latest_result
+      s.puts "Earliest results: #{ t0.iso8601 }"
+      s.puts "Latest result:    #{ t1.iso8601 } (%s)" \
+        % date_helper.distance_of_time_in_words(t0, t1)
+    end
+
+    def date_helper
+      @_date_helper ||= Object.new.tap do |o|
+        o.extend ActionView::Helpers::DateHelper
+      end
+    end
+
+    def fetch_data
+      # Grab all aggregate operations counts/lists inside a transaction
+      # so all the counts are consistent
+      @experiment.transaction do
+        scope = experiment.results
+
+        @earliest_result = scope.minimum(:created_at)
+        @latest_result   = scope.maximum(:created_at)
+
+        @counts = {
+          results:    scope.count,
+          correct:    scope.correct.count,
+          mismatched: scope.mismatched.count,
+          timeout:    scope.timed_out.count,
+          errored:    scope.other_error.count,
+        }
+
+        speedups = experiment.results.correct.pluck(:time_delta, :speedup_factor).map { |time, factor|
+          LabTech::Speedup.new(time: time, factor: factor)
+        }
+        @time_deltas     = speedups.map(&:time).compact.sort
+        @speedup_factors = speedups.map(&:factor).compact.sort
+      end
+    end
+
+    def highlight_bar(bar)
+      left, right = bar.split(VAL)
+
+      left  = left         .gsub("  ", " #{DOT}")
+      right = right.reverse.gsub("  ", " #{DOT}").reverse
+
+      left + VAL + right
+    end
+
+    def humanize(n)
+      width = number_helper.number_with_delimiter( @counts[:results] ).length
+      "%#{width}s" % number_helper.number_with_delimiter( n )
+    end
+
+    def pad_left(s, width)
+      n = [ ( width - s.length ), 0 ].max
+      [ " " * n , s ].join
+    end
+
+    def normalized_bar(x, magnitude, bar_scale: 25, highlight: false)
+      neg, pos = " " * bar_scale, " " * bar_scale
+      normalized = ( bar_scale * ( x.abs / magnitude ) ).floor
+
+      # Select an index that's as close to `normalized` as possible without generating IndexErrors
+      # (TODO: actually understand the math involved so I don't have to chop the ends off like an infidel)
+      index = [ 0, normalized ].max
+      index = [ index, bar_scale - 1 ].min
+
+      case
+      when x == 0 ; mid = VAL
+      when x <  0 ; mid = DOT ; neg[ index ] = VAL ; neg = neg.reverse
+      when x  > 0 ; mid = DOT ; pos[ index ] = VAL
+      end
+
+      bar = "[%s%s%s]" % [ neg, mid, pos ]
+      bar = highlight_bar(bar) if highlight
+      bar
+    end
+
+    def number_helper
+      @_number_helper ||= Object.new.tap {|o| o.send :extend, ActionView::Helpers::NumberHelper }
+    end
+
+    def rate(n)
+      "%2.2f%%" % ( 100.0 * n / @counts[:results] )
+    end
+
+    def speedup_summary_line(n, speedup_magnitude)
+      highlight = n == 50
+      label = "%3d%%" % n
+
+      speedup_factor = LabTech::Percentile.call(n, @speedup_factors)
+      rel_speedup    = "%+.1fx" % speedup_factor
+      bar            = normalized_bar( speedup_factor, speedup_magnitude, highlight: highlight)
+
+      speedup_cue    = pad_left( rel_speedup, speedup_width )
+      speedup_cue += " faster" if speedup_factor > 0
+
+      "#{label}  #{bar}  #{speedup_cue}"
+    end
+
+    def speedup_width
+      @_speedup_width ||= [
+        1, # sign
+        4, # digits
+        1, # decimal point
+        1, # digit after decimal point
+      ].sum
+    end
+
+    def summarize_count(s, count_name, label = nil)
+      count = @counts[count_name]
+      return if count.zero?
+
+      total = @counts[:results]
+      label ||= count_name.to_s
+      s.puts "%s of %s (%s) %s" % [ humanize( count ), humanize( total ), rate( count ), label ]
+    end
+
+  end
+end

data/config/routes.rb

@@ -0,0 +1,2 @@
+LabTech::Engine.routes.draw do
+end

data/db/migrate/20190815192130_create_experiment_tables.rb

@@ -0,0 +1,50 @@
+class CreateExperimentTables < ActiveRecord::Migration[5.1]
+  def change
+
+    #  Quick E-R diagram:
+    #
+    #  +------------+     +--------+     +-------------+
+    #  | Experiment |----E| Result |----E| Observation |
+    #  +------------+     +--------+     +-------------+
+
+    create_table "lab_tech_experiments" do |t|
+      t.string    "name"
+      t.integer   "percent_enabled",   null: false, default: 0
+      t.integer   "equivalent_count",  null: false, default: 0
+      t.integer   "timed_out_count",   null: false, default: 0
+      t.integer   "other_error_count", null: false, default: 0
+
+      t.index [ "name" ], unique: true, name: "index_lab_tech_experiments_by_name"
+    end
+
+    create_table "lab_tech_results" do |t|
+      t.integer  "experiment_id",      null: false
+      t.text     "context"
+      t.boolean  "equivalent",         null: false, default: false
+      t.boolean  "raised_error",       null: false, default: false
+      t.float    "time_delta",         limit: 24
+      t.float    "speedup_factor",     limit: 24
+      t.datetime "created_at"
+      t.boolean  "timed_out",          null: false, default: false
+      t.float    "control_duration",   limit: 24
+      t.float    "candidate_duration", limit: 24
+
+      t.index [ "experiment_id", "equivalent" ],   name: "index_lab_tech_results_by_exp_id_and_equivalent"
+      t.index [ "experiment_id", "raised_error" ], name: "index_lab_tech_results_by_exp_id_and_raised"
+    end
+
+    create_table "lab_tech_observations" do |t|
+      t.integer  "result_id", null: false
+      t.string   "name",      limit: 100
+      t.float    "duration",  limit: 24
+      t.text     "value",     limit: 4294967295
+      t.text     "sql"
+      t.string   "exception_class"
+      t.text     "exception_message"
+      t.text     "exception_backtrace"
+      t.datetime "created_at"
+
+      t.index [ "result_id" ], name: "index_lab_tech_observations_by_result_id"
+    end
+  end
+end

data/lib/lab_tech.rb

@@ -0,0 +1,176 @@
+require "lab_tech/engine"
+require "scientist"
+
+module LabTech
+  extend self
+
+  ########################################################################
+  #
+  #   So, you've come here for science?  EXCELLENT.
+  #
+  #   TL;DR:
+  #
+  #   LabTech.science "experiment-name" do |exp|
+  #     exp.use { STABLE_CODE } # this is the "control"
+  #     exp.try { BETTER_CODE } # this is the "candidate"
+  #
+  #     # Optional, but often useful:
+  #     exp.context foo: "spam", bar: "eggs", yak: "bacon"
+  #     exp.compare {|control, candidate| control.map(&:id) == candidate.map(&:id) }
+  #     exp.clean { |records| records.map(&:id) }
+  #   end
+  #
+  #   See https://github.com/github/scientist for an *extremely* detailed
+  #   README that explains how to use this.  For those purposes, the thing
+  #   passed to the block as `exp` is a Scientist::Experiment.
+  #
+  #   NOTE: You'll probably want to check out the .enable and .disable methods
+  #   below if you want your candidate code to actually *run*...
+  #
+  ########################################################################
+  def science(experiment_name, opts = {}, &block)
+    experiment = Experiment.named( experiment_name )
+
+    yield experiment
+
+    test = opts[:run] if opts # TODO: figure out what this line was supposed to be for ¯\_(ツ)_/¯
+    experiment.run(test)
+  end
+
+  ########################################################################
+  #
+  #   This here is how you turn individual experiments on and off...
+  #
+  ########################################################################
+  def self.enable(*experiment_names, percent: 100)
+    experiments_named( experiment_names ) do |exp|
+      exp.enable percent_enabled: percent
+    end
+  end
+
+  def self.disable(*experiment_names)
+    experiments_named( experiment_names, &:disable )
+  end
+
+  ########################################################################
+  #
+  #   ...with an additional step if you want to record results in the Rails
+  #   test environment.
+  #
+  ########################################################################
+  def self.publish_results_in_test_mode?        ; !!@publish_results_in_test_mode           ; end
+  def self.publish_results_in_test_mode=(value) ;   @publish_results_in_test_mode = !!value ; end
+  def self.publish_results_in_test_mode
+    fail ArgumentError, "a block is required for this method" unless block_given?
+
+    old_value = self.publish_results_in_test_mode?
+    self.publish_results_in_test_mode = true
+    yield
+  ensure
+    self.publish_results_in_test_mode = old_value
+  end
+
+  ########################################################################
+  #
+  #   You'll probably want to see how your experiments are doing...
+  #
+  ########################################################################
+  def self.summarize_results(*experiment_names)
+    experiments_named( experiment_names, &:summarize_results )
+  end
+
+  ########################################################################
+  #
+  #   ...and be annoyed when they're not 100% correct...
+  #
+  ########################################################################
+  #
+  #   By default, this will simply print the values of all mismatches.
+  #   However, if you'd like to pass a block that returns arguments to
+  #   IO#puts, you can probably get more useful results.
+  #
+  #   Here's one example based on an experiment that records the IDs
+  #   returned from a search:
+  #
+  #     comparison = ->(cont, cand) {
+  #       cont_ids, cand_ids = cont.value, cand.value
+  #       case
+  #       when cont_ids      == cand_ids      ; "EVERYTHING IS FINE" # if this were true, it wouldn't be a mismatch
+  #       when cont_ids.sort == cand_ids.sort ; "ORDER DIFFERS"
+  #       else
+  #         [
+  #           "CONTROL   length: #{ cont_ids.length }",
+  #           "CANDIDATE length: #{ cand_ids.length }",
+  #           "    missing: #{ (cont_ids - cand_ids).inspect }",
+  #           "    extra:   #{ (cand_ids - cont_ids).inspect }",
+  #         ]
+  #       end
+  #     }
+  #     e = Experiment.named "isolate-lead-activities-in-lead-search"
+  #     e.compare_mismatches limit: 10, &comparison
+  #
+  #   And here's another one that assumes you've recorded a hash of the form:
+  #   { ids: [ 1, 2, ... ], sql: "SELECT FROM ..." }
+  #
+  #     comparison = ->(cont, cand) {
+  #       cont_ids, cand_ids = cont.value.fetch(:ids), cand.value.fetch(:ids)
+  #       cont_sql, cand_sql = cont.value.fetch(:sql), cand.value.fetch(:sql)
+  #       sql_strings = [ "", "CONTROL SQL", cont_sql, "", "CANDIDATE SQL", cand_sql ]
+  #       case
+  #       when cont_ids      == cand_ids      ; "EVERYTHING IS FINE" # if this were true, it wouldn't be a mismatch
+  #       when cont_ids.sort == cand_ids.sort ; [ "ORDER DIFFERS" ] + sql_strings
+  #       else
+  #         [
+  #           "CONTROL   length: #{ cont_ids.length }",
+  #           "CANDIDATE length: #{ cand_ids.length }",
+  #           "    missing: #{ (cont_ids - cand_ids).inspect }",
+  #           "    extra:   #{ (cand_ids - cont_ids).inspect }",
+  #         ] + sql_strings
+  #       end
+  #     }
+  #     e = Experiment.named "isolate-lead-activities-in-lead-search"
+  #     e.compare_mismatches limit: 10, &comparison
+  #
+  ########################################################################
+  def self.compare_mismatches(experiment_name, limit: nil, io: $stdout, &block)
+    exp = LabTech::Experiment.named( experiment_name )
+    exp.compare_mismatches limit: limit, io: io, &block
+  end
+
+  ########################################################################
+  #
+  #   ...and be curious about the errors...
+  #
+  ########################################################################
+  def self.summarize_errors(experiment_name, limit: nil, io: $stdout)
+    exp = LabTech::Experiment.named( experiment_name )
+    exp.summarize_errors( limit: limit, io: io )
+  end
+
+  ########################################################################
+  #
+  #   Sometimes specs might want to see that an experiment ran, the silly paranoid things
+  #
+  ########################################################################
+  def self.reset_run_count!
+    run_count.clear
+  end
+  def self.run_count
+    @_experiment_run_count ||= Hash.new(0)
+  end
+
+
+  ########################################################################
+  #
+  #   Sometimes we want to act on a batch of experiments
+  #   (this is mostly just plumbing; feel free to ignore it)
+  #
+  ########################################################################
+  def self.experiments_named(*experiment_names, &block)
+    names = experiment_names.flatten.compact
+    names.each do |exp_name|
+      LabTech::Experiment.named(exp_name, &block)
+    end
+  end
+
+end

data/lib/lab_tech/engine.rb

@@ -0,0 +1,6 @@
+module LabTech
+  class Engine < ::Rails::Engine
+    isolate_namespace LabTech
+    config.generators.api_only = true
+  end
+end

data/lib/lab_tech/version.rb

@@ -0,0 +1,3 @@
+module LabTech
+  VERSION = '0.1.0'
+end

data/lib/tasks/lab_tech_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :lab_tech do
+#   # Task goes here
+# end

data/spec/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative 'config/application'
+
+Rails.application.load_tasks

data/spec/dummy/app/assets/config/manifest.js

@@ -0,0 +1 @@
+//= link_directory ../stylesheets .css

data/spec/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,14 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require rails-ujs
+//= require_tree .