abprof 0.2.0

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "abprof"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/examples/alt_ruby.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+STDERR.puts "ABProf example: optcarrot with alt Ruby dir"
+
+ABProf::ABWorker.iteration do
+  `cd ../alt_build && ./tool/runruby.rb ../optcarrot/bin/optcarrot --benchmark ../optcarrot/examples/Lan_Master.nes`
+end
+
+ABProf::ABWorker.start

data/examples/command_dsl.rb

@@ -0,0 +1,13 @@
+require "abprof/benchmark_dsl"
+
+ABProf.compare do
+  warmup 10
+  max_trials 5
+  min_trials 3
+  p_value 0.01
+  iters_per_trial 2
+
+  report_command "ruby examples/for_loop_10k.rb"
+  report_command "ruby examples/sleep.rb"
+
+end

data/examples/for_loop_10k.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+puts "ABProf example: 10,000 empty iterations"
+
+ABProf::ABWorker.iteration do
+  10_000.times {}
+end
+
+ABProf::ABWorker.start

data/examples/inline_ruby_1800.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+STDERR.puts "ABProf example: optcarrot with profiling Ruby dir"
+
+ABProf::ABWorker.iteration do
+  `cd ../inline_ruby_1800 && ./tool/runruby.rb ../optcarrot/bin/optcarrot --benchmark ../optcarrot/examples/Lan_Master.nes`
+end
+
+ABProf::ABWorker.start

data/examples/inline_ruby_2500.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+STDERR.puts "ABProf example: optcarrot with profiling Ruby dir"
+
+ABProf::ABWorker.iteration do
+  `cd ../inline_ruby_2500 && ./tool/runruby.rb ../optcarrot/bin/optcarrot --benchmark ../optcarrot/examples/Lan_Master.nes`
+end
+
+ABProf::ABWorker.start

data/examples/inlined_ruby.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+STDERR.puts "ABProf example: optcarrot with with-inlined-funcs Ruby dir"
+
+ABProf::ABWorker.iteration do
+  `cd ../inline_func_ruby && ./tool/runruby.rb ../optcarrot/bin/optcarrot --benchmark ../optcarrot/examples/Lan_Master.nes`
+end
+
+ABProf::ABWorker.start

data/examples/profiling_ruby.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+STDERR.puts "ABProf example: optcarrot with profiling Ruby dir"
+
+ABProf::ABWorker.iteration do
+  `cd ../profiling_ruby && ./tool/runruby.rb ../optcarrot/bin/optcarrot --benchmark ../optcarrot/examples/Lan_Master.nes`
+end
+
+ABProf::ABWorker.start

data/examples/simple_dsl.rb

@@ -0,0 +1,20 @@
+require "abprof/benchmark_dsl"
+
+ABProf.compare do
+  warmup 10
+  max_trials 5
+  min_trials 3
+  p_value 0.01
+  iters_per_trial 2
+  fail_on_divergence true  # For testing, usually
+  bare true
+
+  report do
+    10_000.times {}
+  end
+
+  report do
+    sleep 0.1
+  end
+
+end

data/examples/sleep.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+puts "ABProf example: sleep 0.1 seconds"
+
+ABProf::ABWorker.iteration do
+  sleep 0.001
+end
+
+ABProf::ABWorker.start

data/examples/sleep_longer.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+ABProf::ABWorker.iteration do
+  sleep 0.0015
+end
+
+ABProf::ABWorker.start

data/examples/vanilla_ruby.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "abprof"
+
+STDERR.puts "ABProf example: optcarrot with vanilla Ruby dir"
+
+ABProf::ABWorker.iteration do
+  `cd ../vanilla_build && ./tool/runruby.rb ../optcarrot/bin/optcarrot --benchmark ../optcarrot/examples/Lan_Master.nes`
+end
+
+ABProf::ABWorker.start

data/exe/abcompare

@@ -0,0 +1 @@
+abprof

data/exe/abprof

@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+
+require "trollop"
+require "abprof"
+require "abprof/benchmark_dsl"
+
+OPTS = Trollop::options do
+  banner <<BANNER
+Specify a first and second command line, and (often) a p-value or other
+parameters.
+
+Example:  #{$0} examples/sleep.rb examples/sleep_longer.rb
+
+The first and second commands are the first two arguments. You'll need to
+quote multi-word commands, as is normal in bash.
+
+Specifying lots of iterations and trials, high burn-in and a low P value
+is accurate, but slow.
+
+Specifying low iterations, trials and burn-in and a high P value gives
+quick, rough results early on.
+
+Specifying more iterations per trial is good for highly variable iteration
+timing.
+
+Specifying a lower max number of trials keeps the test from running *too*
+long when the two are identical.
+
+Specifying a high burn-in is necessary when cache behavior changes timing
+significantly.
+
+Vast numbers of trials can nearly always occasionally show differences
+*somewhere* along the line, just by random chance. To avoid this, pick how
+many samples first, run them all in one go, and then just check the p value
+once.
+
+A p value is often interpreted as the probability we got a wrong answer.
+That's an oversimplification, but not (usually) a terrible one.
+BANNER
+  opt :debug1,      "Print first-process output to console"
+  opt :debug2,      "Print second-process output to console"
+  opt :bare,        "Use bare command-line commands, no Ruby harness", :default => ($0["compare"])
+  opt :pvalue,      "P value (certainty) for Welch's T test", :default => 0.05
+  opt :burnin,      "'Burn in' repetitions before real trials",  :default => 10
+  opt :min_trials,  "Minimum number of sample sets from each process", :default => 1
+  opt :max_trials,  "Maximum number of sample sets from each process", :default => 20
+  opt :iters_per_trial, "Iterations per sample set", :default => 10
+  opt :print_samples, "Print all sample values for later analysis.", :default => false
+  opt :fail_on_divergence, "Return a non-zero code if pvalue is greater than specified."
+end
+
+if ARGV.length != 2
+  puts "Must specify both commands as normal arguments!"
+  exit -1
+end
+
+command1, command2 = ARGV
+
+# Create DSL configuration for known properties,
+# but don't actually run the sampling yet.
+bm_inst = ABProf.compare(:no_at_exit => true) do
+  pvalue OPTS[:pvalue]
+  burnin OPTS[:burnin]
+  min_trials OPTS[:min_trials]
+  max_trials OPTS[:max_trials]
+  iters_per_trial OPTS[:iters_per_trial]
+  bare OPTS[:bare]
+  # No fail_on_divergence - we do this manually for the CLI utilities
+
+  report_command command1
+  report_command command2
+end
+
+state = bm_inst.run_sampling
+p_val = state[:p_tests][-1]
+diverged = false
+if p_val < bm_inst.p_value
+  puts "Based on measured P value #{p_val}, we believe there is a speed difference."
+  puts "As of end of run, p value is #{p_val}. Now run more times to check, or with lower p."
+
+  summary11 = ABProf.summarize("mean", state[:samples][0])
+  summary12 = ABProf.summarize("median", state[:samples][0])
+  summary21 = ABProf.summarize("mean", state[:samples][1])
+  summary22 = ABProf.summarize("median", state[:samples][1])
+
+  fastest = "1"
+  command = bm_inst.reports[0]
+  mean_times = summary21 / summary11
+  median_times = summary22 / summary12
+  if summary11 > summary21
+    fastest = "2"
+    command = bm_inst.reports[1]
+    mean_times = summary11 / summary21
+    median_times = summary12 / summary22
+  end
+
+  puts "Lower (faster?) process is #{fastest}, command line: #{command.inspect}"
+  puts "Lower command is (very) roughly #{median_times} times lower (faster?) -- assuming linear sampling."
+
+  print "\n"
+  puts "Process 1 mean result: #{summary11}"
+  puts "Process 1 median result: #{summary12}"
+  puts "Process 2 mean result: #{summary21}"
+  puts "Process 2 median result: #{summary22}"
+else
+  puts "Based on measured P value #{p_val} and threshold #{bm_inst.pvalue}, we believe there is"
+  puts "no significant difference detectable with this set of trials."
+  puts "If you believe there is a small difference that wasn't detected, try raising the number"
+  puts "of iterations per trial, or the maximum number of trials."
+  diverged = true
+end
+
+if OPTS[:print_samples]
+  puts "Samples for P1: #{state[:samples][0].inspect}"
+  puts "Samples for P2: #{state[:samples][1].inspect}"
+end
+
+exit 2 if diverged && OPTS[:fail_on_divergence]
+
+# Otherwise, return success

data/lib/abprof.rb

@@ -0,0 +1,280 @@
+require "abprof/version"
+
+require "multi_json"
+
+# Protocol:
+#   Controller sends "ITERS [integer]\n"
+#   Controller sends "QUIT\n" when done
+#   Test process responds with "NOT OK\n" or crashes for bad results
+#   Test process responds with "VALUE 27.23432" to explicitly return a single value
+#   Test process responds with "VALUES [1.4, 2.714, 39.4, -71.4]" to explicitly return many values
+#     QUIT requires no response.
+
+module ABProf
+  def self.debug
+    @debug
+  end
+  def self.debug=(new_val)
+    @debug = new_val
+  end
+
+  # These are primarily for DSL use.
+  PROPERTIES = [ :debug, :pvalue, :iters_per_trial, :min_trials, :max_trials, :burnin, :bare, :fail_on_divergence ]
+
+  # This class is used by programs that are *being* profiled.
+  # It's necessarily a singleton since it needs to control STDIN.
+  # The bare mode can do without it, but it's needed for harness
+  # processes.
+  class ABWorker
+    def debug string
+      STDERR.puts(string) if ABProf.debug
+    end
+    def self.debug string
+      STDERR.puts(string) if ABProf.debug
+    end
+
+    def self.iteration(&block)
+      @iter_block = block
+      @return = :none
+    end
+
+    def self.iteration_with_return_value(&block)
+      @iter_block = block
+      @return = :per_iteration
+    end
+
+    def self.n_interations_with_return_value(&block)
+      @iter_block = block
+      @return = :per_n_iterations
+    end
+
+    def self.run_n(n)
+      debug "WORKER #{Process.pid}: running #{n} times"
+
+      case @return
+      when :none
+        n.times do
+          @iter_block.call
+        end
+        STDOUT.write "OK\n"
+      when :per_iteration
+        values = (0..(n-1)).map { |i| @iter_block.call.to_f }
+        STDOUT.write "VALUES #{values.inspect}"
+      when :per_n_iterations
+        value = @iter_block.call(n)
+        if value.respond_to?(:each)
+          # Return array of numbers
+          STDOUT.write "VALUES #{value.to_a.inspect}"
+        else
+          # Return single number
+          STDOUT.write "VALUE #{value.to_f}"
+        end
+      else
+        raise "Unknown @return value #{@return.inspect} inside abprof!"
+      end
+    end
+
+    def self.read_once
+      debug "WORKER #{Process.pid}: read loop"
+      @input ||= ""
+      @input += (STDIN.gets || "")
+      debug "WORKER #{Process.pid}: Input #{@input.inspect}"
+      if @input["\n"]
+        command, @input = @input.split("\n", 2)
+        debug "WORKER #{Process.pid}: command: #{command.inspect}"
+        if command == "QUIT"
+          exit 0
+        elsif command["ITERS"]
+          iters = command[5..-1].to_i
+          values = run_n iters
+          STDOUT.flush  # Why does this synchronous file descriptor not flush when given a string with a newline? Ugh!
+          debug "WORKER #{Process.pid}: finished command ITERS: OK"
+        else
+          STDERR.puts "Unrecognized ABProf command: #{command.inspect}!"
+          exit -1
+        end
+      end
+    end
+
+    def self.start
+      loop do
+        read_once
+      end
+    end
+  end
+
+  SUMMARY_TYPES = {
+    "mean" => proc { |samples|
+      samples.inject(0.0, &:+) / samples.size
+    },
+    "median" => proc { |samples|
+      sz = samples.size
+      sorted = samples.sort
+      if sz % 2 == 1
+        # For odd-length, take middle element
+        sorted[ samples.size / 2 ]
+      else
+        # For even length, mean of two middle elements
+        (sorted[ sz / 2 ] + sorted[ sz / 2 + 1 ]) / 2.0
+      end
+    },
+  }
+  SUMMARY_METHODS = SUMMARY_TYPES.keys
+  def self.summarize(method, samples)
+    raise "Unknown summary method #{method.inspect}!" unless SUMMARY_METHODS.include?(method.to_s)
+    method_proc = SUMMARY_TYPES[method.to_s]
+    method_proc.call(samples)
+  end
+
+  class ABBareProcess
+    attr_reader :last_run
+    attr_reader :last_iters
+
+    def debug string
+      STDERR.puts(string) if @debug && ABProf.debug
+    end
+
+    def initialize command_line, opts = {}
+      @command = command_line
+      @debug = opts[:debug]
+    end
+
+    def quit
+      # No-op
+    end
+
+    def kill
+      # No-op
+    end
+
+    def run_iters(n)
+      t_start = t_end = nil
+      debug "Controller of #{@pid}: #{n} ITERS"
+
+      state = :succeeded
+      n.times do
+        if @command.respond_to?(:call)
+          t_start = Time.now
+          @command.call
+          t_end = Time.now
+        elsif @command.respond_to?(:to_s)
+          t_start = Time.now
+          system(@command.to_s)
+          t_end = Time.now
+          unless $?.success?
+            STDERR.puts "Failing process #{@pid} after failed iteration(s), error code #{state.inspect}"
+            # How to handle error with no self.kill?
+            raise "Failure from command #{@command.inspect}, dying!"
+          end
+        else
+          raise "Don't know how to execute bare object: #{@command.inspect}!"
+        end
+      end
+      @last_run = [(t_end - t_start).to_f]
+      @last_iters = n
+
+      @last_run
+    end
+  end
+
+  class ABHarnessProcess
+    attr_reader :last_run
+    attr_reader :last_iters
+
+    def debug string
+      STDERR.puts(string) if @debug && ABProf.debug
+    end
+
+    def initialize command_line, opts = {}
+      debug "Controller of nobody yet: SPAWN"
+      @in_reader, @in_writer = IO.pipe
+      @out_reader, @out_writer = IO.pipe
+      @in_writer.sync = true
+      @out_writer.sync = true
+
+      @pid = fork do
+        STDOUT.reopen(@out_writer)
+        STDIN.reopen(@in_reader)
+        @out_reader.close
+        @in_writer.close
+        if command_line.respond_to?(:call)
+          puts "Caution! An ABProf Harness process (non-bare) is being used with a block. This is almost never what you want!"
+          command_line.call
+        elsif command_line.respond_to?(:to_s)
+          exec command_line.to_s
+        else
+          raise "Don't know how to execute benchmark code: #{command_line.inspect}!"
+        end
+        exit! 0
+      end
+      @out_writer.close
+      @in_reader.close
+
+      @debug = opts[:debug]
+      debug "Controller spawned #{@pid} (debug: #{@debug.inspect})"
+    end
+
+    def quit
+      debug "Controller of #{@pid}: QUIT"
+      @in_writer.write "QUIT\n"
+    end
+
+    def kill
+      debug "Controller of #{@pid}: DIE"
+      ::Process.detach @pid
+      ::Process.kill "TERM", @pid
+    end
+
+    def run_iters(n)
+      debug "Controller of #{@pid}: #{n} ITERS"
+      @in_writer.write "ITERS #{n.to_i}\n"
+
+      ignored_out = 0
+      state = :failed
+      t_start = Time.now
+      loop do
+        # Read and block
+        output = @out_reader.gets
+        ignored_out += output.length
+        puts "Controller of #{@pid} out: #{output.inspect}" if @debug
+        debug "Controller of #{@pid} out: #{output.inspect}"
+        if output =~ /^VALUES/ # These anchors match newlines, too
+          state = :succeeded
+          vals = MultiJson.load output[7..-1]
+          raise "Must return an array value from iterations!" unless vals.is_a?(Array)
+          raise "Must return an array of numbers from iterations!" unless vals[0].is_a?(Numeric)
+          @last_run = vals
+        elsif output =~ /^VALUE/ # These anchors match newlines, too
+          state = :succeeded
+          val = output[6..-1].to_f
+          raise "Must return a number from iterations!" unless val.is_a?(Numeric)
+          @last_run = [ val ]
+        elsif output =~ /^OK$/   # These anchors match newlines, too
+          state = :succeeded_get_time
+          break
+        end
+        if output =~ /^NOT OK$/ # These anchors match newlines, too
+          # Failed, break
+          state = :explicit_not_ok
+          break
+        end
+        if ignored_out > 10_000
+          # 10k of output and no OK? Bail with failed state.
+          state = :too_much_output_without_status
+          break
+        end
+      end
+      t_end = Time.now
+      unless [:succeeded, :succeeded_get_time].include?(state)
+        self.kill
+        STDERR.puts "Killing process #{@pid} after failed iterations, error code #{state.inspect}"
+      end
+
+      @last_run = [ (t_end - t_start).to_f ] if state == :succeeded_get_time
+      @last_iters = n
+
+      @last_run
+    end
+
+  end
+end