abprof 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae747acd5bc96fca3eafd6e0a1a8b0ba2d4ef62b
-  data.tar.gz: ae5cfee49428221a97f8c45935ef967e3072473a
+  metadata.gz: 0c46230d0f2b162c58c08527cac952ecef8c9396
+  data.tar.gz: 71aa9aeeac60e757c1074fa34dc527093ea67dae
 SHA512:
-  metadata.gz: 201b109cc93df7c18fe8d219ef6e3966abd8236bd950ca6a080a11c1c54d590506de62bb012e1b70cb540847ed50c7779ce7c3ac075328e3459a2b35501e049c
-  data.tar.gz: e79d72f6a0e8c5d439f250bd90755ca770b55c979229b983d9f3c64ce3c43500aaf724d4362ad3cc0107642c1775d8699b40cc4ab802a442c721bb24b5cdbc98
+  metadata.gz: fc6ace0578b40b801af81ebd1e3b315005c43d308a855e07bc5289676bf8ebd2dd47ba5162a3a51a26dcec562442ee701c388e9274d3d60e02528ad9954b69d2
+  data.tar.gz: 1c0c034f694218c58714f4fbd050216b423129cc763b44f5c7cfe48d237616ed82b4701617845862be5bbca7c34efbcfd599e18bdb324c33254fd30c70155e44

data/README.md

@@ -16,6 +16,17 @@ set of measured runtimes to determine how likely the two programs are
 to be different from each other, and after the P value is low enough,
 we give our current estimate of which is faster and by how much.
 
+Want a nice "getting started" introduction? Here are the original blog
+posts on using ABProf.
+
+* <a href="https://appfolio-engineering.squarespace.com/appfolio-engineering/2016/7/18/profiling-ruby-like-an-ab-tester">Profiling Ruby Like an A/B Tester</a>
+* <a href="https://appfolio-engineering.squarespace.com/appfolio-engineering/2016/8/8/abprof-an-accurate-statistical-profiling-harness">ABProf: An Accurate Statistical Profiling Harness</a>
+
+For more about the subtleties of profiling in general, may I recommend
+<a href="https://www.youtube.com/watch?v=kJDOpucaUR4">Matthew Gaudet's
+wonderful talk "Ruby 3x3: How Will Are We Going To Measure 3X?"  from
+RubyKaigi 2016?</a>
+
 ## Installation
 
 Add this line to your application's Gemfile:

data/exe/abcompare

@@ -0,0 +1,86 @@
+#!/usr/bin/env ruby
+
+require "trollop"
+require "abprof"
+require "abprof/benchmark_dsl"
+
+OPTS = Trollop::options do
+  banner <<BANNER
+Specify a first and second command line, and (often) a p-value or other
+parameters.
+
+Example:  #{$0} examples/sleep.rb examples/sleep_longer.rb
+
+The first and second commands are the first two arguments. You'll need to
+quote multi-word commands, as is normal in bash.
+
+Specifying lots of iterations and trials, high burn-in and a low P value
+is accurate, but slow.
+
+Specifying low iterations, trials and burn-in and a high P value gives
+quick, rough results early on.
+
+Specifying more iterations per trial is good for highly variable iteration
+timing.
+
+Specifying a lower max number of trials keeps the test from running *too*
+long when the two are identical.
+
+Specifying a high burn-in is necessary when cache behavior changes timing
+significantly.
+
+Vast numbers of trials can nearly always occasionally show differences
+*somewhere* along the line, just by random chance. To avoid this, pick how
+many samples first, run them all in one go, and then just check the p value
+once.
+
+A p value is often interpreted as the probability we got a wrong answer.
+That's an oversimplification, but not (usually) a terrible one.
+BANNER
+  opt :debug,       "Print more output to console"
+  opt :bare,        "Use bare command-line commands, no Ruby harness", :default => ($0["compare"])
+  opt :pvalue,      "P value (certainty) for Welch's T test", :default => 0.05
+  opt :burnin,      "'Burn in' repetitions before real trials",  :default => 10
+  opt :min_trials,  "Minimum number of sample sets from each process", :default => 1
+  opt :max_trials,  "Maximum number of sample sets from each process", :default => 20
+  opt :iters_per_trial, "Iterations per sample set", :default => 10
+  opt :print_samples, "Print all sample values for later analysis.", :default => false
+  opt :fail_on_divergence, "Return a non-zero code if pvalue is greater than specified."
+  opt :static_order, "Don't randomize the order of sampled processes per trial."
+end
+
+if ARGV.length != 2
+  puts "Must specify both commands as normal arguments!"
+  exit -1
+end
+
+command1, command2 = ARGV
+
+# Create DSL configuration for known properties,
+# but don't actually run the sampling yet.
+bm_inst = ABProf.compare(:no_at_exit => true) do
+  pvalue OPTS[:pvalue]
+  burnin OPTS[:burnin]
+  min_trials OPTS[:min_trials]
+  max_trials OPTS[:max_trials]
+  iters_per_trial OPTS[:iters_per_trial]
+  bare OPTS[:bare]
+  debug OPTS[:debug]
+  static_order OPTS[:static_order]
+  # No fail_on_divergence - we do this manually for the CLI utilities
+
+  report_command command1
+  report_command command2
+end
+
+state = bm_inst.run_sampling(:print_output => true)
+p_val = state[:p_tests][-1]
+
+if OPTS[:print_samples]
+  puts "Samples for P1: #{state[:samples][0].inspect}"
+  puts "Samples for P2: #{state[:samples][1].inspect}"
+end
+
+exit 2 if (p_val >= bm_inst.p_value) && OPTS[:fail_on_divergence]
+
+# Otherwise, return success

data/lib/abprof/version.rb

@@ -1,3 +1,3 @@
 module Abprof
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: abprof
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Noah Gibbs
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-08-15 00:00:00.000000000 Z
+date: 2016-11-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler

data/exe/abcompare

@@ -1 +0,0 @@
-abprof