ips 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c8dde979505d34763745a2f6dcf6f9df0a8e6396fdd2351b769282393839417
-  data.tar.gz: 4225c76063e8a905bbf14efb5eb8ee000a030b4f674d6c2c9cef9571f007d4b6
+  metadata.gz: dd8eb49702573799e4d4754155e2d4c76713e1363f8dc2659ffbc99f96d21fe0
+  data.tar.gz: 7936ccc9a3896c754fd5fa2cd719e668df7317c6c92bc35e8d70fe301d1970e7
 SHA512:
-  metadata.gz: d0178e0e87a6913dc3134f4e748bdb795f16e4fb2a2cca10c3f1fc7e7dc9a4615e5bb3d45c12aff7adb6a0a12afd644392b77f07bd080ee91e382734b0ea1b8e
-  data.tar.gz: 199d08e24ae772bc5030a9f8daf63b4b453cfef6e8623110fd292ae70d5d1ca1f806bcaae7b86e364dae1850deea6a398b4b01426166c153cbc45917ec823644
+  metadata.gz: ac0010e763e43f03af94479fe5a9098a1d8b1caf0b58898f5248035e7a11e9cd946dbc195549f06c161104013c338a59f96440c5f630393be8da99b6728b2f0e
+  data.tar.gz: cf9749c62b609463ab3214ddf158b213ffcd4bc16f456cca71b7a3cbfb29c11dc8a93499271a56c7ced0078120b7ffcf12d33fb41d162fb259f8c469f5e710d4

data/README.md

@@ -39,10 +39,24 @@ Compare across Ruby versions, arguments, and commands:
 $ ips --ruby '3.4' --ruby '3.4 --yjit' --ruby '4.0' --ruby '4.0 --yjit' -e 'Object.new' -e 'Object.allocate'
 ```
 
+Compare the same benchmark across commits of the current repository:
+
+```
+$ ips --sha HEAD~3 --sha HEAD~10..HEAD~5 bench.rb
+```
+
+Each commit is checked out in place with `git checkout --detach`, so the Ruby,
+the bundle, and any per-directory setup stay identical across every commit. The
+working tree must be clean, and the branch you started on is restored when the
+run finishes (or is interrupted).
+
 Options:
 
 - `--ruby VERSION` — run against a specific Ruby (repeatable)
+- `--sha REV` — also run against a commit of the current repo (repeatable; a `A..B` range expands to its commits, oldest first)
 - `-e CODE` — inline benchmark expression (repeatable)
+- `--prelude CODE` — run setup code before the benchmark (repeatable)
+- `-I DIR` — add a directory to the load path (repeatable)
 - `-t SECONDS` — benchmark time (default: 5)
 - `-w SECONDS` — warmup time (default: 20% of benchmark time)
 - `--save PATH` — save results to JSON (appends to existing file)

data/exe/ips

@@ -2,8 +2,11 @@
 # frozen_string_literal: true
 
 versions = []
+revs = []
 inline_code = []
+prelude = []
 ruby_flags = []
+runs = 1
 save_path = nil
 bench_time = nil
 warmup_time = nil
@@ -15,14 +18,26 @@ while args.first
   when "--ruby"
     args.shift
     versions << args.shift
+  when "--sha"
+    args.shift
+    revs << args.shift
   when "-e"
     args.shift
     inline_code << args.shift
   when "-r"
     args.shift
     ruby_flags.push("-r", args.shift)
+  when "-I"
+    args.shift
+    ruby_flags.push("-I", args.shift)
+  when "--prelude"
+    args.shift
+    prelude << args.shift
   when "--disable-gems", /\A--yjit/, /\A--zjit/
     ruby_flags << args.shift
+  when "--runs"
+    args.shift
+    runs = args.shift.to_i
   when "--save"
     args.shift
     save_path = args.shift
@@ -51,31 +66,10 @@ end
 script = args.shift
 
 if script.nil? && inline_code.empty?
-  $stderr.puts "Usage: ips [--ruby VERSION ...] [-e CODE ...] [<script>]"
+  $stderr.puts "Usage: ips [--ruby VERSION ...] [--sha REV ...] [-r LIB ...] [--prelude CODE ...] [-e CODE ...] [<script>]"
   exit 1
 end
 
-if inline_code.any?
-  run_opts = []
-  run_opts << "summary: false" if versions.size > 1
-  run_opts << "time: #{bench_time}" if bench_time
-  run_opts << "warmup: #{warmup_time}" if warmup_time
-  run_opts << "debug: true" if debug
-  run_opts << "frozen_string_literal: false" unless frozen_string_literal
-  run_opts_str = run_opts.any? ? "(#{run_opts.join(", ")})" : ""
-  script_source = +""
-  script_source << "# frozen_string_literal: true\n" if frozen_string_literal
-  script_source << "require 'ips'\nresult = IPS.run#{run_opts_str} do |x|\n"
-  inline_code.each do |code|
-    script_source << "  x.report(#{code.inspect}, #{code.inspect})\n"
-  end
-  script_source << "end\n"
-  script_source << "IO.open(3, 'w') { |f| Marshal.dump(result, f) }\n"
-  ruby_args = ["-e", script_source]
-else
-  ruby_args = [script]
-end
-
 # Shell-escape for display purposes only, not for security.
 def shell_quote(arg)
   return arg if arg =~ /\A[A-Za-z0-9_\-.,:+\/@]+\z/
@@ -86,17 +80,21 @@ RUBY_DIRS = [File.expand_path("~/.rubies"), "/opt/rubies"]
 LIB_DIR = File.expand_path("../lib", __dir__)
 
 def find_ruby(version)
+  match = nil
   RUBY_DIRS.each do |dir|
     next unless Dir.exist?(dir)
     Dir.children(dir).sort.each do |name|
       bare = name.delete_prefix("ruby-")
-      if bare.start_with?(version)
+      if bare == version || name == version
         ruby = File.join(dir, name, "bin", "ruby")
         return ruby if File.executable?(ruby)
+      elsif bare.include?(version) || name.include?(version)
+        ruby = File.join(dir, name, "bin", "ruby")
+        match = ruby if File.executable?(ruby)
       end
     end
   end
-  nil
+  match
 end
 
 def resolve_ruby(str)
@@ -109,6 +107,20 @@ def resolve_ruby(str)
   [ruby, extra_flags]
 end
 
+# Runs git in the current directory, capturing stdout and stderr together.
+def git(*git_args)
+  out = IO.popen(["git", *git_args], err: [:child, :out], &:read)
+  [out, $?]
+end
+
+def git!(*git_args)
+  out, status = git(*git_args)
+  unless status.success?
+    abort "git #{git_args.join(" ")} failed:\n#{out}"
+  end
+  out
+end
+
 if versions.empty?
   rubies = [[nil, nil, []]]
 else
@@ -118,38 +130,176 @@ else
   }
 end
 
+# Each checkout is [label, sha]. Without --sha we stay where we are and label
+# nothing. With --sha the current checkout is always the first variant, so
+# there's a baseline to compare against.
+original_ref = nil
+original_sha = nil
+
+if revs.empty?
+  checkouts = [[nil, nil]]
+else
+  dirty = git!("status", "--porcelain", "-uno")
+  unless dirty.empty?
+    abort "Working tree has uncommitted changes to tracked files; commit or stash them before using --sha:\n#{dirty}"
+  end
+
+  original_sha = git!("rev-parse", "HEAD").strip
+  branch, branch_status = git("symbolic-ref", "--short", "HEAD")
+  original_ref = branch_status.success? ? branch.strip : nil
+
+  full_shas = [original_sha]
+  revs.each do |rev|
+    if rev.include?("..")
+      full_shas.concat(git!("rev-list", "--reverse", rev).split("\n"))
+    else
+      full_shas << git!("rev-parse", "--verify", "#{rev}^{commit}").strip
+    end
+  end
+  full_shas.uniq!
+
+  checkouts = full_shas.map { |sha| [git!("rev-parse", "--short", sha).strip, sha] }
+  # Prefer the branch name for the baseline when we have one.
+  checkouts[0][0] = original_ref if original_ref
+end
+
+# Variants are the product of checkouts and rubies.
+variants = checkouts.flat_map { |sha_label, sha|
+  rubies.map { |version_str, ruby, extra_flags|
+    label =
+      if version_str && sha_label
+        "ruby #{version_str} @ #{sha_label}"
+      elsif version_str
+        "ruby #{version_str}"
+      else
+        sha_label
+      end
+    [label, ruby, extra_flags, sha]
+  }
+}
+
+overrides = []
+overrides << "save: IO.open(3, 'w')"
+overrides << "summary: false" if variants.size > 1 || runs > 1
+overrides << "time: #{bench_time}" if bench_time
+overrides << "warmup: #{warmup_time}" if warmup_time
+overrides << "debug: true" if debug
+overrides << "frozen_string_literal: false" unless frozen_string_literal
+
+script_source = +""
+script_source << "# frozen_string_literal: true\n" if frozen_string_literal
+script_source << "require 'ips'\n"
+script_source << "IPS.overrides = { #{overrides.join(", ")} }\n"
+prelude.each { |code| script_source << code << "\n" }
+
+if inline_code.any?
+  script_source << "IPS.run do |x|\n"
+  inline_code.each do |code|
+    script_source << "  x.report(#{code.inspect}, #{code.inspect})\n"
+  end
+  script_source << "end\n"
+else
+  script_source << "load(#{script.dump})\n"
+end
+ruby_args = ["-e", script_source]
+
 $LOAD_PATH.unshift(LIB_DIR)
 require "ips"
 
-results = []
+# Check out commits in place, never changing directory and never creating a
+# worktree, so the ruby, the bundle, and any per-directory setup stay identical
+# across every variant.
+current_sha = original_sha
+moved = false
 
-rubies.each do |version_str, ruby, extra_flags|
-  if ruby
-    env = { "GEM_HOME" => nil, "GEM_PATH" => nil }
+checkout = lambda do |sha|
+  next if sha.nil? || sha == current_sha
+  $stdout.puts "git checkout --detach #{sha}" if verbose
+  out, status = git("checkout", "--detach", sha)
+  abort "git checkout --detach #{sha} failed:\n#{out}" unless status.success?
+  moved = true
+  current_sha = sha
+end
+
+restore = lambda do
+  next unless moved
+  moved = false
+  git_args = original_ref ? ["checkout", original_ref] : ["checkout", "--detach", original_sha]
+  out, status = git(*git_args)
+  if status.success?
+    current_sha = original_sha
   else
-    ruby = "ruby"
-    env = {}
+    $stderr.puts "WARNING: failed to restore #{original_ref || original_sha}:\n#{out}"
   end
+end
+
+trap("INT") do
+  restore.call
+  exit 130
+end if original_sha
 
-  cmd = [ruby, *extra_flags, *ruby_flags, "-v", "-I", LIB_DIR, *ruby_args]
-  $stderr.puts cmd.map { |a| shell_quote(a) }.join(" ") if verbose
-  rd, wr = IO.pipe
-  pid = spawn(env, *cmd, 3 => wr)
-  wr.close
-  data = rd.read
-  rd.close
-  Process.wait(pid)
-  raise "Child process failed" unless $?.success?
-  unless data.empty?
-    result = Marshal.load(data)
-    result.ruby_executable = ruby
-    result.run_label = "ruby #{version_str}" if version_str
-    results << result
+results_by_variant = variants.map { [] }
+quiet = runs > 1
+
+begin
+  runs.times do |run_index|
+    variants.each_with_index do |(label, ruby, extra_flags, sha), variant_index|
+      checkout.call(sha)
+
+      if ruby
+        env = { "GEM_HOME" => nil, "GEM_PATH" => nil }
+      else
+        ruby = "ruby"
+        env = {}
+      end
+
+      cmd = [ruby, *extra_flags, *ruby_flags]
+      cmd << "-v" unless quiet
+      cmd.push("-I", LIB_DIR, *ruby_args)
+      $stdout.puts cmd.map { |a| shell_quote(a) }.join(" ") if verbose
+
+      rd, wr = IO.pipe
+      spawn_opts = { 3 => wr }
+      spawn_opts[:out] = File::NULL if quiet
+
+      pid = spawn(env, *cmd, **spawn_opts)
+      wr.close
+
+      if quiet
+        $stdout.print "\rRun #{run_index + 1}/#{runs}"
+        $stdout.print " (#{label})" if label
+        $stdout.print "..."
+        $stdout.flush
+      end
+
+      until rd.eof?
+        result = Marshal.load(rd)
+        result.ruby_executable = ruby
+        result.run_label = label if label
+        results_by_variant[variant_index] << result
+      end
+      rd.close
+      _, status = Process.wait2(pid)
+      unless status.success?
+        $stderr.puts if quiet
+        cmd_str = cmd.map { |a| shell_quote(a) }.join(" ")
+        abort "Command failed (exit #{status.exitstatus}): #{cmd_str}"
+      end
+      puts unless quiet
+    end
   end
-  puts
+ensure
+  restore.call
 end
+$stderr.puts if quiet
 
-IPS::Result.compare(results)
+results = results_by_variant.flatten
+
+if runs > 1 && results.any?
+  IPS::Result.compare_aggregate(results_by_variant, out: $stdout)
+else
+  IPS::Result.compare(results)
+end
 
 if save_path && results.any?
   IPS::Result.save!(save_path, results)

data/lib/ips/job/entry.rb

@@ -32,8 +32,9 @@ module IPS
           end
         RUBY
         eval_source = "# frozen_string_literal: #{@frozen_string_literal}\n#{@source}"
-        m = (class << self; self; end)
-        m.class_eval(eval_source)
+        m = Module.new
+        m.module_eval(eval_source)
+        extend(m)
       end
 
       def define_call_times_block(act)

data/lib/ips/job.rb

@@ -3,11 +3,10 @@
 require "ips/job/entry"
 require "ips/result"
 require "ips/display"
+require "ips/warmup"
 
 module IPS
   class Job
-    MAX_ITERATIONS = 1 << 30
-
     attr_accessor :warmup, :time
 
     def initialize(time: 5, warmup: time * 0.2, summary: true, debug: false, frozen_string_literal: true, out: $stdout)
@@ -52,45 +51,29 @@ module IPS
 
     private
 
-    def cycles_per_100ms(time_ns, iters)
-      cycles = ((Timing::NANOSECONDS_PER_100MS.to_f / time_ns) * iters).to_i
-      cycles <= 0 ? 1 : cycles
-    end
-
     def warmup_item(item, display)
       Timing.clean_env
 
-      before = Timing.now
-      target = Timing.add_second(before, @warmup / 2.0)
-
-      cycles = 1
-      begin
+      budget_ns = (@warmup * Timing::NANOSECONDS_PER_SECOND).to_i
+      warmup = Warmup.new
+      last_progress = 0
+      cycles = warmup.run(budget_ns) do |iters|
         t0 = Timing.now
-        item.call_times(cycles)
+        item.call_times(iters)
         t1 = Timing.now
-        warmup_iter = cycles
-        warmup_ns = t1 - t0
-
-        estimate = Timing::NANOSECONDS_PER_SECOND * (warmup_iter.to_f / warmup_ns)
-        display.progress(estimate: estimate)
-
-        break if cycles >= MAX_ITERATIONS
-        cycles *= 2
-      end while Timing.now + warmup_ns * 2 < target
+        elapsed_ns = t1 - t0
+        if t1 - last_progress > Timing::NANOSECONDS_PER_100MS
+          estimate = Timing::NANOSECONDS_PER_SECOND * (iters.to_f / elapsed_ns)
+          display.progress(estimate: estimate)
+          last_progress = t1
+        end
+        @out.printf "  warmup: %d cycles in %dns (%s i/s)\n",
+          iters, elapsed_ns, Display.format_ips(Timing::NANOSECONDS_PER_SECOND * (iters.to_f / elapsed_ns)) if @debug
+        elapsed_ns
+      end
 
-      per_100ms = cycles_per_100ms(warmup_ns, warmup_iter)
-      cycles = per_100ms > MAX_ITERATIONS ? MAX_ITERATIONS : per_100ms
       @timing[item] = cycles
       @out.puts "  cycles: #{cycles}" if @debug
-
-      target = Timing.add_second(before, @warmup)
-      while Timing.now + Timing::NANOSECONDS_PER_100MS < target
-        t0 = Timing.now
-        item.call_times(cycles)
-        t1 = Timing.now
-        estimate = Timing::NANOSECONDS_PER_SECOND * (cycles.to_f / (t1 - t0))
-        display.progress(estimate: estimate)
-      end
     end
 
     def measure_item(item, display)

data/lib/ips/result.rb

@@ -77,6 +77,39 @@ module IPS
       data["runs"].map { |h| from_h(h) }
     end
 
+    def self.compare_aggregate(results_by_group, out: $stdout)
+      run_count = results_by_group.sum(&:size)
+      multiple_groups = results_by_group.size > 1
+
+      pairs = results_by_group.flat_map do |results|
+        next [] if results.empty?
+        run_label = results.first.run_label
+        group_entries(results).map do |entry_label, entries|
+          label = multiple_groups && run_label ? "#{entry_label} (#{run_label})" : entry_label
+          [label, Entry::Aggregate.new(entries)]
+        end
+      end
+
+      max_label = pairs.map { |l, _| l.size }.max
+      max_label = 20 if max_label < 20
+      out.puts "\nAggregate (#{run_count} runs)"
+      pairs.each do |label, agg|
+        error = agg.error_pct > 100 ? ">100" : "%4.1f" % agg.error_pct
+        out.printf "%#{max_label}s: %10s i/s (±%s%%)\n",
+          label, Display.format_ips(agg.ips), error
+      end
+
+      Display.compare(pairs, out: out)
+    end
+
+    def self.group_entries(results)
+      num_entries = results.first.entries.size
+      num_entries.times.map do |i|
+        entries = results.map { |r| r.entries[i] }
+        [entries.first.label, entries]
+      end
+    end
+
     def self.compare(results, out: $stdout)
       return if results.size < 2
 
@@ -202,6 +235,29 @@ module IPS
       def self.from_h(hash)
         new(hash["label"], hash["cycles"], hash["times"], hash["gc_times"])
       end
+
+      class Aggregate
+        attr_reader :entries
+
+        def initialize(entries)
+          @entries = entries
+        end
+
+        def ips
+          ips_values = @entries.map(&:ips)
+          ips_values.sum / ips_values.size
+        end
+
+        def stddev
+          mean = ips
+          variance = @entries.map(&:ips).sum { |v| (v - mean) ** 2 } / @entries.size
+          Math.sqrt(variance)
+        end
+
+        def error_pct
+          (stddev / ips) * 100.0
+        end
+      end
     end
   end
 end

data/lib/ips/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module IPS
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
 end

data/lib/ips/warmup.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module IPS
+  class Warmup
+    MAX_ITERATIONS = 1 << 30
+    TARGET_BATCH_NS = 100_000_000
+    MAX_WARMUP_CYCLES = 1000
+    TARGET_WARMUP_CALLS = 1000
+
+    def run(budget_ns)
+      elapsed = 0
+
+      # 1. Rough calibrate
+      last_ns = yield 1
+      elapsed += last_ns
+
+      # 2. JIT warmup
+      calls_remaining = TARGET_WARMUP_CALLS
+      cycles = 1
+      while elapsed < budget_ns && calls_remaining > 0
+        remaining_ns = budget_ns - elapsed
+        cycles = cycles_for(last_ns, cycles, remaining_ns / calls_remaining)
+        cycles = MAX_WARMUP_CYCLES if cycles > MAX_WARMUP_CYCLES
+        last_ns = yield cycles
+        elapsed += last_ns
+        calls_remaining -= 1
+      end
+
+      # 3. Calibrate to 100ms batches and run out the budget
+      cycles = cycles_for(last_ns, cycles, TARGET_BATCH_NS)
+      cycles = MAX_ITERATIONS if cycles > MAX_ITERATIONS
+      while elapsed < budget_ns
+        last_ns = yield cycles
+        elapsed += last_ns
+      end
+
+      cycles
+    end
+
+    private
+
+    def cycles_for(time_ns, iters, target_ns)
+      # Below the clock's resolution, so there's nothing to extrapolate from.
+      # Grow the batch and let the caller measure again.
+      return iters * 2 if time_ns <= 0
+
+      c = (target_ns * iters) / time_ns
+      c < 1 ? 1 : c
+    end
+  end
+end

data/lib/ips.rb

@@ -6,10 +6,18 @@ require "ips/display"
 require "ips/job"
 
 module IPS
-  def self.run(time: 5, warmup: time * 0.2, summary: true, debug: false, frozen_string_literal: true, out: $stdout)
-    job = Job.new(time: time, warmup: warmup, summary: summary, debug: debug, frozen_string_literal: frozen_string_literal, out: out)
+  @overrides = {}
+
+  def self.run(time: 5, warmup: time * 0.2, summary: true, debug: false, frozen_string_literal: true, save: nil, out: $stdout)
+    opts = { time: time, warmup: warmup, summary: summary, debug: debug, frozen_string_literal: frozen_string_literal, out: out }
+    opts.merge!(@overrides)
+    save = opts.delete(:save)
+
+    job = Job.new(**opts)
     yield job
-    job.run
+    result = job.run
+    Marshal.dump(result, save) if save
+    result
   end
 
   def self.report(label = nil, action = nil, &block)
@@ -21,6 +29,7 @@ module IPS
   end
 
   class << self
+    attr_accessor :overrides
     alias_method :ips, :run
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ips
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - John Hawthorn
@@ -39,6 +39,7 @@ files:
 - lib/ips/result.rb
 - lib/ips/timing.rb
 - lib/ips/version.rb
+- lib/ips/warmup.rb
 homepage: https://github.com/jhawthorn/ips
 licenses:
 - MIT
@@ -59,7 +60,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Iterations per second benchmarking with statistical analysis
 test_files: []