ips 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c8dde979505d34763745a2f6dcf6f9df0a8e6396fdd2351b769282393839417
-  data.tar.gz: 4225c76063e8a905bbf14efb5eb8ee000a030b4f674d6c2c9cef9571f007d4b6
+  metadata.gz: 5bb634cdd8199a1545541b115316fdb80324560b872690ad5138edda74663fb1
+  data.tar.gz: f00fa8882fb682d591c74970988ac6c371adf4174a212abc137a62e046f61fa6
 SHA512:
-  metadata.gz: d0178e0e87a6913dc3134f4e748bdb795f16e4fb2a2cca10c3f1fc7e7dc9a4615e5bb3d45c12aff7adb6a0a12afd644392b77f07bd080ee91e382734b0ea1b8e
-  data.tar.gz: 199d08e24ae772bc5030a9f8daf63b4b453cfef6e8623110fd292ae70d5d1ca1f806bcaae7b86e364dae1850deea6a398b4b01426166c153cbc45917ec823644
+  metadata.gz: c3bd6bf124e9a17ac2e1f02d8d68166f512be659f1fc488ac063213589762fd0f9d81aade8825dc56fa4da4162812cf68640403829a73b907da0ac735bf3f190
+  data.tar.gz: 1858e8651c5472a4c032dc7e2f4d5a4a758d20fad9b179ba55c5e2af05518f93609c2a6c961594798f2d8b639413b82980c038352071182d7d42dd4eee032f13

data/exe/ips

@@ -4,6 +4,7 @@
 versions = []
 inline_code = []
 ruby_flags = []
+runs = 1
 save_path = nil
 bench_time = nil
 warmup_time = nil
@@ -23,6 +24,9 @@ while args.first
     ruby_flags.push("-r", args.shift)
   when "--disable-gems", /\A--yjit/, /\A--zjit/
     ruby_flags << args.shift
+  when "--runs"
+    args.shift
+    runs = args.shift.to_i
   when "--save"
     args.shift
     save_path = args.shift
@@ -55,26 +59,29 @@ if script.nil? && inline_code.empty?
   exit 1
 end
 
+overrides = []
+overrides << "save: IO.open(3, 'w')"
+overrides << "summary: false" if versions.size > 1 || runs > 1
+overrides << "time: #{bench_time}" if bench_time
+overrides << "warmup: #{warmup_time}" if warmup_time
+overrides << "debug: true" if debug
+overrides << "frozen_string_literal: false" unless frozen_string_literal
+
+script_source = +""
+script_source << "# frozen_string_literal: true\n" if frozen_string_literal
+script_source << "require 'ips'\n"
+script_source << "IPS.overrides = { #{overrides.join(", ")} }\n"
+
 if inline_code.any?
-  run_opts = []
-  run_opts << "summary: false" if versions.size > 1
-  run_opts << "time: #{bench_time}" if bench_time
-  run_opts << "warmup: #{warmup_time}" if warmup_time
-  run_opts << "debug: true" if debug
-  run_opts << "frozen_string_literal: false" unless frozen_string_literal
-  run_opts_str = run_opts.any? ? "(#{run_opts.join(", ")})" : ""
-  script_source = +""
-  script_source << "# frozen_string_literal: true\n" if frozen_string_literal
-  script_source << "require 'ips'\nresult = IPS.run#{run_opts_str} do |x|\n"
+  script_source << "IPS.run do |x|\n"
   inline_code.each do |code|
     script_source << "  x.report(#{code.inspect}, #{code.inspect})\n"
   end
   script_source << "end\n"
-  script_source << "IO.open(3, 'w') { |f| Marshal.dump(result, f) }\n"
-  ruby_args = ["-e", script_source]
 else
-  ruby_args = [script]
+  script_source << "load(#{script.dump})\n"
 end
+ruby_args = ["-e", script_source]
 
 # Shell-escape for display purposes only, not for security.
 def shell_quote(arg)
@@ -86,17 +93,21 @@ RUBY_DIRS = [File.expand_path("~/.rubies"), "/opt/rubies"]
 LIB_DIR = File.expand_path("../lib", __dir__)
 
 def find_ruby(version)
+  match = nil
   RUBY_DIRS.each do |dir|
     next unless Dir.exist?(dir)
     Dir.children(dir).sort.each do |name|
       bare = name.delete_prefix("ruby-")
-      if bare.start_with?(version)
+      if bare == version || name == version
         ruby = File.join(dir, name, "bin", "ruby")
         return ruby if File.executable?(ruby)
+      elsif bare.include?(version) || name.include?(version)
+        ruby = File.join(dir, name, "bin", "ruby")
+        match = ruby if File.executable?(ruby)
       end
     end
   end
-  nil
+  match
 end
 
 def resolve_ruby(str)
@@ -121,35 +132,62 @@ end
 $LOAD_PATH.unshift(LIB_DIR)
 require "ips"
 
-results = []
+results_by_ruby = rubies.map { [] }
+quiet = runs > 1
 
-rubies.each do |version_str, ruby, extra_flags|
-  if ruby
-    env = { "GEM_HOME" => nil, "GEM_PATH" => nil }
-  else
-    ruby = "ruby"
-    env = {}
-  end
+runs.times do |run_index|
+  rubies.each_with_index do |(version_str, ruby, extra_flags), ruby_index|
+    if ruby
+      env = { "GEM_HOME" => nil, "GEM_PATH" => nil }
+    else
+      ruby = "ruby"
+      env = {}
+    end
+
+    cmd = [ruby, *extra_flags, *ruby_flags]
+    cmd << "-v" unless quiet
+    cmd.push("-I", LIB_DIR, *ruby_args)
+    $stdout.puts cmd.map { |a| shell_quote(a) }.join(" ") if verbose
+
+    rd, wr = IO.pipe
+    spawn_opts = { 3 => wr }
+    spawn_opts[:out] = File::NULL if quiet
 
-  cmd = [ruby, *extra_flags, *ruby_flags, "-v", "-I", LIB_DIR, *ruby_args]
-  $stderr.puts cmd.map { |a| shell_quote(a) }.join(" ") if verbose
-  rd, wr = IO.pipe
-  pid = spawn(env, *cmd, 3 => wr)
-  wr.close
-  data = rd.read
-  rd.close
-  Process.wait(pid)
-  raise "Child process failed" unless $?.success?
-  unless data.empty?
-    result = Marshal.load(data)
-    result.ruby_executable = ruby
-    result.run_label = "ruby #{version_str}" if version_str
-    results << result
+    pid = spawn(env, *cmd, **spawn_opts)
+    wr.close
+
+    if quiet
+      $stdout.print "\rRun #{run_index + 1}/#{runs}"
+      $stdout.print " (#{version_str})" if version_str
+      $stdout.print "..."
+      $stdout.flush
+    end
+
+    until rd.eof?
+      result = Marshal.load(rd)
+      result.ruby_executable = ruby
+      result.run_label = "ruby #{version_str}" if version_str
+      results_by_ruby[ruby_index] << result
+    end
+    rd.close
+    _, status = Process.wait2(pid)
+    unless status.success?
+      $stderr.puts if quiet
+      cmd_str = cmd.map { |a| shell_quote(a) }.join(" ")
+      abort "Command failed (exit #{status.exitstatus}): #{cmd_str}"
+    end
+    puts unless quiet
   end
-  puts
 end
+$stderr.puts if quiet
+
+results = results_by_ruby.flatten
 
-IPS::Result.compare(results)
+if runs > 1 && results.any?
+  IPS::Result.compare_aggregate(results_by_ruby, out: $stdout)
+else
+  IPS::Result.compare(results)
+end
 
 if save_path && results.any?
   IPS::Result.save!(save_path, results)

data/lib/ips/job/entry.rb

@@ -32,8 +32,9 @@ module IPS
           end
         RUBY
         eval_source = "# frozen_string_literal: #{@frozen_string_literal}\n#{@source}"
-        m = (class << self; self; end)
-        m.class_eval(eval_source)
+        m = Module.new
+        m.module_eval(eval_source)
+        extend(m)
       end
 
       def define_call_times_block(act)

data/lib/ips/job.rb

@@ -3,11 +3,10 @@
 require "ips/job/entry"
 require "ips/result"
 require "ips/display"
+require "ips/warmup"
 
 module IPS
   class Job
-    MAX_ITERATIONS = 1 << 30
-
     attr_accessor :warmup, :time
 
     def initialize(time: 5, warmup: time * 0.2, summary: true, debug: false, frozen_string_literal: true, out: $stdout)
@@ -52,45 +51,29 @@ module IPS
 
     private
 
-    def cycles_per_100ms(time_ns, iters)
-      cycles = ((Timing::NANOSECONDS_PER_100MS.to_f / time_ns) * iters).to_i
-      cycles <= 0 ? 1 : cycles
-    end
-
     def warmup_item(item, display)
       Timing.clean_env
 
-      before = Timing.now
-      target = Timing.add_second(before, @warmup / 2.0)
-
-      cycles = 1
-      begin
+      budget_ns = (@warmup * Timing::NANOSECONDS_PER_SECOND).to_i
+      warmup = Warmup.new
+      last_progress = 0
+      cycles = warmup.run(budget_ns) do |iters|
         t0 = Timing.now
-        item.call_times(cycles)
+        item.call_times(iters)
         t1 = Timing.now
-        warmup_iter = cycles
-        warmup_ns = t1 - t0
-
-        estimate = Timing::NANOSECONDS_PER_SECOND * (warmup_iter.to_f / warmup_ns)
-        display.progress(estimate: estimate)
-
-        break if cycles >= MAX_ITERATIONS
-        cycles *= 2
-      end while Timing.now + warmup_ns * 2 < target
+        elapsed_ns = t1 - t0
+        if t1 - last_progress > Timing::NANOSECONDS_PER_100MS
+          estimate = Timing::NANOSECONDS_PER_SECOND * (iters.to_f / elapsed_ns)
+          display.progress(estimate: estimate)
+          last_progress = t1
+        end
+        @out.printf "  warmup: %d cycles in %dns (%s i/s)\n",
+          iters, elapsed_ns, Display.format_ips(Timing::NANOSECONDS_PER_SECOND * (iters.to_f / elapsed_ns)) if @debug
+        elapsed_ns
+      end
 
-      per_100ms = cycles_per_100ms(warmup_ns, warmup_iter)
-      cycles = per_100ms > MAX_ITERATIONS ? MAX_ITERATIONS : per_100ms
       @timing[item] = cycles
       @out.puts "  cycles: #{cycles}" if @debug
-
-      target = Timing.add_second(before, @warmup)
-      while Timing.now + Timing::NANOSECONDS_PER_100MS < target
-        t0 = Timing.now
-        item.call_times(cycles)
-        t1 = Timing.now
-        estimate = Timing::NANOSECONDS_PER_SECOND * (cycles.to_f / (t1 - t0))
-        display.progress(estimate: estimate)
-      end
     end
 
     def measure_item(item, display)

data/lib/ips/result.rb

@@ -77,6 +77,39 @@ module IPS
       data["runs"].map { |h| from_h(h) }
     end
 
+    def self.compare_aggregate(results_by_group, out: $stdout)
+      run_count = results_by_group.sum(&:size)
+      multiple_groups = results_by_group.size > 1
+
+      pairs = results_by_group.flat_map do |results|
+        next [] if results.empty?
+        run_label = results.first.run_label
+        group_entries(results).map do |entry_label, entries|
+          label = multiple_groups && run_label ? "#{entry_label} (#{run_label})" : entry_label
+          [label, Entry::Aggregate.new(entries)]
+        end
+      end
+
+      max_label = pairs.map { |l, _| l.size }.max
+      max_label = 20 if max_label < 20
+      out.puts "\nAggregate (#{run_count} runs)"
+      pairs.each do |label, agg|
+        error = agg.error_pct > 100 ? ">100" : "%4.1f" % agg.error_pct
+        out.printf "%#{max_label}s: %10s i/s (±%s%%)\n",
+          label, Display.format_ips(agg.ips), error
+      end
+
+      Display.compare(pairs, out: out)
+    end
+
+    def self.group_entries(results)
+      num_entries = results.first.entries.size
+      num_entries.times.map do |i|
+        entries = results.map { |r| r.entries[i] }
+        [entries.first.label, entries]
+      end
+    end
+
     def self.compare(results, out: $stdout)
       return if results.size < 2
 
@@ -202,6 +235,29 @@ module IPS
       def self.from_h(hash)
         new(hash["label"], hash["cycles"], hash["times"], hash["gc_times"])
       end
+
+      class Aggregate
+        attr_reader :entries
+
+        def initialize(entries)
+          @entries = entries
+        end
+
+        def ips
+          ips_values = @entries.map(&:ips)
+          ips_values.sum / ips_values.size
+        end
+
+        def stddev
+          mean = ips
+          variance = @entries.map(&:ips).sum { |v| (v - mean) ** 2 } / @entries.size
+          Math.sqrt(variance)
+        end
+
+        def error_pct
+          (stddev / ips) * 100.0
+        end
+      end
     end
   end
 end

data/lib/ips/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module IPS
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/ips/warmup.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module IPS
+  class Warmup
+    MAX_ITERATIONS = 1 << 30
+    TARGET_BATCH_NS = 100_000_000
+    MAX_WARMUP_CYCLES = 1000
+    TARGET_WARMUP_CALLS = 1000
+
+    def run(budget_ns)
+      elapsed = 0
+
+      # 1. Rough calibrate
+      last_ns = yield 1
+      elapsed += last_ns
+
+      # 2. JIT warmup
+      calls_remaining = TARGET_WARMUP_CALLS
+      cycles = 1
+      while elapsed < budget_ns && calls_remaining > 0
+        remaining_ns = budget_ns - elapsed
+        cycles = cycles_for(last_ns, cycles, remaining_ns / calls_remaining)
+        cycles = MAX_WARMUP_CYCLES if cycles > MAX_WARMUP_CYCLES
+        last_ns = yield cycles
+        elapsed += last_ns
+        calls_remaining -= 1
+      end
+
+      # 3. Calibrate to 100ms batches and run out the budget
+      cycles = cycles_for(last_ns, cycles, TARGET_BATCH_NS)
+      cycles = MAX_ITERATIONS if cycles > MAX_ITERATIONS
+      while elapsed < budget_ns
+        last_ns = yield cycles
+        elapsed += last_ns
+      end
+
+      cycles
+    end
+
+    private
+
+    def cycles_for(time_ns, iters, target_ns)
+      c = (target_ns * iters) / time_ns
+      c < 1 ? 1 : c
+    end
+  end
+end

data/lib/ips.rb

@@ -6,10 +6,18 @@ require "ips/display"
 require "ips/job"
 
 module IPS
-  def self.run(time: 5, warmup: time * 0.2, summary: true, debug: false, frozen_string_literal: true, out: $stdout)
-    job = Job.new(time: time, warmup: warmup, summary: summary, debug: debug, frozen_string_literal: frozen_string_literal, out: out)
+  @overrides = {}
+
+  def self.run(time: 5, warmup: time * 0.2, summary: true, debug: false, frozen_string_literal: true, save: nil, out: $stdout)
+    opts = { time: time, warmup: warmup, summary: summary, debug: debug, frozen_string_literal: frozen_string_literal, out: out }
+    opts.merge!(@overrides)
+    save = opts.delete(:save)
+
+    job = Job.new(**opts)
     yield job
-    job.run
+    result = job.run
+    Marshal.dump(result, save) if save
+    result
   end
 
   def self.report(label = nil, action = nil, &block)
@@ -21,6 +29,7 @@ module IPS
   end
 
   class << self
+    attr_accessor :overrides
     alias_method :ips, :run
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ips
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - John Hawthorn
@@ -39,6 +39,7 @@ files:
 - lib/ips/result.rb
 - lib/ips/timing.rb
 - lib/ips/version.rb
+- lib/ips/warmup.rb
 homepage: https://github.com/jhawthorn/ips
 licenses:
 - MIT
@@ -59,7 +60,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Iterations per second benchmarking with statistical analysis
 test_files: []