ips 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5bb634cdd8199a1545541b115316fdb80324560b872690ad5138edda74663fb1
-  data.tar.gz: f00fa8882fb682d591c74970988ac6c371adf4174a212abc137a62e046f61fa6
+  metadata.gz: dd8eb49702573799e4d4754155e2d4c76713e1363f8dc2659ffbc99f96d21fe0
+  data.tar.gz: 7936ccc9a3896c754fd5fa2cd719e668df7317c6c92bc35e8d70fe301d1970e7
 SHA512:
-  metadata.gz: c3bd6bf124e9a17ac2e1f02d8d68166f512be659f1fc488ac063213589762fd0f9d81aade8825dc56fa4da4162812cf68640403829a73b907da0ac735bf3f190
-  data.tar.gz: 1858e8651c5472a4c032dc7e2f4d5a4a758d20fad9b179ba55c5e2af05518f93609c2a6c961594798f2d8b639413b82980c038352071182d7d42dd4eee032f13
+  metadata.gz: ac0010e763e43f03af94479fe5a9098a1d8b1caf0b58898f5248035e7a11e9cd946dbc195549f06c161104013c338a59f96440c5f630393be8da99b6728b2f0e
+  data.tar.gz: cf9749c62b609463ab3214ddf158b213ffcd4bc16f456cca71b7a3cbfb29c11dc8a93499271a56c7ced0078120b7ffcf12d33fb41d162fb259f8c469f5e710d4

data/README.md

@@ -39,10 +39,24 @@ Compare across Ruby versions, arguments, and commands:
 $ ips --ruby '3.4' --ruby '3.4 --yjit' --ruby '4.0' --ruby '4.0 --yjit' -e 'Object.new' -e 'Object.allocate'
 ```
 
+Compare the same benchmark across commits of the current repository:
+
+```
+$ ips --sha HEAD~3 --sha HEAD~10..HEAD~5 bench.rb
+```
+
+Each commit is checked out in place with `git checkout --detach`, so the Ruby,
+the bundle, and any per-directory setup stay identical across every commit. The
+working tree must be clean, and the branch you started on is restored when the
+run finishes (or is interrupted).
+
 Options:
 
 - `--ruby VERSION` — run against a specific Ruby (repeatable)
+- `--sha REV` — also run against a commit of the current repo (repeatable; a `A..B` range expands to its commits, oldest first)
 - `-e CODE` — inline benchmark expression (repeatable)
+- `--prelude CODE` — run setup code before the benchmark (repeatable)
+- `-I DIR` — add a directory to the load path (repeatable)
 - `-t SECONDS` — benchmark time (default: 5)
 - `-w SECONDS` — warmup time (default: 20% of benchmark time)
 - `--save PATH` — save results to JSON (appends to existing file)

data/exe/ips

@@ -2,7 +2,9 @@
 # frozen_string_literal: true
 
 versions = []
+revs = []
 inline_code = []
+prelude = []
 ruby_flags = []
 runs = 1
 save_path = nil
@@ -16,12 +18,21 @@ while args.first
   when "--ruby"
     args.shift
     versions << args.shift
+  when "--sha"
+    args.shift
+    revs << args.shift
   when "-e"
     args.shift
     inline_code << args.shift
   when "-r"
     args.shift
     ruby_flags.push("-r", args.shift)
+  when "-I"
+    args.shift
+    ruby_flags.push("-I", args.shift)
+  when "--prelude"
+    args.shift
+    prelude << args.shift
   when "--disable-gems", /\A--yjit/, /\A--zjit/
     ruby_flags << args.shift
   when "--runs"
@@ -55,34 +66,10 @@ end
 script = args.shift
 
 if script.nil? && inline_code.empty?
-  $stderr.puts "Usage: ips [--ruby VERSION ...] [-e CODE ...] [<script>]"
+  $stderr.puts "Usage: ips [--ruby VERSION ...] [--sha REV ...] [-r LIB ...] [--prelude CODE ...] [-e CODE ...] [<script>]"
   exit 1
 end
 
-overrides = []
-overrides << "save: IO.open(3, 'w')"
-overrides << "summary: false" if versions.size > 1 || runs > 1
-overrides << "time: #{bench_time}" if bench_time
-overrides << "warmup: #{warmup_time}" if warmup_time
-overrides << "debug: true" if debug
-overrides << "frozen_string_literal: false" unless frozen_string_literal
-
-script_source = +""
-script_source << "# frozen_string_literal: true\n" if frozen_string_literal
-script_source << "require 'ips'\n"
-script_source << "IPS.overrides = { #{overrides.join(", ")} }\n"
-
-if inline_code.any?
-  script_source << "IPS.run do |x|\n"
-  inline_code.each do |code|
-    script_source << "  x.report(#{code.inspect}, #{code.inspect})\n"
-  end
-  script_source << "end\n"
-else
-  script_source << "load(#{script.dump})\n"
-end
-ruby_args = ["-e", script_source]
-
 # Shell-escape for display purposes only, not for security.
 def shell_quote(arg)
   return arg if arg =~ /\A[A-Za-z0-9_\-.,:+\/@]+\z/
@@ -120,6 +107,20 @@ def resolve_ruby(str)
   [ruby, extra_flags]
 end
 
+# Runs git in the current directory, capturing stdout and stderr together.
+def git(*git_args)
+  out = IO.popen(["git", *git_args], err: [:child, :out], &:read)
+  [out, $?]
+end
+
+def git!(*git_args)
+  out, status = git(*git_args)
+  unless status.success?
+    abort "git #{git_args.join(" ")} failed:\n#{out}"
+  end
+  out
+end
+
 if versions.empty?
   rubies = [[nil, nil, []]]
 else
@@ -129,62 +130,173 @@ else
   }
 end
 
+# Each checkout is [label, sha]. Without --sha we stay where we are and label
+# nothing. With --sha the current checkout is always the first variant, so
+# there's a baseline to compare against.
+original_ref = nil
+original_sha = nil
+
+if revs.empty?
+  checkouts = [[nil, nil]]
+else
+  dirty = git!("status", "--porcelain", "-uno")
+  unless dirty.empty?
+    abort "Working tree has uncommitted changes to tracked files; commit or stash them before using --sha:\n#{dirty}"
+  end
+
+  original_sha = git!("rev-parse", "HEAD").strip
+  branch, branch_status = git("symbolic-ref", "--short", "HEAD")
+  original_ref = branch_status.success? ? branch.strip : nil
+
+  full_shas = [original_sha]
+  revs.each do |rev|
+    if rev.include?("..")
+      full_shas.concat(git!("rev-list", "--reverse", rev).split("\n"))
+    else
+      full_shas << git!("rev-parse", "--verify", "#{rev}^{commit}").strip
+    end
+  end
+  full_shas.uniq!
+
+  checkouts = full_shas.map { |sha| [git!("rev-parse", "--short", sha).strip, sha] }
+  # Prefer the branch name for the baseline when we have one.
+  checkouts[0][0] = original_ref if original_ref
+end
+
+# Variants are the product of checkouts and rubies.
+variants = checkouts.flat_map { |sha_label, sha|
+  rubies.map { |version_str, ruby, extra_flags|
+    label =
+      if version_str && sha_label
+        "ruby #{version_str} @ #{sha_label}"
+      elsif version_str
+        "ruby #{version_str}"
+      else
+        sha_label
+      end
+    [label, ruby, extra_flags, sha]
+  }
+}
+
+overrides = []
+overrides << "save: IO.open(3, 'w')"
+overrides << "summary: false" if variants.size > 1 || runs > 1
+overrides << "time: #{bench_time}" if bench_time
+overrides << "warmup: #{warmup_time}" if warmup_time
+overrides << "debug: true" if debug
+overrides << "frozen_string_literal: false" unless frozen_string_literal
+
+script_source = +""
+script_source << "# frozen_string_literal: true\n" if frozen_string_literal
+script_source << "require 'ips'\n"
+script_source << "IPS.overrides = { #{overrides.join(", ")} }\n"
+prelude.each { |code| script_source << code << "\n" }
+
+if inline_code.any?
+  script_source << "IPS.run do |x|\n"
+  inline_code.each do |code|
+    script_source << "  x.report(#{code.inspect}, #{code.inspect})\n"
+  end
+  script_source << "end\n"
+else
+  script_source << "load(#{script.dump})\n"
+end
+ruby_args = ["-e", script_source]
+
 $LOAD_PATH.unshift(LIB_DIR)
 require "ips"
 
-results_by_ruby = rubies.map { [] }
+# Check out commits in place, never changing directory and never creating a
+# worktree, so the ruby, the bundle, and any per-directory setup stay identical
+# across every variant.
+current_sha = original_sha
+moved = false
+
+checkout = lambda do |sha|
+  next if sha.nil? || sha == current_sha
+  $stdout.puts "git checkout --detach #{sha}" if verbose
+  out, status = git("checkout", "--detach", sha)
+  abort "git checkout --detach #{sha} failed:\n#{out}" unless status.success?
+  moved = true
+  current_sha = sha
+end
+
+restore = lambda do
+  next unless moved
+  moved = false
+  git_args = original_ref ? ["checkout", original_ref] : ["checkout", "--detach", original_sha]
+  out, status = git(*git_args)
+  if status.success?
+    current_sha = original_sha
+  else
+    $stderr.puts "WARNING: failed to restore #{original_ref || original_sha}:\n#{out}"
+  end
+end
+
+trap("INT") do
+  restore.call
+  exit 130
+end if original_sha
+
+results_by_variant = variants.map { [] }
 quiet = runs > 1
 
-runs.times do |run_index|
-  rubies.each_with_index do |(version_str, ruby, extra_flags), ruby_index|
-    if ruby
-      env = { "GEM_HOME" => nil, "GEM_PATH" => nil }
-    else
-      ruby = "ruby"
-      env = {}
-    end
+begin
+  runs.times do |run_index|
+    variants.each_with_index do |(label, ruby, extra_flags, sha), variant_index|
+      checkout.call(sha)
 
-    cmd = [ruby, *extra_flags, *ruby_flags]
-    cmd << "-v" unless quiet
-    cmd.push("-I", LIB_DIR, *ruby_args)
-    $stdout.puts cmd.map { |a| shell_quote(a) }.join(" ") if verbose
+      if ruby
+        env = { "GEM_HOME" => nil, "GEM_PATH" => nil }
+      else
+        ruby = "ruby"
+        env = {}
+      end
 
-    rd, wr = IO.pipe
-    spawn_opts = { 3 => wr }
-    spawn_opts[:out] = File::NULL if quiet
+      cmd = [ruby, *extra_flags, *ruby_flags]
+      cmd << "-v" unless quiet
+      cmd.push("-I", LIB_DIR, *ruby_args)
+      $stdout.puts cmd.map { |a| shell_quote(a) }.join(" ") if verbose
 
-    pid = spawn(env, *cmd, **spawn_opts)
-    wr.close
+      rd, wr = IO.pipe
+      spawn_opts = { 3 => wr }
+      spawn_opts[:out] = File::NULL if quiet
 
-    if quiet
-      $stdout.print "\rRun #{run_index + 1}/#{runs}"
-      $stdout.print " (#{version_str})" if version_str
-      $stdout.print "..."
-      $stdout.flush
-    end
+      pid = spawn(env, *cmd, **spawn_opts)
+      wr.close
 
-    until rd.eof?
-      result = Marshal.load(rd)
-      result.ruby_executable = ruby
-      result.run_label = "ruby #{version_str}" if version_str
-      results_by_ruby[ruby_index] << result
-    end
-    rd.close
-    _, status = Process.wait2(pid)
-    unless status.success?
-      $stderr.puts if quiet
-      cmd_str = cmd.map { |a| shell_quote(a) }.join(" ")
-      abort "Command failed (exit #{status.exitstatus}): #{cmd_str}"
+      if quiet
+        $stdout.print "\rRun #{run_index + 1}/#{runs}"
+        $stdout.print " (#{label})" if label
+        $stdout.print "..."
+        $stdout.flush
+      end
+
+      until rd.eof?
+        result = Marshal.load(rd)
+        result.ruby_executable = ruby
+        result.run_label = label if label
+        results_by_variant[variant_index] << result
+      end
+      rd.close
+      _, status = Process.wait2(pid)
+      unless status.success?
+        $stderr.puts if quiet
+        cmd_str = cmd.map { |a| shell_quote(a) }.join(" ")
+        abort "Command failed (exit #{status.exitstatus}): #{cmd_str}"
+      end
+      puts unless quiet
     end
-    puts unless quiet
   end
+ensure
+  restore.call
 end
 $stderr.puts if quiet
 
-results = results_by_ruby.flatten
+results = results_by_variant.flatten
 
 if runs > 1 && results.any?
-  IPS::Result.compare_aggregate(results_by_ruby, out: $stdout)
+  IPS::Result.compare_aggregate(results_by_variant, out: $stdout)
 else
   IPS::Result.compare(results)
 end

data/lib/ips/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module IPS
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/ips/warmup.rb

@@ -40,6 +40,10 @@ module IPS
     private
 
     def cycles_for(time_ns, iters, target_ns)
+      # Below the clock's resolution, so there's nothing to extrapolate from.
+      # Grow the batch and let the caller measure again.
+      return iters * 2 if time_ns <= 0
+
       c = (target_ns * iters) / time_ns
       c < 1 ? 1 : c
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ips
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - John Hawthorn
@@ -60,7 +60,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Iterations per second benchmarking with statistical analysis
 test_files: []