memo_wise 1.0.0 → 1.4.0

data/benchmarks/benchmarks.rb

@@ -2,17 +2,21 @@
 
 require "benchmark/ips"
 
+require "tempfile"
 require "memo_wise"
 
 # Some gems do not yet work in Ruby 3 so we only require them if they're loaded
 # in the Gemfile.
-%w[memery memoist memoized memoizer].
+%w[memery memoist memoized memoizer ddmemoize dry-core].
   each { |gem| require gem if Gem.loaded_specs.key?(gem) }
 
 # The VERSION constant does not get loaded above for these gems.
 %w[memoized memoizer].
   each { |gem| require "#{gem}/version" if Gem.loaded_specs.key?(gem) }
 
+# The Memoizable module from dry-core needs to be required manually
+require "dry/core/memoizable" if Gem.loaded_specs.key?("dry-core")
+
 class BenchmarkSuiteWithoutGC
   def warming(*)
     run_gc
@@ -36,7 +40,7 @@ class BenchmarkSuiteWithoutGC
 end
 suite = BenchmarkSuiteWithoutGC.new
 
-BenchmarkGem = Struct.new(:klass, :inheritance_method, :memoization_method) do
+BenchmarkGem = Struct.new(:klass, :activation_code, :memoization_method) do
   def benchmark_name
     "#{klass} (#{klass::VERSION})"
   end
@@ -47,204 +51,171 @@ end
 # NOTE: Some gems do not yet work in Ruby 3 so we only test with them if they've
 # been `require`d.
 BENCHMARK_GEMS = [
-  BenchmarkGem.new(MemoWise, :prepend, :memo_wise),
-  (BenchmarkGem.new(Memery, :include, :memoize) if defined?(Memery)),
-  (BenchmarkGem.new(Memoist, :extend, :memoize) if defined?(Memoist)),
-  (BenchmarkGem.new(Memoized, :include, :memoize) if defined?(Memoized)),
-  (BenchmarkGem.new(Memoizer, :include, :memoize) if defined?(Memoizer))
+  BenchmarkGem.new(MemoWise, "prepend MemoWise", :memo_wise),
+  (BenchmarkGem.new(DDMemoize, "DDMemoize.activate(self)", :memoize) if defined?(DDMemoize)),
+  (BenchmarkGem.new(Dry::Core, "include Dry::Core::Memoizable", :memoize) if defined?(Dry::Core)),
+  (BenchmarkGem.new(Memery, "include Memery", :memoize) if defined?(Memery)),
+  (BenchmarkGem.new(Memoist, "extend Memoist", :memoize) if defined?(Memoist)),
+  (BenchmarkGem.new(Memoized, "include Memoized", :memoize) if defined?(Memoized)),
+  (BenchmarkGem.new(Memoizer, "include Memoizer", :memoize) if defined?(Memoizer))
 ].compact.shuffle
 
 # Use metaprogramming to ensure that each class is created in exactly the
 # the same way.
 BENCHMARK_GEMS.each do |benchmark_gem|
-  # rubocop:disable Security/Eval
-  # rubocop:disable Style/DocumentDynamicEvalDefinition
-  eval <<-CLASS, binding, __FILE__, __LINE__ + 1
+  eval <<~HEREDOC, binding, __FILE__, __LINE__ + 1 # rubocop:disable Security/Eval
+    # For these methods, we alternately return truthy and falsey values in
+    # order to benchmark memoization when the result of a method is falsey.
+    #
+    # We do this by checking if the first argument to a method is even.
     class #{benchmark_gem.klass}Example
-      #{benchmark_gem.inheritance_method} #{benchmark_gem.klass}
+      #{benchmark_gem.activation_code}
 
       def no_args
         100
       end
       #{benchmark_gem.memoization_method} :no_args
 
+      # For the no_args case, we can't depend on arguments to alternate between
+      # returning truthy and falsey values, so instead make two separate
+      # no_args methods
+      def no_args_falsey
+        nil
+      end
+      #{benchmark_gem.memoization_method} :no_args_falsey
+
       def one_positional_arg(a)
-        100
+        100 if a.positive?
       end
       #{benchmark_gem.memoization_method} :one_positional_arg
 
       def positional_args(a, b)
-        100
+        100 if a.positive?
       end
       #{benchmark_gem.memoization_method} :positional_args
 
       def one_keyword_arg(a:)
-        100
+        100 if a.positive?
       end
       #{benchmark_gem.memoization_method} :one_keyword_arg
 
       def keyword_args(a:, b:)
-        100
+        100 if a.positive?
       end
       #{benchmark_gem.memoization_method} :keyword_args
 
       def positional_and_keyword_args(a, b:)
-        100
+        100 if a.positive?
       end
       #{benchmark_gem.memoization_method} :positional_and_keyword_args
 
       def positional_and_splat_args(a, *args)
-        100
+        100 if a.positive?
       end
       #{benchmark_gem.memoization_method} :positional_and_splat_args
 
       def keyword_and_double_splat_args(a:, **kwargs)
-        100
+        100 if a.positive?
       end
       #{benchmark_gem.memoization_method} :keyword_and_double_splat_args
 
       def positional_splat_keyword_and_double_splat_args(a, *args, b:, **kwargs)
-        100
+        100 if a.positive?
       end
       #{benchmark_gem.memoization_method} :positional_splat_keyword_and_double_splat_args
     end
-  CLASS
-  # rubocop:enable Style/DocumentDynamicEvalDefinition
-  # rubocop:enable Security/Eval
+  HEREDOC
 end
 
 # We pre-create argument lists for our memoized methods with arguments, so that
 # our benchmarks are running the exact same inputs for each case.
-N_UNIQUE_ARGUMENTS = 100
+#
+# NOTE: The proportion of falsey results is 1/N_UNIQUE_ARGUMENTS (because for
+# the methods with arguments we are truthy for all but the first unique argument
+# set, and for zero-arity methods we manually execute `no_args` N_TRUTHY_RESULTS
+# times per each execution of `no_args_falsey`). This number was selected as the
+# lowest number such that this logic:
+#
+#   output = hash[key]
+#   if output || hash.key?(key)
+#     output
+#   else
+#     hash[key] = _original_method(...)
+#   end
+#
+# is consistently faster for cached lookups than:
+#
+#   hash.fetch(key) do
+#     hash[key] = _original_method(...)
+#   end
+#
+# as a result of `Hash#[]` having less overhead than `Hash#fetch`.
+#
+# We believe this is a reasonable choice because we believe most memoized method
+# results will be truthy, and so that is the case we should most optimize for.
+# However, we do not want to completely remove falsey method results from these
+# benchmarks because we do want to catch performance regressions for that case,
+# since it has its own "hot path."
+N_UNIQUE_ARGUMENTS = 30
 ARGUMENTS = Array.new(N_UNIQUE_ARGUMENTS) { |i| [i, i + 1] }
-
-# We benchmark different cases separately, to ensure that slow performance in
-# one method or code path isn't hidden by fast performance in another.
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once to memoize the result value, so our benchmark only tests memoized
-    # retrieval time.
+N_TRUTHY_RESULTS = N_UNIQUE_ARGUMENTS - 1
+N_RESULT_DECIMAL_DIGITS = 2
+
+# Each method within these benchmarks is initially run once to memoize the
+# result value, so our benchmark only tests memoized retrieval time.
+benchmark_lambdas = [
+  lambda do |x, instance, benchmark_gem|
+    instance.no_args_falsey
     instance.no_args
 
-    x.report("#{benchmark_gem.benchmark_name}: ()") { instance.no_args }
-  end
-
-  x.compare!
-end
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once with each set of arguments to memoize the result values, so our
-    # benchmark only tests memoized retrieval time.
+    x.report("#{benchmark_gem.benchmark_name}: ()") do
+      instance.no_args_falsey
+      N_TRUTHY_RESULTS.times { instance.no_args }
+    end
+  end,
+  lambda do |x, instance, benchmark_gem|
     ARGUMENTS.each { |a, _| instance.one_positional_arg(a) }
 
     x.report("#{benchmark_gem.benchmark_name}: (a)") do
       ARGUMENTS.each { |a, _| instance.one_positional_arg(a) }
     end
-  end
-
-  x.compare!
-end
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once with each set of arguments to memoize the result values, so our
-    # benchmark only tests memoized retrieval time.
+  end,
+  lambda do |x, instance, benchmark_gem|
     ARGUMENTS.each { |a, b| instance.positional_args(a, b) }
 
     x.report("#{benchmark_gem.benchmark_name}: (a, b)") do
       ARGUMENTS.each { |a, b| instance.positional_args(a, b) }
     end
-  end
-
-  x.compare!
-end
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once with each set of arguments to memoize the result values, so our
-    # benchmark only tests memoized retrieval time.
+  end,
+  lambda do |x, instance, benchmark_gem|
     ARGUMENTS.each { |a, _| instance.one_keyword_arg(a: a) }
 
     x.report("#{benchmark_gem.benchmark_name}: (a:)") do
       ARGUMENTS.each { |a, _| instance.one_keyword_arg(a: a) }
     end
-  end
-
-  x.compare!
-end
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once with each set of arguments to memoize the result values, so our
-    # benchmark only tests memoized retrieval time.
+  end,
+  lambda do |x, instance, benchmark_gem|
     ARGUMENTS.each { |a, b| instance.keyword_args(a: a, b: b) }
 
     x.report("#{benchmark_gem.benchmark_name}: (a:, b:)") do
       ARGUMENTS.each { |a, b| instance.keyword_args(a: a, b: b) }
     end
-  end
-
-  x.compare!
-end
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once with each set of arguments to memoize the result values, so our
-    # benchmark only tests memoized retrieval time.
+  end,
+  lambda do |x, instance, benchmark_gem|
     ARGUMENTS.each { |a, b| instance.positional_and_keyword_args(a, b: b) }
 
     x.report("#{benchmark_gem.benchmark_name}: (a, b:)") do
       ARGUMENTS.each { |a, b| instance.positional_and_keyword_args(a, b: b) }
     end
-  end
-
-  x.compare!
-end
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once with each set of arguments to memoize the result values, so our
-    # benchmark only tests memoized retrieval time.
+  end,
+  lambda do |x, instance, benchmark_gem|
     ARGUMENTS.each { |a, b| instance.positional_and_splat_args(a, b) }
 
     x.report("#{benchmark_gem.benchmark_name}: (a, *args)") do
       ARGUMENTS.each { |a, b| instance.positional_and_splat_args(a, b) }
     end
-  end
-
-  x.compare!
-end
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once with each set of arguments to memoize the result values, so our
-    # benchmark only tests memoized retrieval time.
+  end,
+  lambda do |x, instance, benchmark_gem|
     ARGUMENTS.each { |a, b| instance.keyword_and_double_splat_args(a: a, b: b) }
 
     x.report(
@@ -254,18 +225,8 @@ Benchmark.ips do |x|
         instance.keyword_and_double_splat_args(a: a, b: b)
       end
     end
-  end
-
-  x.compare!
-end
-
-Benchmark.ips do |x|
-  x.config(suite: suite)
-  BENCHMARK_GEMS.each do |benchmark_gem|
-    instance = Object.const_get("#{benchmark_gem.klass}Example").new
-
-    # Run once with each set of arguments to memoize the result values, so our
-    # benchmark only tests memoized retrieval time.
+  end,
+  lambda do |x, instance, benchmark_gem|
     ARGUMENTS.each do |a, b|
       instance.positional_splat_keyword_and_double_splat_args(a, b, b: b, a: a)
     end
@@ -279,6 +240,62 @@ Benchmark.ips do |x|
       end
     end
   end
+]
+
+# We benchmark different cases separately, to ensure that slow performance in
+# one method or code path isn't hidden by fast performance in another.
+benchmark_lambdas.map do |benchmark|
+  json_file = Tempfile.new
+
+  Benchmark.ips do |x|
+    x.config(suite: suite)
+    BENCHMARK_GEMS.each do |benchmark_gem|
+      instance = Object.const_get("#{benchmark_gem.klass}Example").new
+
+      benchmark.call(x, instance, benchmark_gem)
+    end
+
+    x.compare!
+    x.json! json_file.path
+  end
+
+  JSON.parse(json_file.read)
+end.each_with_index do |benchmark_json, i|
+  # We print a comparison table after we run each benchmark to copy into our
+  # README.md
+
+  # MemoWise will not appear in the comparison table, but we will use it to
+  # compare against other gems' benchmarks
+  memo_wise = benchmark_json.find { _1["name"].include?("MemoWise") }
+  benchmark_json.delete(memo_wise)
+
+  # Sort benchmarks by gem name to alphabetize our final output table.
+  benchmark_json.sort_by! { _1["name"] }
+
+  # Print headers based on the first benchmark_json
+  if i.zero?
+    benchmark_headers = benchmark_json.map do |benchmark_gem|
+      # Gem name is of the form:
+      # "MemoWise (1.1.0): ()"
+      # We use this mapping to get a header of the form
+      # "`MemoWise` (1.1.0)
+      gem_name_parts = benchmark_gem["name"].split
+      "`#{gem_name_parts[0]}` #{gem_name_parts[1][...-1]}"
+    end.join("|")
+    puts "|Method arguments|#{benchmark_headers}|"
+    puts "#{'|--' * (benchmark_json.size + 1)}|"
+  end
 
-  x.compare!
+  output_str = benchmark_json.map do |bgem|
+    # "%.2f" % 12.345 => "12.34" (instead of "12.35")
+    #   See: https://bugs.ruby-lang.org/issues/12548
+    # 1.00.round(2).to_s => "1.0" (instead of "1.00")
+    #
+    # So to round and format correctly, we first use Float#round and then %
+    "%.#{N_RESULT_DECIMAL_DIGITS}fx" %
+      (memo_wise["central_tendency"] / bgem["central_tendency"]).round(N_RESULT_DECIMAL_DIGITS)
+  end.join("|")
+
+  name = memo_wise["name"].partition(": ").last
+  puts "|`#{name}`#{' (none)' if name == '()'}|#{output_str}|"
 end