gitlab-derailed_benchmarks 1.6.1

data/lib/derailed_benchmarks/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DerailedBenchmarks
+  VERSION = "1.6.1"
+end

data/test/derailed_benchmarks/core_ext/kernel_require_test.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class KernelRequireTest < ActiveSupport::TestCase
+
+  setup do
+    require 'derailed_benchmarks/core_ext/kernel_require'
+    GC.disable
+  end
+
+  teardown do
+    GC.enable
+  end
+
+  def assert_node_in_parent(file_name, parent)
+    file = fixtures_dir(File.join("require", file_name))
+    node = parent[file]
+    assert node,                    "Expected:\n#{parent.children}\nto include:\n#{file.inspect}"
+    assert node.cost < parent.cost, "Expected:\n#{node.inspect}\nto cost less than:\n#{parent.inspect}" unless parent == TOP_REQUIRE
+    node
+  end
+
+  test "core extension profiles useage" do
+    require fixtures_dir("require/parent_one.rb")
+    parent    = assert_node_in_parent("parent_one.rb", TOP_REQUIRE)
+    assert_node_in_parent("child_one.rb", parent)
+    child_two = assert_node_in_parent("child_two.rb", parent)
+    assert_node_in_parent("relative_child", parent)
+    assert_node_in_parent("relative_child_two", parent)
+    assert_node_in_parent("raise_child.rb", child_two)
+  end
+end

data/test/derailed_benchmarks/require_tree_test.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class RequireTree < ActiveSupport::TestCase
+
+  def tree(name)
+    DerailedBenchmarks::RequireTree.new(name)
+  end
+
+  def teardown
+    DerailedBenchmarks::RequireTree.const_set("REQUIRED_BY",  {})
+  end
+
+  test "default_cost" do
+    parent =  tree("parent")
+    assert_equal 0,       parent.cost
+    value       = rand(0..100)
+    parent.cost = value
+
+    assert_equal value, parent.cost
+  end
+
+  test "stores child" do
+    parent =  tree("parent")
+    child  =  tree("child")
+    parent << child
+
+    # [](name)
+    assert_equal child,   parent["child"]
+    # children
+    assert_equal [child], parent.children
+    assert_equal [child], parent.sorted_children
+  end
+
+  test "sorts children" do
+    parent = tree("parent")
+    parent.cost = rand(5..10)
+    small  = tree("small")
+    small.cost = rand(10..100)
+
+    large  = tree("large")
+    large.cost = small.cost + 1
+
+    parent << small
+    parent << large
+
+    expected = [large, small]
+    assert_equal expected, parent.sorted_children
+
+    expected = <<-OUT
+parent: #{ parent.cost.round(4) } MiB
+  large: #{ large.cost.round(4) } MiB
+  small: #{ small.cost.round(4) } MiB
+OUT
+    capture  = StringIO.new
+
+    parent.print_sorted_children(0, capture)
+
+    assert_equal expected, capture.string
+  end
+
+  test "attributes duplicate children" do
+    parent = tree("parent")
+    parent.cost = rand(5..10)
+    small  = tree("small")
+    small.cost = rand(10..100)
+
+    large  = tree("large")
+    large.cost = small.cost + 1
+
+    dup    = tree("large")
+    dup.cost = 0.4
+    small << dup
+
+    parent << small
+    parent << large
+
+    expected = [large, small]
+    assert_equal expected, parent.sorted_children
+
+    expected = [dup]
+    assert_equal expected, small.sorted_children
+
+    expected = <<-OUT
+parent: #{ parent.cost.round(4) } MiB
+  large: #{ large.cost.round(4) } MiB (Also required by: small)
+  small: #{ small.cost.round(4) } MiB
+    large: #{ dup.cost.round(4) } MiB (Also required by: parent)
+OUT
+    capture  = StringIO.new
+    parent.print_sorted_children(0, capture)
+    assert_equal expected, capture.string
+  end
+end

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class StatsFromDirTest < ActiveSupport::TestCase
+  test "that it works" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+
+    newest = stats.newest
+    oldest = stats.oldest
+
+    assert newest.average < oldest.average
+
+    assert_equal "winner", newest.name
+    assert_equal "loser", oldest.name
+
+    assert_in_delta 0.26, stats.d_max, 0.01
+    assert_in_delta 0.2145966026289347, stats.d_critical, 0.00001
+    assert_equal true, stats.significant?
+
+    format = DerailedBenchmarks::StatsFromDir::FORMAT
+    assert_equal "1.0062", format % stats.x_faster
+    assert_equal "0.6147", format % stats.percent_faster
+
+    assert_equal "11.3844", format % newest.median
+  end
+
+  test "alignment" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+    def stats.percent_faster
+      -0.1
+    end
+
+    def stats.x_faster
+      0.9922
+    end
+
+    assert_equal 1, stats.align.length
+  end
+
+  test "banner faster" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+    newest = stats.newest
+    oldest = stats.oldest
+
+    # Test fixture for banner
+    def stats.d_max
+      "0.037"
+    end
+
+    def stats.d_critical
+      "0.001"
+    end
+
+    def newest.median
+      10.5
+    end
+
+    def oldest.median
+      11.0
+    end
+
+    expected = <<~EOM
+[winner] "I am the new commit" - (10.5 seconds)
+  FASTER 🚀🚀🚀 by:
+    1.0476x [older/newer]
+    4.5455% [(older - newer) / older * 100]
+[loser] "Old commit" - (11.0 seconds)
+EOM
+
+    actual = StringIO.new
+    stats.banner(actual)
+
+    assert_match expected, actual.string
+  end
+
+  test "banner slower" do
+    dir = fixtures_dir("stats/significant")
+    branch_info = {}
+    branch_info["loser"]  = { desc: "I am the new commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
+    branch_info["winner"] = { desc: "Old commit", time: Time.now - 10, file: dir.join("winner.bench.txt"), name: "winner" }
+    stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
+    newest = stats.newest
+    oldest = stats.oldest
+
+    def oldest.median
+      10.5
+    end
+
+    def newest.median
+      11.0
+    end
+
+    expected = <<~EOM
+[loser] "I am the new commit" - (11.0 seconds)
+  SLOWER 🐢🐢🐢 by:
+     0.9545x [older/newer]
+    -4.7619% [(older - newer) / older * 100]
+[winner] "Old commit" - (10.5 seconds)
+EOM
+
+    actual = StringIO.new
+    stats.banner(actual)
+
+    assert_match expected, actual.string
+  end
+
+  test "stats from samples with slightly different sizes" do
+    stats = DerailedBenchmarks::StatsFromDir.new({})
+    out = stats.statistical_test([100,101,102, 100, 101, 99], [1,3, 3, 2])
+    assert out[:alternative]
+  end
+end

data/test/derailed_test.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class DerailedBenchmarksTest < ActiveSupport::TestCase
+  test "truth" do
+    assert_kind_of Module, DerailedBenchmarks
+  end
+
+  test "gem_is_bundled?" do
+    assert DerailedBenchmarks.gem_is_bundled?("rack")
+    refute DerailedBenchmarks.gem_is_bundled?("wicked")
+  end
+end

data/test/fixtures/require/child_one.rb

@@ -0,0 +1,4 @@
+class ChildOne
+  @retained = +""
+  50_000.times.map { @retained << "A" }
+end

data/test/fixtures/require/child_two.rb

@@ -0,0 +1,9 @@
+class ChildTwo
+  @retained = +""
+  200_000.times.map { @retained << "A" }
+end
+
+begin
+  require File.expand_path('../raise_child.rb', __FILE__)
+rescue
+end

data/test/fixtures/require/parent_one.rb

@@ -0,0 +1,8 @@
+class ParentOne
+  @retained = +""
+  1_000_000.times.map { @retained << "A" }
+end
+require File.expand_path('../child_one.rb', __FILE__)
+require File.expand_path('../child_two.rb', __FILE__)
+require_relative 'relative_child'
+require_relative File.expand_path('relative_child_two', __dir__)

data/test/fixtures/require/raise_child.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class RaiseChild
+end
+
+raise "Ohno"

data/test/fixtures/require/relative_child.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class RelativeChild
+end

data/test/fixtures/require/relative_child_two.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class RelativeChildTwo
+end

data/test/fixtures/stats/significant/loser.bench.txt

@@ -0,0 +1,100 @@
+  9.590142   0.831269  10.457801 ( 11.437769)
+  9.836019   0.837319  10.728024 ( 11.792425)
+  9.889497   0.837097  10.762795 ( 11.747066)
+  9.532349   0.835770  10.407767 ( 11.343758)
+  9.498824   0.821246  10.366225 ( 11.282013)
+  9.531201   0.834812  10.412715 ( 11.330127)
+  9.583804   0.830178  10.449280 ( 11.384867)
+  9.681340   0.834776  10.553697 ( 11.496842)
+  9.629079   0.820234  10.487276 ( 11.406677)
+  9.616845   0.818370  10.481193 ( 11.395825)
+  9.738706   0.825397  10.600923 ( 11.541978)
+  9.613132   0.827242  10.477559 ( 11.406065)
+  9.486763   0.818159  10.342470 ( 11.305299)
+  9.590618   0.833308  10.468615 ( 11.428548)
+  9.725126   0.842955  10.604727 ( 11.580922)
+  9.598757   0.846951  10.491222 ( 11.462324)
+  9.484803   0.836242  10.366269 ( 11.328111)
+  9.591107   0.818305  10.455931 ( 11.381660)
+  9.745620   0.841107  10.623715 ( 11.572584)
+  9.670502   0.826673  10.535566 ( 11.464770)
+  9.573957   0.825235  10.439166 ( 11.358507)
+  9.468308   0.817744  10.330541 ( 11.240717)
+  9.799312   0.833922  10.670213 ( 11.645225)
+  9.575413   0.828039  10.444712 ( 11.366111)
+  9.632808   0.828399  10.506581 ( 11.439761)
+  9.599766   0.829294  10.467003 ( 11.427869)
+  9.521930   0.828257  10.388242 ( 11.347915)
+  9.842608   0.815513  10.694427 ( 11.667158)
+  9.590377   0.837459  10.467418 ( 11.433524)
+  9.729984   0.819101  10.587020 ( 11.539660)
+  9.540025   0.819396  10.396442 ( 11.314444)
+  9.615953   0.827258  10.479946 ( 11.414527)
+  9.572009   0.824862  10.438432 ( 11.362800)
+  9.612657   0.818645  10.476568 ( 11.385235)
+  9.755889   0.823267  10.615302 ( 11.545301)
+  9.493372   0.813202  10.345646 ( 11.254617)
+  9.529610   0.816457  10.391484 ( 11.305237)
+  9.575646   0.828898  10.449636 ( 11.374993)
+  9.533278   0.828915  10.405592 ( 11.347798)
+  9.692731   0.845925  10.577313 ( 11.545701)
+  9.662406   0.835481  10.543032 ( 11.511367)
+  9.588803   0.834782  10.468166 ( 11.427231)
+  9.696038   0.832612  10.573877 ( 11.545533)
+  9.612567   0.833363  10.491381 ( 11.410244)
+  9.471584   0.836005  10.352883 ( 11.303213)
+  9.682906   0.829932  10.558423 ( 11.466843)
+  9.676123   0.825750  10.548111 ( 11.468789)
+  9.509686   0.826678  10.380435 ( 11.290658)
+  9.552683   0.826631  10.421387 ( 11.337799)
+  9.579964   0.829423  10.447095 ( 11.358198)
+  9.506519   0.812635  10.357147 ( 11.313867)
+  9.654363   0.839408  10.531093 ( 11.515562)
+  9.576167   0.833579  10.447897 ( 11.421267)
+  9.498507   0.826285  10.370417 ( 11.336780)
+  9.758637   0.842156  10.645638 ( 11.595915)
+  9.635031   0.836329  10.516094 ( 11.475492)
+  9.934052   0.825471  10.794286 ( 11.812346)
+  9.652537   0.821982  10.520060 ( 11.434903)
+  9.526788   0.820300  10.384780 ( 11.306397)
+  9.473180   0.812507  10.329689 ( 11.233813)
+  9.862016   0.841529  10.757393 ( 11.734586)
+  9.534627   0.821267  10.392666 ( 11.313970)
+  9.640884   0.837997  10.515254 ( 11.489616)
+  9.535812   0.826216  10.407273 ( 11.318032)
+  9.588703   0.851935  10.476997 ( 11.462256)
+  9.574569   0.834756  10.454909 ( 11.404434)
+  9.650073   0.839516  10.535755 ( 11.488113)
+  9.551275   0.822510  10.415396 ( 11.378195)
+  9.627255   0.829954  10.500136 ( 11.458503)
+  9.560385   0.814457  10.419578 ( 11.333235)
+  9.572809   0.819290  10.438854 ( 11.349594)
+  9.660163   0.824722  10.530198 ( 11.443437)
+  9.661319   0.837408  10.550881 ( 11.512634)
+  9.637423   0.837322  10.520727 ( 11.432594)
+  9.664915   0.825478  10.526599 ( 11.464716)
+  9.644935   0.814938  10.505644 ( 11.424535)
+  9.799771   0.835598  10.671993 ( 11.613622)
+  9.791496   0.840368  10.676233 ( 11.643770)
+  9.760101   0.850254  10.648067 ( 11.619884)
+  9.784358   0.829651  10.658058 ( 11.632889)
+  9.727932   0.844568  10.616464 ( 11.578881)
+  9.776381   0.847439  10.663001 ( 11.648257)
+  9.839221   0.835333  10.714699 ( 11.670378)
+  9.697873   0.836432  10.570815 ( 11.541265)
+  9.867105   0.836122  10.741859 ( 11.681261)
+  9.675377   0.826509  10.539536 ( 11.465271)
+  9.703541   0.830895  10.578611 ( 11.502074)
+  9.717583   0.832110  10.586737 ( 11.531415)
+  9.784151   0.842351  10.662311 ( 11.647167)
+  9.741646   0.832834  10.612608 ( 11.580701)
+  9.687384   0.798745  10.525026 ( 11.493736)
+  9.698579   0.851183  10.586010 ( 11.588731)
+  9.712651   0.823867  10.573837 ( 11.540969)
+  9.657543   0.829349  10.524768 ( 11.443846)
+  9.675987   0.807980  10.521943 ( 11.451106)
+  9.744757   0.817850  10.600094 ( 11.535379)
+  9.683474   0.836913  10.557015 ( 11.525771)
+  9.922540   0.843157  10.805096 ( 11.808377)
+  9.696813   0.821768  10.554695 ( 11.464342)
+  9.760965   0.836511  10.636968 ( 11.594082)

data/test/fixtures/stats/significant/winner.bench.txt

@@ -0,0 +1,100 @@
+  9.558387   0.795543  10.392696 ( 11.309311)
+  9.524045   0.803011  10.364301 ( 11.318477)
+  9.534609   0.804759  10.383564 ( 11.340585)
+  9.535700   0.800444  10.373048 ( 11.289682)
+  9.532372   0.794646  10.371722 ( 11.287656)
+  9.556350   0.822103  10.425949 ( 11.413659)
+  9.586525   0.824110  10.456246 ( 11.429651)
+  9.551907   0.830509  10.428443 ( 11.411978)
+  9.518711   0.834491  10.398652 ( 11.376422)
+  9.569772   0.827570  10.442956 ( 11.413585)
+  9.618950   0.829319  10.485139 ( 11.440848)
+  9.556727   0.807981  10.401758 ( 11.328267)
+  9.480701   0.804683  10.322360 ( 11.245781)
+  9.563369   0.801410  10.409686 ( 11.334188)
+  9.493082   0.805298  10.335983 ( 11.248441)
+  9.681861   0.803602  10.524930 ( 11.456107)
+  9.614529   0.781155  10.444055 ( 11.364476)
+  9.597825   0.806409  10.442217 ( 11.365743)
+  9.538346   0.813941  10.388972 ( 11.346084)
+  9.538091   0.808328  10.391165 ( 11.346197)
+  9.502600   0.812638  10.360783 ( 11.306602)
+  9.571149   0.826238  10.449697 ( 11.411387)
+  9.531260   0.821429  10.390722 ( 11.532200)
+  9.611447   0.783734  10.431579 ( 11.351863)
+  9.533522   0.806067  10.384192 ( 11.296454)
+  9.586843   0.820340  10.444013 ( 11.383357)
+  9.615441   0.804255  10.456321 ( 11.385184)
+  9.462530   0.803438  10.302507 ( 11.223665)
+  9.676985   0.789649  10.511461 ( 11.427901)
+  9.574692   0.816601  10.427670 ( 11.374204)
+  9.596892   0.803796  10.437442 ( 11.362358)
+  9.562942   0.815001  10.415687 ( 11.383593)
+  9.622502   0.804110  10.470848 ( 11.488275)
+  9.766782   0.828892  10.632272 ( 11.635267)
+  9.612909   0.804247  10.455650 ( 11.421374)
+  9.537415   0.805782  10.390754 ( 11.294518)
+  9.763286   0.805568  10.614687 ( 11.533764)
+  9.507627   0.806313  10.350967 ( 11.299277)
+  9.469710   0.803944  10.312100 ( 11.232190)
+  9.535007   0.795200  10.371960 ( 11.292289)
+  9.530755   0.797043  10.372644 ( 11.289316)
+  9.588961   0.806621  10.431681 ( 11.368492)
+  9.592512   0.808849  10.446866 ( 11.359820)
+  9.653610   0.803463  10.501491 ( 11.419194)
+  9.547770   0.812003  10.405405 ( 11.368690)
+  9.682181   0.812963  10.530854 ( 11.485025)
+  9.491677   0.807396  10.344595 ( 11.281067)
+  9.587365   0.813596  10.442915 ( 11.394766)
+  9.569528   0.814968  10.421925 ( 11.395829)
+  9.499610   0.806958  10.342308 ( 11.266410)
+  9.470981   0.802210  10.311858 ( 11.228286)
+  9.562924   0.794929  10.395599 ( 11.322258)
+  9.601453   0.810256  10.456259 ( 11.374217)
+  9.505371   0.799272  10.354669 ( 11.279456)
+  9.457992   0.795362  10.289520 ( 11.205184)
+  9.628120   0.787671  10.453407 ( 11.377989)
+  9.627611   0.805838  10.470388 ( 11.399739)
+  9.675034   0.812966  10.532779 ( 11.515440)
+  9.612906   0.807182  10.457964 ( 11.434272)
+  9.480996   0.803877  10.325013 ( 11.265876)
+  9.717399   0.823376  10.577638 ( 11.569749)
+  9.665028   0.809491  10.511645 ( 11.488256)
+  9.512832   0.805858  10.363675 ( 11.339722)
+  9.654066   0.807307  10.506755 ( 11.426100)
+  9.865550   0.794908  10.703626 ( 11.618194)
+  9.652618   0.793610  10.493186 ( 11.419415)
+  9.499487   0.796346  10.341364 ( 11.250758)
+  9.544258   0.797515  10.385862 ( 11.284281)
+  9.739863   0.794279  10.570723 ( 11.509588)
+  9.487554   0.785309  10.316760 ( 11.233325)
+  9.481721   0.803731  10.329705 ( 11.255686)
+  9.466643   0.802025  10.313663 ( 11.234516)
+  9.565479   0.798706  10.406513 ( 11.374955)
+  9.546849   0.818211  10.409684 ( 11.368566)
+  9.559145   0.813582  10.418666 ( 11.401304)
+  9.547626   0.787676  10.380384 ( 11.305801)
+  9.731920   0.806463  10.576084 ( 11.499545)
+  9.634309   0.804944  10.477565 ( 11.398455)
+  9.663389   0.797418  10.499369 ( 11.418504)
+  9.741374   0.818880  10.597056 ( 11.575796)
+  9.683985   0.804469  10.527844 ( 11.457434)
+  9.739006   0.808335  10.587852 ( 11.513780)
+  9.761998   0.818945  10.618427 ( 11.614032)
+  9.737508   0.819736  10.593885 ( 11.588014)
+  9.735949   0.821038  10.595284 ( 11.552597)
+  9.750022   0.814069  10.601283 ( 11.567239)
+  9.700983   0.801116  10.542112 ( 11.471005)
+  9.720313   0.798207  10.555314 ( 11.473235)
+  9.685407   0.811225  10.534452 ( 11.467112)
+  9.677940   0.809071  10.526291 ( 11.447495)
+  9.609120   0.813429  10.467227 ( 11.372680)
+  9.712403   0.810281  10.560867 ( 11.485852)
+  9.748022   0.817132  10.603028 ( 11.522460)
+  9.737389   0.801790  10.576720 ( 11.522855)
+  9.709541   0.795349  10.542238 ( 11.544047)
+  9.658660   0.819237  10.515718 ( 11.520783)
+  9.765426   0.829642  10.632481 ( 11.615062)
+  9.731822   0.809695  10.578871 ( 11.558062)
+  9.575340   0.800450  10.421430 ( 11.318465)
+  9.682845   0.796365  10.515529 ( 11.435012)