derailed_benchmarks 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cefb45902f24e4e4b0d8083b59ad9733005943db3df8a8b97b30552a27ff400b
-  data.tar.gz: 9d5921ac19df4db40e78ec2ca6396593f1f5235c3caaae84c332163ed3c5e4ec
+  metadata.gz: 581626c86185aaf6f5155ff71279a5bddc26beac617d894523b97ea534960f08
+  data.tar.gz: 93b3a57bcde0653c6682e0757137e4f5612e6e4f3444dd4fe94d556ec5ef77a8
 SHA512:
-  metadata.gz: 3335a72cc6b965b76f6804c3dd71ae5cf61d56f5203447870f0ad31a30c6cef5c47b28cf569311d7c033fe2898f2653ab041d7c7dceefe85e5501c4b79ab06b4
-  data.tar.gz: f591582236f863286b65d9a96b33b481e0a2c9cb05a6985ecb4123d121452ebda81a1b5d123f1d0a77c297939e8f6f2de16ca14dad892436109d819ff2b7ee91
+  metadata.gz: 0e9eb3bd1494b2cca18f2b61adc0bc62794cff238afb6c7713826aaea44f4987648520d735ffd93f2e81b2bd69dc7ac7adc5d0966fda75d3ebdc728ed6c5c476
+  data.tar.gz: 442a77045473b79c8a4bdeb82e3002f63c3eb1195cc321754ae8ebc905c02d0300c58eb5c95f8488e58b27fb33a7c7ca5a536b99da305a30470b32e2e788a6ef

data/.github/workflows/check_changelog.yml

@@ -0,0 +1,10 @@
+name: Check Changelog
+on: [pull_request]
+jobs:
+ build:
+   runs-on: ubuntu-latest
+   steps:
+   - uses: actions/checkout@v1
+   - name: Check that CHANGELOG is touched
+     run: |
+       cat $GITHUB_EVENT_PATH | jq .pull_request.title |  grep -i '\[\(\(changelog skip\)\|\(ci skip\)\)\]' ||  git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # A Log of Changes!
 
+## 1.4.1
+
+- Rake dependency now allows for Rake 13 (https://github.com/schneems/derailed_benchmarks/pull/151)
+
 ## 1.4.0
 
 - Allow configuration of `perf:ips` benchmark.

data/README.md

@@ -443,9 +443,6 @@ When the test is done it will output which commit "won" and by how much:
     1.0870x [older/newer]
     8.0026% [(older - newer) / older * 100]
 [13d6aa3a7b] "Merge pull request #36284 from kamipo/fix_eager_loading_with_string_joins" - (11.9255485 seconds)
-
-P-value: 4.635595463712749e-05
-Is significant? (P-value < 0.05): true
 ```
 
 You can provide this to the Rails team along with the example app you used to benchmark (so they can independently verify if needed).

data/derailed_benchmarks.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "get_process_mem", "~> 0"
   gem.add_dependency "benchmark-ips",   "~> 2"
   gem.add_dependency "rack",            ">= 1"
-  gem.add_dependency "rake",            "> 10", "< 13"
+  gem.add_dependency "rake",            "> 10", "< 14"
   gem.add_dependency "thor",            "~> 0.19"
   gem.add_dependency "ruby-statistics", ">= 2.1"
 

data/gemfiles/rails_6_0.gemfile

@@ -4,7 +4,7 @@
 
 source "https://rubygems.org"
 
-gem "rails", "6.0.0.rc1"
+gem "rails", "6.0.0"
 
 group :development, :test do
   gem "sqlite3", platform: [:ruby, :mswin, :mingw]

data/lib/derailed_benchmarks/core_ext/kernel_require.rb

@@ -77,6 +77,14 @@ Kernel.define_singleton_method(:require) do |file|
   end
 end
 
+class Object
+  private
+
+  def require(path)
+    Kernel.require(path)
+  end
+end
+
 # Don't forget to assign a cost to the top level
 cost_before_requiring_anything = GetProcessMem.new.mb
 TOP_REQUIRE.cost = cost_before_requiring_anything

data/lib/derailed_benchmarks/stats_from_dir.rb

@@ -42,16 +42,14 @@ module DerailedBenchmarks
 
     def call
       @files.each(&:call)
-      @stats = students_t_test
+      @stats = statistical_test
       self
     end
 
-    def students_t_test(series_1=oldest.values, series_2=newest.values)
-      StatisticalTest::TTest.perform(
-        alpha = 0.05,
-        :two_tail,
-        series_1,
-        series_2
+    def statistical_test(series_1=oldest.values, series_2=newest.values)
+      StatisticalTest::KSTest.two_samples(
+        group_one: series_1,
+        group_two: series_2
       )
     end
 
@@ -59,8 +57,12 @@ module DerailedBenchmarks
       @stats[:alternative]
     end
 
-    def p_value
-      @stats[:p_value].to_f
+    def d_max
+      @stats[:d_max].to_f
+    end
+
+    def d_critical
+      @stats[:d_critical].to_f
     end
 
     def x_faster
@@ -91,8 +93,11 @@ module DerailedBenchmarks
       io.puts
       io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
       io.puts "Samples: #{newest.values.length}"
-      io.puts "P-value: #{p_value}"
-      io.puts "Is significant? (P-value < 0.05): #{significant?}"
+      io.puts
+      io.puts "Test type: Kolmogorov Smirnov"
+      io.puts "Is significant? (max > critical): #{significant?}"
+      io.puts "D critical: #{d_critical}"
+      io.puts "D max: #{d_max}"
       io.puts
     end
   end

data/lib/derailed_benchmarks/tasks.rb

@@ -53,7 +53,7 @@ namespace :perf do
     end
 
     stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
-    ENV["DERAILED_STOP_VALID_COUNT"] ||= "10"
+    ENV["DERAILED_STOP_VALID_COUNT"] ||= "50"
     stop_valid_count = Integer(ENV["DERAILED_STOP_VALID_COUNT"])
 
     times_significant = 0

data/lib/derailed_benchmarks/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DerailedBenchmarks
-  VERSION = "1.4.0"
+  VERSION = "1.4.1"
 end

data/test/derailed_benchmarks/stats_from_dir_test.rb

@@ -18,8 +18,8 @@ class StatsFromDirTest < ActiveSupport::TestCase
     assert_equal "winner", newest.name
     assert_equal "loser", oldest.name
 
-    assert 3.6e-05 < stats.p_value
-    assert 3.8e-05 > stats.p_value
+    assert_in_delta 0.26, stats.d_max, 0.01
+    assert_in_delta 0.1730818382602285, stats.d_critical, 0.00001
     assert_equal true, stats.significant?
 
     assert_equal "1.0062", stats.x_faster
@@ -36,8 +36,12 @@ class StatsFromDirTest < ActiveSupport::TestCase
     oldest = stats.oldest
 
     # Test fixture for banner
-    def stats.p_value
-      "0.000037"
+    def stats.d_max
+      "0.037"
+    end
+
+    def stats.d_critical
+      "0.001"
     end
 
     def newest.average
@@ -95,7 +99,7 @@ EOM
 
   test "stats from samples with slightly different sizes" do
     stats = DerailedBenchmarks::StatsFromDir.new({})
-    out = stats.students_t_test([100,101,102], [1,3])
+    out = stats.statistical_test([100,101,102, 100, 101, 99], [1,3, 3, 2])
     assert out[:alternative]
   end
 end

data/test/rails_app/config/environment.rb

@@ -4,6 +4,8 @@
 require File.expand_path('../application', __FILE__)
 
 # Initialize the rails application
-Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true
+if Rails.application.config.active_record.sqlite3.respond_to?(:represent_boolean_as_integer)
+  Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true
+end
 
 Dummy::Application.initialize!

data/test/rails_app/db/schema.rb

@@ -32,5 +32,4 @@ ActiveRecord::Schema.define(version: 20141210070547) do
 
   add_index "users", ["email"], name: "index_users_on_email", unique: true
   add_index "users", ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
-
 end

data/test/test_helper.rb

@@ -29,16 +29,19 @@ Capybara.default_driver   = :rack_test
 Capybara.default_selector = :css
 
 require_relative "rails_app/config/environment"
-ActiveRecord::Migrator.migrations_paths = [File.expand_path("../rails_app/db/migrate", __dir__)]
 
-if Rails.gem_version >= Gem::Version.new('5.2.0')
-  ActiveRecord::MigrationContext
-    .new(File.expand_path("../rails_app/db/migrate/", __FILE__))
-    .migrate
+# https://github.com/plataformatec/devise/blob/master/test/orm/active_record.rb
+migrate_path = File.expand_path("../rails_app/db/migrate", __FILE__)
+if Rails.version.start_with? '6'
+  ActiveRecord::MigrationContext.new(migrate_path, ActiveRecord::SchemaMigration).migrate
+elsif Rails.version.start_with? '5.2'
+  ActiveRecord::MigrationContext.new(migrate_path).migrate
 else
-  ActiveRecord::Migrator.migrate(File.expand_path("../rails_app/db/migrate/", __FILE__))
+  ActiveRecord::Migrator.migrate(migrate_path)
 end
 
+ActiveRecord::Migration.maintain_test_schema!
+
 # Load support files
 Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: derailed_benchmarks
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Richard Schneeman
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-25 00:00:00.000000000 Z
+date: 2019-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: heapy
@@ -89,7 +89,7 @@ dependencies:
         version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '13'
+        version: '14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -99,7 +99,7 @@ dependencies:
         version: '10'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '13'
+        version: '14'
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -218,6 +218,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/check_changelog.yml"
 - ".gitignore"
 - ".travis.yml"
 - Appraisals