theorem 1.2.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 433696da9dd29c04e52b86d753dbbc73aba33fdb3e2a14a1c7e03dc4f7f1c2c9
-  data.tar.gz: 42f41fc806176cecbcce43441ace89126e751bed7ab23b103b91477451b243c5
+  metadata.gz: '09d0acbeb340968a19cb27122ba6c852b66b2cc4347fdfca7f6ba6a9f7781ce0'
+  data.tar.gz: aef702741a59ff1a11f08e6225b0804b30b813ed0cc8eb5184cdd50c54587901
 SHA512:
-  metadata.gz: 336ef51f1a4b3bfe1b34da109e8ab193a10aff873a3e4151c4d3a754be479bec455aae731886f8a35c769effc1f89b439ef66055b6a62b096b5fa53ec966c3a4
-  data.tar.gz: e5454f1c81f833480f2b5eccbe9f516f9a4826c7ae743f47c1e4112123b0c558220b2dde81eb918929622aca286ba67b8679f3a3cc1a71dee14313a80e8f1b9c
+  metadata.gz: 6055b29fd80529f514b73363a18582122a5e23b312fda318b3d83a546bb7d8b062fcaf87f57858340e432bc1327a183f1b494c107d0970861b9548974f34996f
+  data.tar.gz: 40c0dc5d7660db43bdf6606aae81a43c48cb5015dd29eacd63a4f2afc2e0daa2d70b34f1ce04560f190d2dd282caef3bc67d4c37236c5bcda3abfce5bed65aad

data/src/harness.rb

@@ -16,26 +16,4 @@ module Theorem
       filtered_registry(options)
     end
   end
-
-  # module retry harness
-  module RetryHarness
-    include Control::Harness
-
-    on_run do |tests|
-      arr_of_tests = tests.map { |test| { test: test, index: 0 } }
-
-      final_results = []
-      arr_of_tests.each do |test|
-        test[:index] += 1
-        results = test[:test].run!
-        if results.any?(&:failed?) && test[:index] <= 3
-          puts "Retrying iteration: #{test[:index]}\n#{results.map(&:full_name).join("\n")}"
-          redo
-        end
-        final_results.concat results
-      end
-
-      final_results
-    end
-  end
 end

data/src/publishers/stdout_reporter.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require_relative '../theorem/reporter'
+
+module Theorem
+  module StringRefinements
+    refine String do
+      # colorization
+      def colorize(color_code)
+        "\e[#{color_code}m#{self}\e[0m"
+      end
+
+      def red
+        colorize(31)
+      end
+
+      def green
+        colorize(32)
+      end
+
+      def yellow
+        colorize(33)
+      end
+
+      def blue
+        colorize(34)
+      end
+
+      def pink
+        colorize(35)
+      end
+
+      def light_blue
+        colorize(36)
+      end
+    end
+  end
+
+  # Default Stdout reporter
+  module StdoutReporter
+    extend Control::Reporter
+    using StringRefinements
+
+    subscribe :test_finished do |test|
+      print test.failed? ? 'x'.red : '.'.green
+    end
+
+    subscribe :suite_finished do |results, duration|
+      puts "\n"
+      report_summary(results)
+
+      failed_tests = results.select(&:failed?)
+
+      report_failures(failed_tests)
+
+      puts "\nTotal time: #{duration} seconds"
+      puts "Total tests: #{results.size}\n"
+    end
+
+    def inflate_percentiles(tests)
+      sorted = tests.reject { |t| t.duration.nil? }.sort_by(&:duration)
+      tests.each_with_object([]) do |test, arr|
+        _, below = sorted.partition do |duration_test|
+          if test.duration.nil?
+            true
+          else
+            test.duration >= duration_test.duration
+          end
+        end
+        hash = {}
+        hash[:percentile] = (below.size.to_f / sorted.size.to_f) * 100
+        hash[:test] = test
+        arr << hash
+      end
+    end
+
+    def report_summary(tests)
+      inflated = inflate_percentiles(tests)
+
+      top_20 = 80 / (100.0 * (tests.size + 1).to_f)
+      lowest_20 = 20 / (100.0 * (tests.size + 1).to_f)
+
+      inflated.each do |test|
+        icon = test[:test].failed? ? '❌'.red : '✓'.green
+        puts "#{icon} #{test[:test].full_name.blue} : #{duration(test)}"
+      end
+    end
+
+    def duration(test)
+      return 'Not run' if test[:test].duration.nil?
+
+      str = "#{format('%<num>0.10f', num: test[:test].duration)} seconds"
+      rank = ((test[:percentile] / 100) * (test[:test].duration + 1)) * 100
+      if rank < 5
+        str.red
+      elsif rank > 95
+        str.green
+      elsif rank < 10
+        str.yellow
+      else
+        str
+      end
+    end
+
+    def report_failures(tests)
+      tests.each do |failure|
+        puts "\n\nFailure in #{failure.full_name}\nError: #{failure.error.message.to_s.red}\nBacktrace:\n------\n#{failure.error.backtrace.map(&:red).join("\n")}"
+      end
+    end
+  end
+end

data/src/theorem/beaker.rb

@@ -9,6 +9,10 @@ module Theorem
         @ctx = ctx
       end
 
+      def name
+        @test.name
+      end
+
       def run!
         @test.run!(@ctx)
       end
@@ -54,6 +58,10 @@ module Theorem
         end
       end
 
+      def empty?
+        @state.empty?
+      end
+
       def reverse_prepare(&block)
         @state.unshift block
       end

data/src/theorem/completed_test.rb

@@ -4,16 +4,14 @@ module Theorem
   module Control
     # error class
     class CompletedTest
-      attr_reader :test, :error
+      attr_reader :test, :duration
+      attr_accessor :error, :notary
 
-      def initialize(test, error = nil, notary:)
+      def initialize(test, error = nil, notary:, duration: nil)
         @test = test
         @error = error
         @notary = notary
-      end
-
-      def notary
-        @notary
+        @duration = duration
       end
 
       def full_name

data/src/theorem/harness.rb

@@ -11,9 +11,21 @@ module Theorem
         mod.define_singleton_method :included do |inner|
           inner.define_singleton_method :run! do |options: {}|
             tests = inner.instance_exec options, &mod.test_loader
+
+            inner.suite_started_subscribers.each do |subscriber|
+              subscriber.call tests.map(&:tests).flatten.map do |test|
+                { name: test.name, metadata: test.metadata }
+              end
+            end
+
+            starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
             results = inner.instance_exec tests, options, &mod.run_loader
-            inner.completed_suite_subscribers.each do |subscriber|
-              subscriber.call(results)
+            ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+            duration = ending - starting
+
+            inner.suite_finished_subscribers.each do |subscriber|
+              subscriber.call(results, duration)
             end
             exit results.any?(&:failed?) ? 1 : 0
           end
@@ -22,19 +34,6 @@ module Theorem
 
       # harness helpers
       module ClassMethods
-        DEFAULT_LOADER = ->(options) do
-          directory = options[:directory] || '.'
-
-          ExtendedDir.require_all("./#{directory}")
-
-          registry
-        end
-
-        DEFAULT_RUNNER = ->(tests, options) do
-          tests.each_with_object([]) do |test, memo|
-            memo.concat test.run!
-          end
-        end
 
         def load_tests(&block)
           @on_load_tests = block
@@ -45,11 +44,31 @@ module Theorem
         end
 
         def run_loader
-          @on_run || DEFAULT_RUNNER
+          @on_run || default_runner
         end
 
         def test_loader
-          @on_load_tests || DEFAULT_LOADER
+          @on_load_tests || default_loader
+        end
+
+        private
+
+        def default_loader
+          lambda do |options|
+            directory = options[:directory] || '.'
+
+            ExtendedDir.require_all("./#{directory}")
+
+            registry
+          end
+        end
+
+        def default_runner
+          lambda do |tests, options|
+            tests.each_with_object([]) do |test, memo|
+              memo.concat test.run!
+            end
+          end
         end
       end
     end

data/src/theorem/hypothesis.rb

@@ -39,9 +39,9 @@ module Theorem
           mod.add_to_registry(klass)
         end
 
-        mod.const_set(:Beaker, Beaker)
-        mod.const_set(:Test, Test)
-        mod.const_set(:CompletedTest, CompletedTest)
+        mod.const_set(:Beaker, Beaker) unless mod.const_defined?(:Beaker)
+        mod.const_set(:Test, Test) unless mod.const_defined?(:Test)
+        mod.const_set(:CompletedTest, CompletedTest) unless mod.const_defined?(:CompletedTest)
         mod.extend(Registry)
       end
     end

data/src/theorem/registry.rb

@@ -28,31 +28,17 @@ module Theorem
         registry << klass
       end
 
-      def on_extra_event(&block)
-        @extra_events ||= []
-        @extra_events << block
-      end
-
-      def extra_event_subscribers
-        @extra_events || []
-      end
-
-      def on_completed_test(&block)
-        @completed_tests ||= []
-        @completed_tests << block
-      end
+      %i[suite_started test_started test_finished suite_finished].each do |method|
+        define_method method do |&block|
+          instance_variable_set("@#{method}_subscribers", []) unless instance_variable_get("@#{method}_subscribers")
+          instance_variable_get("@#{method}_subscribers").append(block)
+        end
 
-      def completed_test_subscribers
-        @completed_tests || []
-      end
+        define_method "#{method}_subscribers" do
+          return [] unless instance_variable_get("@#{method}_subscribers")
 
-      def on_completed_suite(&block)
-        @completed_suites ||= []
-        @completed_suites << block
-      end
-
-      def completed_suite_subscribers
-        @completed_suites || []
+          instance_variable_get("@#{method}_subscribers")
+        end
       end
     end
   end

data/src/theorem/test.rb

@@ -110,6 +110,9 @@ module Theorem
         before_failures = run_before_all_beakers(test_case)
 
         if before_failures.any?
+          before_failures.each do |failure|
+            publish_test_completion(failure)
+          end
           return before_failures
         end
 
@@ -118,29 +121,41 @@ module Theorem
 
         results = []
         @tests.each do |test|
+          test_start = clock_time
+
+          publish_test_start(test)
+
           error ||= run_before_each_beakers(test_case)
-          before_test_case = test_case.clone
 
+          before_test_case = test_case.clone
           error ||= run_test(test, before_test_case)
-
           error ||= run_after_each_beakers(before_test_case)
 
           notary = test_case.notary.merge(test.notary)
 
-          completed_test = CompletedTest.new(test, error, notary: notary.dump)
-          publish_test_completion(completed_test)
+          duration = clock_time - test_start
+
+          completed_test = CompletedTest.new(test, error, duration: duration, notary: notary.dump)
+
+          # publish_early if there are no after_all beakers
+          publish_test_completion(completed_test) if @after_all.empty?
+
           results << completed_test
         end
 
-        after_failures = run_after_all_beakers(duplicate_test_case)
+        after_failures = run_after_all_beakers(results, duplicate_test_case)
 
         if after_failures.any?
+          after_failures.each do |failure|
+            publish_test_completion(failure)
+          end
           return after_failures
         end
 
         results.each do |completed_test|
           # merge any after_all notations
           completed_test.notary.merge!(duplicate_test_case.notary.dump)
+          publish_test_completion(completed_test) unless @after_all.empty?
         end
 
         results
@@ -148,6 +163,10 @@ module Theorem
 
       private
 
+      def clock_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
       def run_test(test, test_case)
         if @around.empty?
           begin
@@ -163,7 +182,7 @@ module Theorem
         end
       end
 
-      def run_after_all_beakers(test_case)
+      def run_after_all_beakers(results, test_case)
         @after_all.run!(test_case)
 
         @parent_after_all&.each do |beaker|
@@ -174,9 +193,12 @@ module Theorem
       rescue Exception => error
         Theorem.handle_exception(error)
 
-        @tests.map do |test|
-          CompletedTest.new(test, error, notary: test_case.notary)
+        results.each do |test|
+          test.error = error
+          test.notary = test_case.notary
         end
+
+        results
       end
 
       def run_after_each_beakers(test_case)
@@ -219,11 +241,16 @@ module Theorem
       end
 
       def publish_test_completion(completed_test)
-        control.completed_test_subscribers.each do |subscriber|
+        control.test_finished_subscribers.each do |subscriber|
           subscriber.call(completed_test)
         end
       end
 
+      def publish_test_start(test)
+        control.test_started_subscribers.each do |subscriber|
+          subscriber.call(test)
+        end
+      end
     end
   end
 end

data/src/theorem.rb

@@ -1,8 +1,7 @@
 require_relative 'theorem/hypothesis'
 require_relative 'harness'
 require_relative 'experiment'
-require_relative 'stdout_reporter'
-
+require_relative 'publishers/stdout_reporter'
 require_relative 'theorem/harness'
 require 'json'
 
@@ -45,18 +44,6 @@ module Theorem
     mod.run!(options: options)
   end
 
-  module JsonReporter
-    extend Control::Reporter
-
-    subscribe :on_completed_suite do |results|
-      results = results.map do |result|
-        { name: result.full_name, failed: result.failed? }
-      end
-      puts results.to_json
-      puts "\n\n"
-    end
-  end
-
   module Hypothesis
     include Control::Hypothesis
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: theorem
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Sean Gregory
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-09 00:00:00.000000000 Z
+date: 2022-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extended_dir
@@ -80,6 +80,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-cobertura
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: simple and extensible test library toolkit
 email: sean.christopher.gregory@gmail.com
 executables:
@@ -90,7 +118,7 @@ files:
 - bin/theorize
 - src/experiment.rb
 - src/harness.rb
-- src/stdout_reporter.rb
+- src/publishers/stdout_reporter.rb
 - src/theorem.rb
 - src/theorem/beaker.rb
 - src/theorem/completed_test.rb

data/src/stdout_reporter.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-require_relative './theorem/reporter'
-
-module Theorem
-  module StringRefinements
-    refine String do
-      # colorization
-      def colorize(color_code)
-        "\e[#{color_code}m#{self}\e[0m"
-      end
-
-      def red
-        colorize(31)
-      end
-
-      def green
-        colorize(32)
-      end
-
-      def yellow
-        colorize(33)
-      end
-
-      def blue
-        colorize(34)
-      end
-
-      def pink
-        colorize(35)
-      end
-
-      def light_blue
-        colorize(36)
-      end
-    end
-  end
-
-  # Default Stdout reporter
-  module StdoutReporter
-    extend Control::Reporter
-    using StringRefinements
-
-    subscribe :on_completed_test do |test|
-      print test.failed? ? 'x'.red : '.'.green
-    end
-
-    subscribe :on_completed_suite do |results|
-      puts "\n"
-      report_summary(results)
-
-      failed_tests = results.select(&:failed?)
-
-      report_failures(failed_tests)
-    end
-
-    def report_summary(tests)
-      tests.each do |test|
-        icon = test.failed? ? '❌'.red : '✓'.green
-        puts "#{icon} #{test.full_name.blue}"
-      end
-    end
-
-    def report_failures(tests)
-      tests.each do |failure|
-        puts "Failure in #{failure.full_name}\nError: #{failure.error.message.to_s.red}\nBacktrace:\n------\n#{failure.error.backtrace.map(&:red).join("\n")}"
-      end
-    end
-  end
-end