theorem 1.1.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 561cfd58d73936664ce10aad722bab0fc21d97a28b421a5cc0e154bbf8f14bfa
-  data.tar.gz: 9e392733f121689bb139d908730ad2e7008f263e713d5193ede30d31f2309612
+  metadata.gz: fa09d87e0b219e171e02728791f787a6ecd68a759e8a21143e85302361f68a1d
+  data.tar.gz: 739e1c4e6e19a3145d7c87c8ea688d10be70cce45735939e7f5d0bdf1817d385
 SHA512:
-  metadata.gz: 7980418d3772ed5c314893f44cec5cd3cc166d3cf4e193580a0f1c65484e7020119c32db80ebba07e2a53e880e9f3ad5d83e45d6910dd3b3a08561642071abba
-  data.tar.gz: 25e8132963cab65e4d5822e15c1aa3dc5a8f28f225330a06f9a08bcff8d64a273bcdc86ed42035429e0bd1ac8e79676e0677061d898ba7ad460c781c6e16d7a4
+  metadata.gz: 49ef938d77de1eacfd29a61c312acb396de525adb5c9b9699d3698cd31d8511c05665c560b55de61bc0fb9884024b32dbc61771cc0e44ae5267f26301cd46a18
+  data.tar.gz: 2fe7a59bec06dfd7b14c02e9245cdd0e9ca1ccf013f83254706cccd8e72a3aba9ac3048d9b0ed083b14f3d3e552c8cdd1b42ed75d24cc6ee67581740e5048f7f

data/src/harness.rb

@@ -16,26 +16,4 @@ module Theorem
       filtered_registry(options)
     end
   end
-
-  # module retry harness
-  module RetryHarness
-    include Control::Harness
-
-    on_run do |tests|
-      arr_of_tests = tests.map { |test| { test: test, index: 0 } }
-
-      final_results = []
-      arr_of_tests.each do |test|
-        test[:index] += 1
-        results = test[:test].run!
-        if results.any?(&:failed?) && test[:index] <= 3
-          puts "Retrying iteration: #{test[:index]}\n#{results.map(&:full_name).join("\n")}"
-          redo
-        end
-        final_results.concat results
-      end
-
-      final_results
-    end
-  end
 end

data/src/publishers/stdout_reporter.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require_relative '../theorem/reporter'
+
+module Theorem
+  module StringRefinements
+    refine String do
+      # colorization
+      def colorize(color_code)
+        "\e[#{color_code}m#{self}\e[0m"
+      end
+
+      def red
+        colorize(31)
+      end
+
+      def green
+        colorize(32)
+      end
+
+      def yellow
+        colorize(33)
+      end
+
+      def blue
+        colorize(34)
+      end
+
+      def pink
+        colorize(35)
+      end
+
+      def light_blue
+        colorize(36)
+      end
+    end
+  end
+
+  # Default Stdout reporter
+  module StdoutReporter
+    extend Control::Reporter
+    using StringRefinements
+
+    subscribe :test_finished do |test|
+      print test.failed? ? 'x'.red : '.'.green
+    end
+
+    subscribe :suite_finished do |results, duration|
+      puts "\n"
+      report_summary(results)
+
+      failed_tests = results.select(&:failed?)
+
+      report_failures(failed_tests)
+
+      puts "\nTotal time: #{duration} seconds"
+      puts "Total tests: #{results.size}\n"
+    end
+
+    def inflate_percentiles(tests)
+      sorted = tests.reject { |t| t.duration.nil? }.sort_by(&:duration)
+      tests.each_with_object([]) do |test, arr|
+        _, below = sorted.partition do |duration_test|
+          if test.duration.nil?
+            true
+          else
+            test.duration >= duration_test.duration
+          end
+        end
+        hash = {}
+        hash[:percentile] = (below.size.to_f / (sorted.size.to_f + 1)) * 100
+        hash[:test] = test
+        arr << hash
+      end
+    end
+
+    def report_summary(tests)
+      inflated = inflate_percentiles(tests)
+
+      top_20 = 80 / (100.0 * (tests.size + 1).to_f)
+      lowest_20 = 20 / (100.0 * (tests.size + 1).to_f)
+
+      inflated.each do |test|
+        icon = test[:test].failed? ? '❌'.red : '✓'.green
+        puts "#{icon} #{test[:test].full_name.blue} : #{duration(test)}"
+      end
+    end
+
+    def duration(test)
+      return 'Not run' if test[:test].duration.nil?
+
+      str = "#{format('%<num>0.10f', num: test[:test].duration)} seconds"
+      rank = test[:percentile]
+      if rank < 10
+        str.red
+      elsif rank > 90
+        str.green
+      elsif rank < 30
+        str.yellow
+      else
+        str
+      end
+    end
+
+    def report_failures(tests)
+      tests.each do |failure|
+        puts "\n\nFailure in #{failure.full_name}\nError: #{failure.error.message.to_s.red}\nBacktrace:\n------\n#{failure.error.backtrace.map(&:red).join("\n")}"
+      end
+    end
+  end
+end

data/src/theorem/beaker.rb

@@ -9,6 +9,10 @@ module Theorem
         @ctx = ctx
       end
 
+      def name
+        @test.name
+      end
+
       def run!
         @test.run!(@ctx)
       end
@@ -54,6 +58,10 @@ module Theorem
         end
       end
 
+      def empty?
+        @state.empty?
+      end
+
       def reverse_prepare(&block)
         @state.unshift block
       end

data/src/theorem/completed_test.rb

@@ -4,11 +4,14 @@ module Theorem
   module Control
     # error class
     class CompletedTest
-      attr_reader :test, :error
+      attr_reader :test, :duration
+      attr_accessor :error, :notary
 
-      def initialize(test, error = nil)
+      def initialize(test, error = nil, notary:, duration: nil)
         @test = test
         @error = error
+        @notary = notary
+        @duration = duration
       end
 
       def full_name

data/src/theorem/harness.rb

@@ -11,9 +11,21 @@ module Theorem
         mod.define_singleton_method :included do |inner|
           inner.define_singleton_method :run! do |options: {}|
             tests = inner.instance_exec options, &mod.test_loader
+
+            inner.suite_started_subscribers.each do |subscriber|
+              subscriber.call tests.map(&:tests).flatten.map do |test|
+                { name: test.name, metadata: test.metadata }
+              end
+            end
+
+            starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
             results = inner.instance_exec tests, options, &mod.run_loader
-            inner.completed_suite_subscribers.each do |subscriber|
-              subscriber.call(results)
+            ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+            duration = ending - starting
+
+            inner.suite_finished_subscribers.each do |subscriber|
+              subscriber.call(results, duration)
             end
             exit results.any?(&:failed?) ? 1 : 0
           end
@@ -22,19 +34,6 @@ module Theorem
 
       # harness helpers
       module ClassMethods
-        DEFAULT_LOADER = ->(options) do
-          directory = options[:directory] || '.'
-
-          ExtendedDir.require_all("./#{directory}")
-
-          registry
-        end
-
-        DEFAULT_RUNNER = ->(tests, options) do
-          tests.each_with_object([]) do |test, memo|
-            memo.concat test.run!
-          end
-        end
 
         def load_tests(&block)
           @on_load_tests = block
@@ -45,11 +44,31 @@ module Theorem
         end
 
         def run_loader
-          @on_run || DEFAULT_RUNNER
+          @on_run || default_runner
         end
 
         def test_loader
-          @on_load_tests || DEFAULT_LOADER
+          @on_load_tests || default_loader
+        end
+
+        private
+
+        def default_loader
+          lambda do |options|
+            directory = options[:directory] || '.'
+
+            ExtendedDir.require_all("./#{directory}")
+
+            registry
+          end
+        end
+
+        def default_runner
+          lambda do |tests, options|
+            tests.each_with_object([]) do |test, memo|
+              memo.concat test.run!
+            end
+          end
         end
       end
     end

data/src/theorem/hypothesis.rb

@@ -15,6 +15,16 @@ module Theorem
             mod
           end
 
+          klass.attr_reader :notary
+
+          klass.define_method :initialize do
+            @notary = Notation.new
+          end
+
+          klass.define_method :notate do |&block|
+            block.call(@notary)
+          end
+
           klass.extend ClassMethods
           klass.instance_eval do
             @before_all ||= Beaker.new
@@ -29,9 +39,9 @@ module Theorem
           mod.add_to_registry(klass)
         end
 
-        mod.const_set(:Beaker, Beaker)
-        mod.const_set(:Test, Test)
-        mod.const_set(:CompletedTest, CompletedTest)
+        mod.const_set(:Beaker, Beaker) unless mod.const_defined?(:Beaker)
+        mod.const_set(:Test, Test) unless mod.const_defined?(:Test)
+        mod.const_set(:CompletedTest, CompletedTest) unless mod.const_defined?(:CompletedTest)
         mod.extend(Registry)
       end
     end

data/src/theorem/notation.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Theorem
+  module Control
+    class Notation
+      def initialize(state = {})
+        @state = state
+      end
+
+      def write(key, value)
+        @state[key] = value
+      end
+
+      def read(key)
+        @state[key]
+      end
+
+      def dump
+        @state
+      end
+
+      def merge(notary)
+        Notation.new(@state.merge(notary.dump))
+      end
+
+      def edit(key, &block)
+        data = read(key)
+        block.call(data)
+        write(key, data)
+      end
+    end
+  end
+end

data/src/theorem/registry.rb

@@ -28,31 +28,17 @@ module Theorem
         registry << klass
       end
 
-      def on_extra_event(&block)
-        @extra_events ||= []
-        @extra_events << block
-      end
-
-      def extra_event_subscribers
-        @extra_events || []
-      end
-
-      def on_completed_test(&block)
-        @completed_tests ||= []
-        @completed_tests << block
-      end
+      %i[suite_started test_started test_finished suite_finished].each do |method|
+        define_method method do |&block|
+          instance_variable_set("@#{method}_subscribers", []) unless instance_variable_get("@#{method}_subscribers")
+          instance_variable_get("@#{method}_subscribers").append(block)
+        end
 
-      def completed_test_subscribers
-        @completed_tests || []
-      end
+        define_method "#{method}_subscribers" do
+          return [] unless instance_variable_get("@#{method}_subscribers")
 
-      def on_completed_suite(&block)
-        @completed_suites ||= []
-        @completed_suites << block
-      end
-
-      def completed_suite_subscribers
-        @completed_suites || []
+          instance_variable_get("@#{method}_subscribers")
+        end
       end
     end
   end

data/src/theorem/test.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 require_relative './beaker'
+require_relative 'notation'
 
 module Theorem
   module Control
@@ -11,14 +12,19 @@ module Theorem
         @block = block
         @arguments = arguments
         @metadata = metadata
+        @notary = Notation.new
       end
 
-      attr_reader :block, :name, :arguments, :namespace, :metadata
+      attr_reader :block, :name, :arguments, :namespace, :metadata, :notary
 
       def full_name
         "#{namespace} #{name}"
       end
 
+      def notate(&block)
+        block.call(notary)
+      end
+
       def run!(ctx)
         ctx.instance_exec self, **arguments, &block
       end
@@ -100,32 +106,67 @@ module Theorem
       def run!
         test_case = new
 
+        # run before all beakers to create state in test case
         before_failures = run_before_all_beakers(test_case)
+
         if before_failures.any?
+          before_failures.each do |failure|
+            publish_test_completion(failure)
+          end
           return before_failures
         end
 
+        # duplicate the before_all arrangement for the after all hook
+        duplicate_test_case = test_case.clone
+
         results = []
         @tests.each do |test|
+          test_start = clock_time
+
+          publish_test_start(test)
+
           error ||= run_before_each_beakers(test_case)
-          error ||= run_test(test, test_case)
-          error ||= run_after_each_beakers(test_case)
 
-          completed_test = CompletedTest.new(test, error)
-          publish_test_completion(completed_test)
+          before_test_case = test_case.clone
+          error ||= run_test(test, before_test_case)
+          error ||= run_after_each_beakers(before_test_case)
+
+          notary = test_case.notary.merge(test.notary)
+
+          duration = clock_time - test_start
+
+          completed_test = CompletedTest.new(test, error, duration: duration, notary: notary.dump)
+
+          # publish_early if there are no after_all beakers
+          publish_test_completion(completed_test) if @after_all.empty?
+
           results << completed_test
         end
 
-        after_failures = run_after_all_beakers(test_case)
+        after_failures = run_after_all_beakers(results, duplicate_test_case)
+
         if after_failures.any?
+          after_failures.each do |failure|
+            publish_test_completion(failure)
+          end
           return after_failures
         end
 
+        results.each do |completed_test|
+          # merge any after_all notations
+          completed_test.notary.merge!(duplicate_test_case.notary.dump)
+          publish_test_completion(completed_test) unless @after_all.empty?
+        end
+
         results
       end
 
       private
 
+      def clock_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
       def run_test(test, test_case)
         if @around.empty?
           begin
@@ -141,7 +182,7 @@ module Theorem
         end
       end
 
-      def run_after_all_beakers(test_case)
+      def run_after_all_beakers(results, test_case)
         @after_all.run!(test_case)
 
         @parent_after_all&.each do |beaker|
@@ -152,9 +193,12 @@ module Theorem
       rescue Exception => error
         Theorem.handle_exception(error)
 
-        @tests.map do |test|
-          CompletedTest.new(test, error)
+        results.each do |test|
+          test.error = error
+          test.notary = test_case.notary
         end
+
+        results
       end
 
       def run_after_each_beakers(test_case)
@@ -192,16 +236,21 @@ module Theorem
         Theorem.handle_exception(error)
 
         @tests.map do |test|
-          CompletedTest.new(test, error)
+          CompletedTest.new(test, error, notary: test_case.notary)
         end
       end
 
       def publish_test_completion(completed_test)
-        control.completed_test_subscribers.each do |subscriber|
+        control.test_finished_subscribers.each do |subscriber|
           subscriber.call(completed_test)
         end
       end
 
+      def publish_test_start(test)
+        control.test_started_subscribers.each do |subscriber|
+          subscriber.call(test)
+        end
+      end
     end
   end
 end

data/src/theorem.rb

@@ -1,8 +1,7 @@
 require_relative 'theorem/hypothesis'
 require_relative 'harness'
 require_relative 'experiment'
-require_relative 'stdout_reporter'
-
+require_relative 'publishers/stdout_reporter'
 require_relative 'theorem/harness'
 require 'json'
 
@@ -45,18 +44,6 @@ module Theorem
     mod.run!(options: options)
   end
 
-  module JsonReporter
-    extend Control::Reporter
-
-    subscribe :on_completed_suite do |results|
-      results = results.map do |result|
-        { name: result.full_name, failed: result.failed? }
-      end
-      puts results.to_json
-      puts "\n\n"
-    end
-  end
-
   module Hypothesis
     include Control::Hypothesis
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: theorem
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.2
 platform: ruby
 authors:
 - Sean Gregory
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-09 00:00:00.000000000 Z
+date: 2022-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extended_dir
@@ -80,6 +80,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-cobertura
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: simple and extensible test library toolkit
 email: sean.christopher.gregory@gmail.com
 executables:
@@ -90,12 +118,13 @@ files:
 - bin/theorize
 - src/experiment.rb
 - src/harness.rb
-- src/stdout_reporter.rb
+- src/publishers/stdout_reporter.rb
 - src/theorem.rb
 - src/theorem/beaker.rb
 - src/theorem/completed_test.rb
 - src/theorem/harness.rb
 - src/theorem/hypothesis.rb
+- src/theorem/notation.rb
 - src/theorem/registry.rb
 - src/theorem/reporter.rb
 - src/theorem/test.rb

data/src/stdout_reporter.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-require_relative './theorem/reporter'
-
-module Theorem
-  module StringRefinements
-    refine String do
-      # colorization
-      def colorize(color_code)
-        "\e[#{color_code}m#{self}\e[0m"
-      end
-
-      def red
-        colorize(31)
-      end
-
-      def green
-        colorize(32)
-      end
-
-      def yellow
-        colorize(33)
-      end
-
-      def blue
-        colorize(34)
-      end
-
-      def pink
-        colorize(35)
-      end
-
-      def light_blue
-        colorize(36)
-      end
-    end
-  end
-
-  # Default Stdout reporter
-  module StdoutReporter
-    extend Control::Reporter
-    using StringRefinements
-
-    subscribe :on_completed_test do |test|
-      print test.failed? ? 'x'.red : '.'.green
-    end
-
-    subscribe :on_completed_suite do |results|
-      puts "\n"
-      report_summary(results)
-
-      failed_tests = results.select(&:failed?)
-
-      report_failures(failed_tests)
-    end
-
-    def report_summary(tests)
-      tests.each do |test|
-        icon = test.failed? ? '❌'.red : '✓'.green
-        puts "#{icon} #{test.full_name.blue}"
-      end
-    end
-
-    def report_failures(tests)
-      tests.each do |failure|
-        puts "Failure in #{failure.full_name}\nError: #{failure.error.message.to_s.red}\nBacktrace:\n------\n#{failure.error.backtrace.map(&:red).join("\n")}"
-      end
-    end
-  end
-end