theorem 1.0.0 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91647688dc2f743ee8d64461f26ee5f2265be617c82623a97e0cfd7cdec58aa8
-  data.tar.gz: 410086697fb9d6718c3f05a64cbac277b59fa4ced15cf679484b9eed11bb9edf
+  metadata.gz: '09d0acbeb340968a19cb27122ba6c852b66b2cc4347fdfca7f6ba6a9f7781ce0'
+  data.tar.gz: aef702741a59ff1a11f08e6225b0804b30b813ed0cc8eb5184cdd50c54587901
 SHA512:
-  metadata.gz: f66749de7a52cee707d0c149e7e9b273d05b46248a69e7acb4f3669d400e6a833fcac9d268eb0f87acd6f5b3cbc9d8c5bfaa836491d705c8666563ae9dfe6fe8
-  data.tar.gz: dc737567dc4303c362f5dcb3bfa6a75474981d6b355367b6d8d711c4ad5170c8f3e0c0cf252197f4007ef16f9e403234f75277559e3d9ff8ce908cc5faff94e9
+  metadata.gz: 6055b29fd80529f514b73363a18582122a5e23b312fda318b3d83a546bb7d8b062fcaf87f57858340e432bc1327a183f1b494c107d0970861b9548974f34996f
+  data.tar.gz: 40c0dc5d7660db43bdf6606aae81a43c48cb5015dd29eacd63a4f2afc2e0daa2d70b34f1ce04560f190d2dd282caef3bc67d4c37236c5bcda3abfce5bed65aad

data/bin/theorize

@@ -9,10 +9,13 @@ opts = Slop.parse do |o|
   o.string '-h', '--harness', 'harness (default: Theorem::Harness)', default: 'Theorem::Harness'
   o.array '-p', '--publisher', 'publishers (default: Theorem::StdoutReporter)', delimiter: ',', default: ['Theorem::StdoutReporter']
   o.string '-d', '--directory', 'directory to load tests (default: ".")', default: '.'
+  o.array '-i', '--include', 'tags to specify what tests should be included (default: wont filter)', default: []
+  o.array '-e', '--exclude', 'tags the specify what tests should be excluded (default: wont filter)', default: []
+  o.string '--meta', 'string of metadata to pass to the harness', default: ''
   o.on '--help' do
     puts o
     exit
   end
 end
 
-Theorem.run! opts
+Theorem.run! opts.to_h

data/src/harness.rb

@@ -7,27 +7,13 @@ module Theorem
   # default test harness
   module Harness
     include Control::Harness
-  end
-
-  # module retry harness
-  module RetryHarness
-    include Control::Harness
 
-    on_run do |tests|
-      arr_of_tests = tests.map { |test| { test: test, index: 0 } }
+    load_tests do |options|
+      directory = options[:directory] || '.'
 
-      final_results = []
-      arr_of_tests.each do |test|
-        test[:index] += 1
-        results = test[:test].run!
-        if results.any?(&:failed?) && test[:index] <= 3
-          puts "Retrying iteration: #{test[:index]}\n#{results.map(&:full_name).join("\n")}"
-          redo
-        end
-        final_results.concat results
-      end
+      ExtendedDir.require_all("./#{directory}")
 
-      final_results
+      filtered_registry(options)
     end
   end
 end

data/src/publishers/stdout_reporter.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require_relative '../theorem/reporter'
+
+module Theorem
+  module StringRefinements
+    refine String do
+      # colorization
+      def colorize(color_code)
+        "\e[#{color_code}m#{self}\e[0m"
+      end
+
+      def red
+        colorize(31)
+      end
+
+      def green
+        colorize(32)
+      end
+
+      def yellow
+        colorize(33)
+      end
+
+      def blue
+        colorize(34)
+      end
+
+      def pink
+        colorize(35)
+      end
+
+      def light_blue
+        colorize(36)
+      end
+    end
+  end
+
+  # Default Stdout reporter
+  module StdoutReporter
+    extend Control::Reporter
+    using StringRefinements
+
+    subscribe :test_finished do |test|
+      print test.failed? ? 'x'.red : '.'.green
+    end
+
+    subscribe :suite_finished do |results, duration|
+      puts "\n"
+      report_summary(results)
+
+      failed_tests = results.select(&:failed?)
+
+      report_failures(failed_tests)
+
+      puts "\nTotal time: #{duration} seconds"
+      puts "Total tests: #{results.size}\n"
+    end
+
+    def inflate_percentiles(tests)
+      sorted = tests.reject { |t| t.duration.nil? }.sort_by(&:duration)
+      tests.each_with_object([]) do |test, arr|
+        _, below = sorted.partition do |duration_test|
+          if test.duration.nil?
+            true
+          else
+            test.duration >= duration_test.duration
+          end
+        end
+        hash = {}
+        hash[:percentile] = (below.size.to_f / sorted.size.to_f) * 100
+        hash[:test] = test
+        arr << hash
+      end
+    end
+
+    def report_summary(tests)
+      inflated = inflate_percentiles(tests)
+
+      top_20 = 80 / (100.0 * (tests.size + 1).to_f)
+      lowest_20 = 20 / (100.0 * (tests.size + 1).to_f)
+
+      inflated.each do |test|
+        icon = test[:test].failed? ? '❌'.red : '✓'.green
+        puts "#{icon} #{test[:test].full_name.blue} : #{duration(test)}"
+      end
+    end
+
+    def duration(test)
+      return 'Not run' if test[:test].duration.nil?
+
+      str = "#{format('%<num>0.10f', num: test[:test].duration)} seconds"
+      rank = ((test[:percentile] / 100) * (test[:test].duration + 1)) * 100
+      if rank < 5
+        str.red
+      elsif rank > 95
+        str.green
+      elsif rank < 10
+        str.yellow
+      else
+        str
+      end
+    end
+
+    def report_failures(tests)
+      tests.each do |failure|
+        puts "\n\nFailure in #{failure.full_name}\nError: #{failure.error.message.to_s.red}\nBacktrace:\n------\n#{failure.error.backtrace.map(&:red).join("\n")}"
+      end
+    end
+  end
+end

data/src/theorem/beaker.rb

@@ -9,6 +9,10 @@ module Theorem
         @ctx = ctx
       end
 
+      def name
+        @test.name
+      end
+
       def run!
         @test.run!(@ctx)
       end
@@ -54,6 +58,10 @@ module Theorem
         end
       end
 
+      def empty?
+        @state.empty?
+      end
+
       def reverse_prepare(&block)
         @state.unshift block
       end

data/src/theorem/completed_test.rb

@@ -4,11 +4,14 @@ module Theorem
   module Control
     # error class
     class CompletedTest
-      attr_reader :test, :error
+      attr_reader :test, :duration
+      attr_accessor :error, :notary
 
-      def initialize(test, error = nil)
+      def initialize(test, error = nil, notary:, duration: nil)
         @test = test
         @error = error
+        @notary = notary
+        @duration = duration
       end
 
       def full_name

data/src/theorem/harness.rb

@@ -11,9 +11,21 @@ module Theorem
         mod.define_singleton_method :included do |inner|
           inner.define_singleton_method :run! do |options: {}|
             tests = inner.instance_exec options, &mod.test_loader
+
+            inner.suite_started_subscribers.each do |subscriber|
+              subscriber.call tests.map(&:tests).flatten.map do |test|
+                { name: test.name, metadata: test.metadata }
+              end
+            end
+
+            starting = Process.clock_gettime(Process::CLOCK_MONOTONIC)
             results = inner.instance_exec tests, options, &mod.run_loader
-            inner.completed_suite_subscribers.each do |subscriber|
-              subscriber.call(results)
+            ending = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+            duration = ending - starting
+
+            inner.suite_finished_subscribers.each do |subscriber|
+              subscriber.call(results, duration)
             end
             exit results.any?(&:failed?) ? 1 : 0
           end
@@ -22,19 +34,6 @@ module Theorem
 
       # harness helpers
       module ClassMethods
-        DEFAULT_LOADER = ->(options) do
-          directory = options[:directory] || '.'
-
-          ExtendedDir.require_all("./#{directory}")
-
-          registry
-        end
-
-        DEFAULT_RUNNER = ->(tests, options) do
-          tests.each_with_object([]) do |test, memo|
-            memo.concat test.run!
-          end
-        end
 
         def load_tests(&block)
           @on_load_tests = block
@@ -45,11 +44,31 @@ module Theorem
         end
 
         def run_loader
-          @on_run || DEFAULT_RUNNER
+          @on_run || default_runner
         end
 
         def test_loader
-          @on_load_tests || DEFAULT_LOADER
+          @on_load_tests || default_loader
+        end
+
+        private
+
+        def default_loader
+          lambda do |options|
+            directory = options[:directory] || '.'
+
+            ExtendedDir.require_all("./#{directory}")
+
+            registry
+          end
+        end
+
+        def default_runner
+          lambda do |tests, options|
+            tests.each_with_object([]) do |test, memo|
+              memo.concat test.run!
+            end
+          end
         end
       end
     end

data/src/theorem/hypothesis.rb

@@ -15,6 +15,16 @@ module Theorem
             mod
           end
 
+          klass.attr_reader :notary
+
+          klass.define_method :initialize do
+            @notary = Notation.new
+          end
+
+          klass.define_method :notate do |&block|
+            block.call(@notary)
+          end
+
           klass.extend ClassMethods
           klass.instance_eval do
             @before_all ||= Beaker.new
@@ -29,9 +39,9 @@ module Theorem
           mod.add_to_registry(klass)
         end
 
-        mod.const_set(:Beaker, Beaker)
-        mod.const_set(:Test, Test)
-        mod.const_set(:CompletedTest, CompletedTest)
+        mod.const_set(:Beaker, Beaker) unless mod.const_defined?(:Beaker)
+        mod.const_set(:Test, Test) unless mod.const_defined?(:Test)
+        mod.const_set(:CompletedTest, CompletedTest) unless mod.const_defined?(:CompletedTest)
         mod.extend(Registry)
       end
     end

data/src/theorem/notation.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Theorem
+  module Control
+    class Notation
+      def initialize(state = {})
+        @state = state
+      end
+
+      def write(key, value)
+        @state[key] = value
+      end
+
+      def read(key)
+        @state[key]
+      end
+
+      def dump
+        @state
+      end
+
+      def merge(notary)
+        Notation.new(@state.merge(notary.dump))
+      end
+
+      def edit(key, &block)
+        data = read(key)
+        block.call(data)
+        write(key, data)
+      end
+    end
+  end
+end

data/src/theorem/registry.rb

@@ -8,35 +8,37 @@ module Theorem
         @registry ||= []
       end
 
-      def add_to_registry(klass)
-        registry << klass
-      end
+      def filtered_registry(options)
+        registry.each do |test_class|
+          if options[:include]&.any?
+            test_class.tests.select! do |test|
+              test.metadata[:tags]&.intersection(options[:include])&.any?
+            end
+          end
 
-      def on_extra_event(&block)
-        @extra_events ||= []
-        @extra_events << block
-      end
+          next unless options[:exclude]&.any?
 
-      def extra_event_subscribers
-        @extra_events || []
+          test_class.tests.reject! do |test|
+            test.metadata[:tags]&.intersection(options[:include])&.any?
+          end
+        end
       end
 
-      def on_completed_test(&block)
-        @completed_tests ||= []
-        @completed_tests << block
+      def add_to_registry(klass)
+        registry << klass
       end
 
-      def completed_test_subscribers
-        @completed_tests || []
-      end
+      %i[suite_started test_started test_finished suite_finished].each do |method|
+        define_method method do |&block|
+          instance_variable_set("@#{method}_subscribers", []) unless instance_variable_get("@#{method}_subscribers")
+          instance_variable_get("@#{method}_subscribers").append(block)
+        end
 
-      def on_completed_suite(&block)
-        @completed_suites ||= []
-        @completed_suites << block
-      end
+        define_method "#{method}_subscribers" do
+          return [] unless instance_variable_get("@#{method}_subscribers")
 
-      def completed_suite_subscribers
-        @completed_suites || []
+          instance_variable_get("@#{method}_subscribers")
+        end
       end
     end
   end

data/src/theorem/test.rb

@@ -1,23 +1,30 @@
 # frozen_string_literal: true
 require_relative './beaker'
+require_relative 'notation'
 
 module Theorem
   module Control
     # test new
     class Test
-      def initialize(name, namespace, **opts, &block)
+      def initialize(name, namespace, arguments: {}, **metadata, &block)
         @name = name
         @namespace = namespace
         @block = block
-        @arguments = opts
+        @arguments = arguments
+        @metadata = metadata
+        @notary = Notation.new
       end
 
-      attr_reader :block, :name, :arguments, :namespace
+      attr_reader :block, :name, :arguments, :namespace, :metadata, :notary
 
       def full_name
         "#{namespace} #{name}"
       end
 
+      def notate(&block)
+        block.call(notary)
+      end
+
       def run!(ctx)
         ctx.instance_exec self, **arguments, &block
       end
@@ -68,7 +75,7 @@ module Theorem
         obj.include(control)
         obj.instance_eval &block if block
         obj.instance_exec self, klass, opts do |consumer, experiment_klass, params|
-          @tests.concat experiment_klass.tests(_experiment_namespace: consumer.to_s, **params)
+          @tests.concat experiment_klass.tests(_experiment_namespace: consumer.to_s, arguments: params)
         end
       end
 
@@ -92,39 +99,74 @@ module Theorem
         @after_all
       end
 
-      def test(name, &block)
-        @tests << Test.new(name, to_s, &block)
+      def test(name, **hargs, &block)
+        @tests << Test.new(name, to_s, **hargs, &block)
       end
 
       def run!
         test_case = new
 
+        # run before all beakers to create state in test case
         before_failures = run_before_all_beakers(test_case)
+
         if before_failures.any?
+          before_failures.each do |failure|
+            publish_test_completion(failure)
+          end
           return before_failures
         end
 
+        # duplicate the before_all arrangement for the after all hook
+        duplicate_test_case = test_case.clone
+
         results = []
         @tests.each do |test|
+          test_start = clock_time
+
+          publish_test_start(test)
+
           error ||= run_before_each_beakers(test_case)
-          error ||= run_test(test, test_case)
-          error ||= run_after_each_beakers(test_case)
 
-          completed_test = CompletedTest.new(test, error)
-          publish_test_completion(completed_test)
+          before_test_case = test_case.clone
+          error ||= run_test(test, before_test_case)
+          error ||= run_after_each_beakers(before_test_case)
+
+          notary = test_case.notary.merge(test.notary)
+
+          duration = clock_time - test_start
+
+          completed_test = CompletedTest.new(test, error, duration: duration, notary: notary.dump)
+
+          # publish_early if there are no after_all beakers
+          publish_test_completion(completed_test) if @after_all.empty?
+
           results << completed_test
         end
 
-        after_failures = run_after_all_beakers(test_case)
+        after_failures = run_after_all_beakers(results, duplicate_test_case)
+
         if after_failures.any?
+          after_failures.each do |failure|
+            publish_test_completion(failure)
+          end
           return after_failures
         end
 
+        results.each do |completed_test|
+          # merge any after_all notations
+          completed_test.notary.merge!(duplicate_test_case.notary.dump)
+          publish_test_completion(completed_test) unless @after_all.empty?
+        end
+
         results
       end
 
       private
 
+      def clock_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
       def run_test(test, test_case)
         if @around.empty?
           begin
@@ -140,7 +182,7 @@ module Theorem
         end
       end
 
-      def run_after_all_beakers(test_case)
+      def run_after_all_beakers(results, test_case)
         @after_all.run!(test_case)
 
         @parent_after_all&.each do |beaker|
@@ -151,9 +193,12 @@ module Theorem
       rescue Exception => error
         Theorem.handle_exception(error)
 
-        @tests.map do |test|
-          CompletedTest.new(test, error)
+        results.each do |test|
+          test.error = error
+          test.notary = test_case.notary
         end
+
+        results
       end
 
       def run_after_each_beakers(test_case)
@@ -191,16 +236,21 @@ module Theorem
         Theorem.handle_exception(error)
 
         @tests.map do |test|
-          CompletedTest.new(test, error)
+          CompletedTest.new(test, error, notary: test_case.notary)
         end
       end
 
       def publish_test_completion(completed_test)
-        control.completed_test_subscribers.each do |subscriber|
+        control.test_finished_subscribers.each do |subscriber|
           subscriber.call(completed_test)
         end
       end
 
+      def publish_test_start(test)
+        control.test_started_subscribers.each do |subscriber|
+          subscriber.call(test)
+        end
+      end
     end
   end
 end

data/src/theorem.rb

@@ -1,7 +1,8 @@
 require_relative 'theorem/hypothesis'
 require_relative 'harness'
 require_relative 'experiment'
-require_relative 'stdout_reporter'
+require_relative 'publishers/stdout_reporter'
+require_relative 'theorem/harness'
 require 'json'
 
 module Theorem
@@ -43,18 +44,6 @@ module Theorem
     mod.run!(options: options)
   end
 
-  module JsonReporter
-    extend Control::Reporter
-
-    subscribe :on_completed_suite do |results|
-      results = results.map do |result|
-        { name: result.full_name, failed: result.failed? }
-      end
-      puts results.to_json
-      puts "\n\n"
-    end
-  end
-
   module Hypothesis
     include Control::Hypothesis
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: theorem
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.1
 platform: ruby
 authors:
 - Sean Gregory
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-09 00:00:00.000000000 Z
+date: 2022-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: extended_dir
@@ -80,6 +80,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov-cobertura
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: simple and extensible test library toolkit
 email: sean.christopher.gregory@gmail.com
 executables:
@@ -90,12 +118,13 @@ files:
 - bin/theorize
 - src/experiment.rb
 - src/harness.rb
-- src/stdout_reporter.rb
+- src/publishers/stdout_reporter.rb
 - src/theorem.rb
 - src/theorem/beaker.rb
 - src/theorem/completed_test.rb
 - src/theorem/harness.rb
 - src/theorem/hypothesis.rb
+- src/theorem/notation.rb
 - src/theorem/registry.rb
 - src/theorem/reporter.rb
 - src/theorem/test.rb

data/src/stdout_reporter.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-require_relative './theorem/reporter'
-
-module Theorem
-  module StringRefinements
-    refine String do
-      # colorization
-      def colorize(color_code)
-        "\e[#{color_code}m#{self}\e[0m"
-      end
-
-      def red
-        colorize(31)
-      end
-
-      def green
-        colorize(32)
-      end
-
-      def yellow
-        colorize(33)
-      end
-
-      def blue
-        colorize(34)
-      end
-
-      def pink
-        colorize(35)
-      end
-
-      def light_blue
-        colorize(36)
-      end
-    end
-  end
-
-  # Default Stdout reporter
-  module StdoutReporter
-    extend Control::Reporter
-    using StringRefinements
-
-    subscribe :on_completed_test do |test|
-      print test.failed? ? 'x'.red : '.'.green
-    end
-
-    subscribe :on_completed_suite do |results|
-      puts "\n"
-      report_summary(results)
-
-      failed_tests = results.select(&:failed?)
-
-      report_failures(failed_tests)
-    end
-
-    def report_summary(tests)
-      tests.each do |test|
-        icon = test.failed? ? '❌'.red : '✓'.green
-        puts "#{icon} #{test.full_name.blue}"
-      end
-    end
-
-    def report_failures(tests)
-      tests.each do |failure|
-        puts "Failure in #{failure.full_name}\nError: #{failure.error.message.to_s.red}\nBacktrace:\n------\n#{failure.error.backtrace.map(&:red).join("\n")}"
-      end
-    end
-  end
-end