flores 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 93d262135443a719e7d26ac03f57cc016d63a216
-  data.tar.gz: d92cf5cb04a2553c1369948f2cd13d452bee6bff
+  metadata.gz: c0fb409bcd13936e24dae9b9f932577b83321cfa
+  data.tar.gz: 0168c710951ccd26e4bff91434d4e811526cabe3
 SHA512:
-  metadata.gz: 72e01594ea803e4fc901cb5e9e825524045cc9812cf9b7f2d1430363f8733907b1a5949677e902159198056af873c2ec677f392c41dd56f9cef01c3a4126ad0a
-  data.tar.gz: 0e9b10da498eee2d17b2f3e7557f61e151c1adb2fa08a14ed7e4094cd7a38c4b15b83ee8516cb59d1734b2a145c0c52edad65d93ca0e34f407dc94ce2c71c8ed
+  metadata.gz: 9da3201822ace87268b2cd900d75c95ee7b9d02ac3b7801b5456039d3fc55e8a35739e8f88ff40524f4db7ba3c6bf6036967d652d2d94f19181eb4e67fdb3891
+  data.tar.gz: 1d79f887fbbeeb0cd4f641b31382b074fc20a3e7e2b6a3da55bfde4d35dbb491ef51463869981e9e53c862b506e4785182359cb2c084d64ca6d7ecedde0c603a

data/Gemfile

@@ -2,4 +2,6 @@ source "https://rubygems.org"
 
 group "development" do
   gem "rspec", ">= 3.0.0"
+  gem "fuubar"
+  gem "pry"
 end

data/Gemfile.lock

@@ -1,7 +1,22 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    coderay (1.1.0)
     diff-lcs (1.2.5)
+    ffi (1.9.6-java)
+    fuubar (2.0.0)
+      rspec (~> 3.0)
+      ruby-progressbar (~> 1.4)
+    method_source (0.8.2)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry (0.10.1-java)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+      spoon (~> 0.0)
     rspec (3.2.0)
       rspec-core (~> 3.2.0)
       rspec-expectations (~> 3.2.0)
@@ -15,9 +30,16 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.2.0)
     rspec-support (3.2.1)
+    ruby-progressbar (1.7.1)
+    slop (3.6.0)
+    spoon (0.0.4)
+      ffi
 
 PLATFORMS
+  java
   ruby
 
 DEPENDENCIES
+  fuubar
+  pry
   rspec (>= 3.0.0)

data/Makefile

@@ -29,3 +29,15 @@ install: $(GEM)
 .PHONY: clean
 clean:
 	-rm -rf .yardoc $(GEM) $(NAME)-$(VERSION)/
+
+.PHONY: rubocop
+rubocop:
+	rubocop -D
+
+.PHONY: test
+test: rubocop
+	rspec -f Flores::RSpec::Formatters::Analyze
+
+.PHONY: test
+test-fast: rubocop
+	ITERATIONS=10 rspec -f Flores::RSpec::Formatters::Analyze

data/README.md

@@ -0,0 +1,177 @@
+# Flores - a stress testing library
+
+When writing tests, it is often good to test a wide variety of inputs to ensure
+your entire input range behaves correctly.
+
+Further, adding a bit of randomness in your tests can help find bugs.
+
+## Why Flores?
+
+Randomization helps you cover a wider range of inputs to your tests to find bugs. Stress
+testing (run a test repeatedly) helps you find bugs faster. We can use stress testing results
+to find common patterns in failures!
+
+Let's look at a sample situation. Ruby's TCPServer. Let's write a spec to cover a spec covering port binding:
+
+```ruby
+describe TCPServer do
+  subject(:socket) { Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0) }
+  let(:port) { 5000 }
+  let(:sockaddr) { Socket.sockaddr_in(port, "127.0.0.1") }
+
+  after { socket.close}
+
+  it "should bind successfully" do
+    socket.bind(sockaddr)
+    expect(socket.local_address.ip_port).to(be == port)
+  end
+end
+```
+
+Running it:
+
+```
+% rspec tcpserver_spec.rb
+.
+
+Finished in 0.00248 seconds (files took 0.16294 seconds to load)
+1 example, 0 failures
+```
+
+That's cool. We now have some confidence that TCPServer on port 5000 will bind successfully.
+
+What about the other ports? What ranges of values should work? What shouldn't?
+
+Let's assume I don't know anything about tcp port ranges and test randomly in the range -100,000 to +100,000:
+
+```ruby
+describe TCPServer do
+  let(:port) { Randomized.integer(-100_000..100_000) }
+  ...
+end
+```
+
+Running it:
+
+```
+% rspec tcpserver_spec.rb
+F
+
+Failures:
+
+  1) TCPServer should bind successfully
+     Failure/Error: expect(socket.local_address.ip_port).to(be == port)
+       expected: == 83359
+            got:    17823
+     # ./tcpserver_spec.rb:12:in `block (2 levels) in <top (required)>'
+
+Finished in 0.00155 seconds (files took 0.10221 seconds to load)
+1 example, 1 failure
+```
+
+Well that's weird. Binding port 83359 actually made it bind on port 17823!
+
+If we run it more times, we'll see all kinds of different results:
+
+* Run 1:
+  ```
+     Failure/Error: expect(socket.local_address.ip_port).to(be == port)
+       expected: == 83359
+            got:    17823
+  ```
+* Run 2:
+  ```
+     Failure/Error: let(:sockaddr) { Socket.sockaddr_in(port, "127.0.0.1") }
+     SocketError:
+       getaddrinfo: nodename nor servname provided, or not known
+  ```
+* Run 3:
+  ```
+     Errno::EACCES:
+       Permission denied - bind(2) for 127.0.0.1:615
+  ```
+* Run 4:
+  ```
+     Finished in 0.00161 seconds (files took 0.10356 seconds to load)
+     1 example, 0 failures
+  ```
+
+## Analyze the results
+
+The above example showed that there were many different kinds of failures when
+we introduced randomness to our test inputs.
+
+We can go further and run a given spec example many times and group the
+failures by similarity and include context (what the inputs were, etc)
+
+This library provides an `analyze_it` helper which behaves similarly to rspec's
+`it` except that it runs the block a random number of times and clears the `let` cache
+each time. This lets you run  a given test many times with many random inputs!
+
+The result is grouped by failure and includes context. Let's see how it works:
+
+We'll change `it` to use `analyze_it` instead:
+
+```diff
+- it "should bind successfully" do
++ analyze_it "should bind successfully", [:port] do
+```
+
+Now rerunning the test. With barely any spec changes from the original, we have
+now enough randomness and stress testing to identify many different failure cases
+and input ranges for those failures.
+
+```
+Failures:
+
+  1) TCPServer should bind successfully
+     Failure/Error: raise StandardError, Analysis.new(results) if results.any? { |k, _| k != :success }
+     StandardError:
+       31.14% tests successful of 2563 tests
+       Failure analysis:
+         50.57% -> [1296] SocketError
+           Sample exception for {:port=>-94900}
+             getaddrinfo: nodename nor servname provided, or not known
+           Samples causing SocketError:
+             {:port=>-49441}
+             {:port=>-1991}
+             {:port=>-54074}
+             {:port=>-1733}
+             {:port=>-21868}
+         16.89% -> [433] RSpec::Expectations::ExpectationNotMetError
+           Sample exception for {:port=>93844}
+             expected: == 93844
+                  got:    28308
+           Samples causing RSpec::Expectations::ExpectationNotMetError:
+             {:port=>89451}
+             {:port=>95627}
+             {:port=>95225}
+             {:port=>73106}
+             {:port=>77167}
+         1.01% -> [26] Errno::EACCES
+           Sample exception for {:port=>65649}
+             Permission denied - bind(2) for 127.0.0.1:113
+           Samples causing Errno::EACCES:
+             {:port=>913}
+             {:port=>141}
+             {:port=>66194}
+             {:port=>66217}
+             {:port=>66408}
+         0.39% -> [10] Errno::EADDRINUSE
+           Sample exception for {:port=>34402}
+             Address already in use - bind(2) for 127.0.0.1:34402
+           Samples causing Errno::EADDRINUSE:
+             {:port=>50905}
+             {:port=>71202}
+             {:port=>34402}
+             {:port=>28235}
+             {:port=>85641}
+     # ./lib/rspec/stress_it.rb:103:in `block in analyze_it'
+
+Finished in 0.0735 seconds (files took 0.10247 seconds to load)
+1 example, 1 failure
+
+Failed examples:
+
+rspec ./tcpserver_spec.rb:8 # TCPServer should bind successfully
+```

data/examples/analyze_number.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 # This file is part of ruby-flores.
 # Copyright (C) 2015 Jordan Sissel
 # 
@@ -13,18 +14,20 @@
 # 
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# encoding: utf-8
-require "randomized"
-require "rspec/stress_it"
+require "flores/rspec"
+require "flores/random"
 
 RSpec.configure do |c|
-  c.extend RSpec::StressIt
+  Flores::RSpec.configure(c)
+  c.add_formatter("Flores::RSpec::Formatters::Analyze")
 end
 
-describe "number" do
-  let(:number) { Randomized.number(0..200) }
-  analyze_it "should be less than 100", [:number] do
-    expect(number).to(be < 100)
+describe "a random number" do
+  context "between 0 and 200 inclusive" do
+    let(:number) { Flores::Random.number(0..200) }
+    analyze_results
+    stress_it "should be less than 100" do
+      expect(number).to(be < 100)
+    end
   end
 end

data/examples/socket_acceptance_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 # This file is part of ruby-flores.
 # Copyright (C) 2015 Jordan Sissel
 # 
@@ -13,14 +14,12 @@
 # 
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# encoding: utf-8
-require "randomized"
+require "flores/random"
 require "socket"
-require "rspec/stress_it"
+require "flores/rspec"
 
 RSpec.configure do |c|
-  c.extend RSpec::StressIt
+  Flores::RSpec.configure(c)
 end
 
 # A factory for encapsulating behavior of a tcp server and client for the
@@ -62,12 +61,12 @@ class TCPIntegrationTestFactory
 end
 
 describe "TCPServer+TCPSocket" do
-  let(:port) { Randomized.integer(1024..65535) }
-  let(:text) { Randomized.text(1..2000) }
+  let(:port) { Flores::Random.integer(1024..65535) }
+  let(:text) { Flores::Random.text(1..2000) }
   subject { TCPIntegrationTestFactory.new(port) }
   
   describe "using stress_it" do
-    analyze_it "should send data correctly", [:port, :text] do
+    stress_it2 "should send data correctly", [:port, :text] do
       begin
         subject.setup
       rescue Errno::EADDRINUSE

data/examples/socket_analyze_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 # This file is part of ruby-flores.
 # Copyright (C) 2015 Jordan Sissel
 # 
@@ -13,26 +14,27 @@
 # 
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# encoding: utf-8
-require "randomized"
+require "flores/random"
 require "socket"
-require "rspec/stress_it"
+require "flores/rspec"
 
 RSpec.configure do |c|
-  c.extend RSpec::StressIt
+  Flores::RSpec.configure(c)
 end
 
 describe TCPServer do
   subject(:socket) { Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0) }
   let(:sockaddr) { Socket.sockaddr_in(port, "127.0.0.1") }
-  after { socket.close }
 
   context "on a random port" do
-    let(:port) { Randomized.integer(-100_000..100_000) }
-    analyze_it "should bind successfully", [:port] do
-      socket.bind(sockaddr)
-      expect(socket.local_address.ip_port).to(be == port)
+    let(:port) { Flores::Random.integer(-100_000..100_000) }
+    stress_it2 "should bind successfully", [:port] do
+      begin
+        socket.bind(sockaddr)
+        expect(socket.local_address.ip_port).to(be == port)
+      ensure
+        socket.close
+      end
     end
   end
 end

data/examples/socket_stress_spec.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 # This file is part of ruby-flores.
 # Copyright (C) 2015 Jordan Sissel
 # 
@@ -13,17 +14,17 @@
 # 
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# encoding: utf-8
-require "randomized"
+require "flores/random"
 require "socket"
-require "rspec/stress_it"
+require "flores/rspec"
 
 RSpec.configure do |c|
-  c.extend RSpec::StressIt
+  Flores::RSpec.configure(c)
 end
 
 describe TCPServer do
+  analyze
+
   subject(:socket) { Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0) }
   let(:sockaddr) { Socket.sockaddr_in(port, "127.0.0.1") }
   let(:ignore_eaddrinuse) do
@@ -41,14 +42,14 @@ describe TCPServer do
   end
 
   context "on privileged ports" do
-    let(:port) { Randomized.integer(1..1023) }
+    let(:port) { Flores::Random.integer(1..1023) }
     stress_it "should raise Errno::EACCESS" do
       expect { socket.bind(sockaddr) }.to(raise_error(Errno::EACCES))
     end
   end
 
   context "on unprivileged ports" do
-    let(:port) { Randomized.integer(1025..65535) }
+    let(:port) { Flores::Random.integer(1025..65535) }
     stress_it "should bind on a port" do
       # EADDRINUSE is expected since we are picking ports at random
       # Let's ignore this specific exception

data/flores.gemspec

@@ -2,7 +2,7 @@ Gem::Specification.new do |spec|
   files = %x(git ls-files).split("\n")
 
   spec.name = "flores"
-  spec.version = "0.0.2"
+  spec.version = "0.0.3"
   spec.summary = "Fuzz, randomize, and stress your tests"
   spec.description = <<-DESCRIPTION
     Add fuzzing, randomization, and stress to your tests.

data/lib/flores/namespace.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+# This file is part of ruby-flores.
+# Copyright (C) 2015 Jordan Sissel
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# :nodoc:
+module Flores # rubocop:disable Style/ClassAndModuleChildren
+  module RSpec # rubocop:disable Style/ClassAndModuleChildren
+    module Formatters # rubocop:disable Style/ClassAndModuleChildren
+    end
+  end
+end

data/lib/{randomized.rb → flores/random.rb}

@@ -1,3 +1,4 @@
+# encoding: utf-8
 # This file is part of ruby-flores.
 # Copyright (C) 2015 Jordan Sissel
 # 
@@ -13,11 +14,11 @@
 # 
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# encoding: utf-8
+
+require "flores/namespace"
 
 # A collection of methods intended for use in randomized testing.
-module Randomized
+module Flores::Random
   # A selection of UTF-8 characters
   #
   # I'd love to generate this, but I don't yet know enough about how unicode
@@ -43,7 +44,7 @@ module Randomized
   # * Negative lengths are not permitted and will raise an ArgumentError
   #
   # @param length [Fixnum or Range] the length of text to generate
-  # @return [String] the 
+  # @return [String] the generated text
   def self.text(length)
     return text_range(length) if length.is_a?(Range)
 
@@ -51,6 +52,10 @@ module Randomized
     length.times.collect { character }.join
   end # def text
 
+  # Generate text with random characters of a length within the given range.
+  #
+  # @param range [Range] the range of length to generate, inclusive
+  # @return [String] the generated text
   def self.text_range(range)
     raise ArgumentError, "Requires ascending range, you gave #{range}." if range.end < range.begin
     raise ArgumentError, "A negative range values are not permitted, I received range #{range}" if range.begin < 0
@@ -77,9 +82,9 @@ module Randomized
   # @param range [Range]
   def self.number(range)
     raise ArgumentError, "Range not given, got #{range.class}: #{range.inspect}" if !range.is_a?(Range)
-    # Range#size returns the number of elements in the range, not the length of the range.
-    # This makes #size return (abs(range.begin - range.end) + 1), and we want the length, so subtract 1.
-    rand * (range.size - 1) + range.begin
+    # Ruby 1.9.3 and below do not have Enumerable#size, so we have to compute the size of the range
+    # ourselves.
+    rand * (range.end - range.begin) + range.begin
   end # def number
    
   # Run a block a random number of times.

data/lib/{rspec/stress_it.rb → flores/rspec/analyze.rb}

@@ -1,3 +1,4 @@
+# encoding: utf-8
 # This file is part of ruby-flores.
 # Copyright (C) 2015 Jordan Sissel
 # 
@@ -13,9 +14,8 @@
 # 
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# encoding: utf-8
-require "rspec/core"
+require "flores/namespace"
+require "flores/rspec"
 
 # RSpec helpers for stress testing examples
 #
@@ -26,83 +26,53 @@ require "rspec/core"
 #     end
 #
 # TODO(sissel): Show an example of stress_it and analyze_it
-module RSpec::StressIt
-  DEFAULT_ITERATIONS = 1..5000
-
-  # Wraps `it` and runs the block many times. Each run has will clear the `let` cache.
+module Flores::RSpec::Analyze
+  # Save state after each example so it can be used in analysis after specs are completed.
   #
-  # The intent of this is to allow randomized testing for fuzzing and stress testing
-  # of APIs to help find edge cases and weird behavior.
+  # If you use this, you'll want to set your RSpec formatter to
+  # Flores::RSpec::Formatter::Analyze
   #
-  # The default number of iterations is randomly selected between 1 and 1000 inclusive
-  def stress_it(name, options = {}, &block)
-    stress__iterations = Randomized.iterations(options.delete(:stress_iterations) || DEFAULT_ITERATIONS)
-    it(name, options) do
-      # Run the block of an example many times
-      stress__iterations.each do
-        # Run the block within 'it' scope
-        instance_eval(&block)
-
-        # clear the internal rspec `let` cache this lets us run a test
-        # repeatedly with fresh `let` evaluations.
-        # Reference: https://github.com/rspec/rspec-core/blob/5fc29a15b9af9dc1c9815e278caca869c4769767/lib/rspec/core/memoized_helpers.rb#L124-L127
-        __memoized.clear
-      end
-    end # it ...
-  end # def stress_it
-
-  # Generate a random number of copies of a given example.
-  # The idea is to take 1 `it` and run it N times to help tease out failures.
-  # Of course, the teasing requires you have randomized `let` usage, for example:
+  # Let's show an example that fails sometimes.
   #
-  #     let(:number) { Randomized.number(0..200) }
-  #     it "should be less than 100" do
-  #       expect(number).to(be < 100)
+  #     describe "Addition of two numbers" do
+  #       context "positive numbers" do
+  #         analyze_results
+  #         let(:a) { Flores::Random.number(1..1000) }
+  #     
+  #         # Here we make negative numbers possible to cause failure in our test.
+  #         let(:b) { Flores::Random.number(-200..1000) }
+  #         subject { a + b }
+  #     
+  #         stress_it "should be positive" do
+  #           expect(subject).to(be > 0)
+  #         end
+  #       end
   #     end
-  def stress_it2(name, options = {}, &block)
-    stress__iterations = Randomized.iterations(options.delete(:stress_iterations) || DEFAULT_ITERATIONS)
-    stress__iterations.each do |i|
-      it(name + " [#{i}]", *args) do
-        instance_eval(&block)
-      end # it ...
-    end # .times
-  end 
-
-  # Perform analysis on failure scenarios of a given example
-  #
-  # This will run the given example a random number of times and aggregate the
-  # results. If any failures occur, the spec will fail and a report will be
-  # given on that test.
   #
-  # Example spec:
-  #
-  #     let(:number) { Randomized.number(0..200) }
-  #     fuzz "should be less than 100" do
-  #       expect(number).to(be < 100)
-  #     end
+  # And running it:
   #
-  # Example report:
-  def analyze_it(name, variables, &block) # rubocop:disable Metrics/AbcSize
-    it(name) do
-      results = Hash.new { |h, k| h[k] = [] }
-      Randomized.iterations(DEFAULT_ITERATIONS).each do
-        state = Hash[variables.collect { |l| [l, __send__(l)] }]
-        begin
-          instance_eval(&block)
-          results[:success] << [state, nil]
-        rescue => e
-          results[e.class] << [state, e]
-        rescue Exception => e # rubocop:disable Lint/RescueException
-          results[e.class] << [state, e]
-        end
-
-        # Clear `let` memoizations
-        __memoized.clear
-      end
-
-      raise StandardError, Analysis.new(results) if results.any? { |k, _| k != :success }
+  #     % rspec -f Flores::RSpec::Formatter::Analyze
+  #     Addition of two numbers positive numbers should be positive
+  #       98.20% tests successful of 3675 tests
+  #       Failure analysis:
+  #         1.80% -> [66] RSpec::Expectations::ExpectationNotMetError
+  #           Sample exception for {:a=>126.21705882478048, :b=>-139.54814492675024, :subject=>-13.33108610196976}
+  #             expected: > 0
+  #                  got:   -13.33108610196976
+  #           Samples causing RSpec::Expectations::ExpectationNotMetError:
+  #             {:a=>90.67298249206425, :b=>-136.6237821353908, :subject=>-45.95079964332655}
+  #             {:a=>20.35865155878871, :b=>-39.592417377658876, :subject=>-19.233765818870165}
+  #             {:a=>158.07905166101787, :b=>-177.5864470909581, :subject=>-19.50739542994023}
+  #             {:a=>31.80445518715138, :b=>-188.51942190504894, :subject=>-156.71496671789757}
+  #             {:a=>116.1479954937354, :b=>-146.18477887927958, :subject=>-30.036783385544183}
+  def analyze_results
+    # TODO(sissel): Would be lovely to figure out how to inject an 'after' for
+    # all examples if we are using the Analyze formatter.
+    # Then this method could be implied by using the right formatter, or something.
+    after do |example|
+      example.metadata[:values] = __memoized.clone
     end
-  end # def analyze_it
+  end
 
   # A formatter to show analysis of an `analyze_it` example. 
   class Analysis < StandardError
@@ -170,4 +140,4 @@ module RSpec::StressIt
       ]
     end # def error_sample_states
   end # class Analysis
-end # module RSpec::StressIt
+end # Flores::RSpec::Analyze

data/lib/flores/rspec/formatters/analyze.rb

@@ -0,0 +1,61 @@
+# encoding: utf-8
+# This file is part of ruby-flores.
+# Copyright (C) 2015 Jordan Sissel
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+require "flores/namespace"
+require "rspec/core/formatters/base_text_formatter"
+
+Flores::RSpec::Formatters::Analyze = Class.new(RSpec::Core::Formatters::BaseTextFormatter) do
+  RSpec::Core::Formatters.register self, :dump_failures, :dump_summary
+
+  def dump_summary(event)
+    # The event is an RSpec::Core::Notifications::SummaryNotification
+    # Let's mimic the BaseTextFormatter but without the failing test report
+    output.puts 
+    output.puts "Finished in #{event.formatted_duration}"
+    output.puts "#{event.colorized_totals_line}"
+  end
+
+  def dump_failures(event)
+    output.puts
+    group = event.examples.each_with_object(Hash.new { |h, k| h[k] = [] }) do |e, m| 
+      m[e.metadata[:full_description]] << e
+      m
+    end
+    group.each { |description, examples| dump_example_summary(description, examples) }
+  end
+
+  def dump_example_summary(description, examples)
+    output.puts description
+    analysis = Flores::RSpec::Analyze::Analysis.new(group_by_result(examples))
+    output.puts(analysis.to_s.gsub(/^/, "  "))
+  end
+
+  def group_by_result(examples) # rubocop:disable Metrics/AbcSize
+    examples.each_with_object(Hash.new { |h, k| h[k] = [] }) do |example, results|
+      if example.metadata[:execution_result].status == :passed
+        results[:success] << [example.metadata[:values], nil]
+      else
+        exception = example.metadata[:execution_result].exception
+        results[exception.class] << [example.metadata[:values], exception]
+      end
+      results
+    end
+  end
+
+  def method_missing(m, *args)
+    p m => args
+  end
+end

data/lib/flores/rspec/stress.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+# This file is part of ruby-flores.
+# Copyright (C) 2015 Jordan Sissel
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+require "flores/namespace"
+require "flores/rspec"
+require "flores/random"
+
+# This module adds helpers useful in doing stress testing within rspec.
+#
+# The number of iterations in a stress test is random.
+#
+# By way of example, let's have a silly test for adding two positive numbers and expecting
+# the result to not be negative:
+#
+#     describe "Addition" do
+#       context "of two positive numbers" do
+#         let(:a) { Flores::Random.number(1..10000) }
+#         let(:b) { Flores::Random.number(1..10000) }
+#         subject { a + b }
+#
+#         # Note the use of 'stress_it' here!
+#         stress_it "should be greater than zero" do
+#           expect(subject).to(be > 0)
+#         end
+#       end
+#     end
+#
+#  Running this:
+#
+#     % rspec 
+#     <lots of dots>
+#
+#     Finished in 0.45412 seconds (files took 0.32963 seconds to load)
+#     4795 examples, 0 failures
+#
+#  In this way, instead of testing 1 fixed case or 1 randomized case, we test
+#  *many* cases in one rspec run.
+module Flores::RSpec::Stress
+  # Wraps `it` and runs the block many times. Each run has will clear the `let` cache.
+  #
+  # The implementation of this is roughly that the given block will be run N times within an `it`:
+  #
+  #     stress_it_internal "my test" do
+  #        expect(...)
+  #      end
+  #
+  # is roughly equivalent to
+  #
+  #     it "my test" do
+  #       1000.times do
+  #         expect(...)
+  #         __memoized.clear
+  #       end
+  #     end
+  #
+  # The intent of this is to allow randomized testing for fuzzing and stress testing
+  # of APIs to help find edge cases and weird behavior.
+  #
+  # The default number of iterations is randomly selected between 1 and 1000 inclusive
+  def stress_it_internal(name, options = {}, &block)
+    stress__iterations = Flores::Random.iterations(options.delete(:stress_iterations) || Flores::RSpec::DEFAULT_ITERATIONS)
+    it(name, options) do
+      # Run the block of an example many times
+      stress__iterations.each do
+        # Run the block within 'it' scope
+        instance_eval(&block)
+
+        # clear the internal rspec `let` cache this lets us run a test
+        # repeatedly with fresh `let` evaluations.
+        # Reference: https://github.com/rspec/rspec-core/blob/5fc29a15b9af9dc1c9815e278caca869c4769767/lib/rspec/core/memoized_helpers.rb#L124-L127
+        __memoized.clear
+      end
+    end # it ...
+  end # def stress_it_internal
+
+  # Generate a random number of copies of a given example.
+  # The idea is to take 1 `it` and run it N times to help tease out failures.
+  # Of course, the teasing requires you have randomized `let` usage, for example:
+  #
+  #     let(:number) { Flores::Random.number(0..200) }
+  #     it "should be less than 100" do
+  #       expect(number).to(be < 100)
+  #     end
+  #
+  # This creates N (random) copies of your spec example. Using `stress_it` is
+  # preferred instead of `stress_it_internal` because this method will cause
+  # before, after, and around clauses to be invoked correctly.
+  def stress_it(name, *args, &block)
+    Flores::Random.iterations(Flores::RSpec.iterations).each do
+      it(name, *args, &block)
+    end # each
+  end  # def stress_it
+end # Flores::RSpec::Stress

data/lib/flores/rspec.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+# This file is part of ruby-flores.
+# Copyright (C) 2015 Jordan Sissel
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# :nodoc:
+require "flores/namespace"
+
+# The root of the rspec helpers the Flores library provides
+module Flores::RSpec
+  DEFAULT_ITERATIONS = 1..1000
+
+  # Sets up rspec with the Flores RSpec helpers. Usage looks like this:
+  #
+  #     RSpec.configure do |config|
+  #       Flores::RSpec.configure(config)
+  #     end
+  def self.configure(rspec_configuration)
+    require "flores/rspec/stress"
+    require "flores/rspec/analyze"
+    rspec_configuration.extend(Flores::RSpec::Stress)
+    rspec_configuration.extend(Flores::RSpec::Analyze)
+  end # def self.configure
+
+  def self.iterations
+    return @iterations if @iterations
+    if ENV["ITERATIONS"]
+      @iterations = 0..ENV["ITERATIONS"].to_i
+    else
+      @iterations = DEFAULT_ITERATIONS
+    end
+    @iterations
+  end
+end # def Flores::RSpec

data/spec/{randomized_spec.rb → flores/random_spec.rb}

@@ -1,3 +1,4 @@
+# encoding: utf-8
 # This file is part of ruby-flores.
 # Copyright (C) 2015 Jordan Sissel
 # 
@@ -13,25 +14,20 @@
 # 
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-#
-# encoding: utf-8
-require "randomized"
-require "rspec/stress_it"
+require "spec_init"
 
-RSpec.configure do |c|
-  c.extend RSpec::StressIt
-end
-
-shared_examples_for String do |variables|
-  analyze_it "should be a String", variables do
+shared_examples_for String do
+  stress_it "should be a String" do
     expect(subject).to(be_a(String))
   end
-  analyze_it "have valid encoding", variables do
+  stress_it "have valid encoding" do
     expect(subject).to(be_valid_encoding)
   end
 end
 
-describe Randomized do
+describe Flores::Random do
+  analyze_results
+
   describe "#text" do
     context "with no arguments" do
       stress_it "should raise ArgumentError" do
@@ -43,30 +39,30 @@ describe Randomized do
       subject { described_class.text(length) }
 
       context "that is positive" do
-        let(:length) { rand(1..1000) }
+        let(:length) { Flores::Random.integer(1..1000) }
         it_behaves_like String, [:length]
-        analyze_it "has correct length", [:length] do
+        stress_it "has correct length" do
           expect(subject.length).to(eq(length))
         end
       end
 
       context "that is negative" do
-        let(:length) { -1 * rand(1..1000) }
-        analyze_it "should raise ArgumentError", [:length] do
+        let(:length) { -1 * Flores::Random.integer(1..1000) }
+        stress_it "should raise ArgumentError" do
           expect { subject }.to(raise_error(ArgumentError))
         end
       end
     end
 
     context "with 1 range argument" do
-      let(:start)  { rand(1..1000) }
-      let(:length) { rand(1..1000) }
+      let(:start)  { Flores::Random.integer(2..1000) }
+      let(:length) { Flores::Random.integer(1..1000) }
       subject { described_class.text(range) }
 
       context "that is ascending" do
         let(:range) { start..(start + length) }
         it_behaves_like String, [:range]
-        analyze_it "should give a string within that length range", [:range] do
+        stress_it "should give a string within that length range" do
           expect(subject).to(be_a(String))
           expect(range).to(include(subject.length))
         end
@@ -74,7 +70,7 @@ describe Randomized do
 
       context "that is descending" do
         let(:range) { start..(start - length) }
-        analyze_it "should raise ArgumentError", [:range] do
+        stress_it "should raise ArgumentError" do
           expect { subject }.to(raise_error(ArgumentError))
         end
       end
@@ -84,49 +80,53 @@ describe Randomized do
   describe "#character" do
     subject { described_class.character }
     it_behaves_like String, [:subject]
-    analyze_it "has length == 1", [:subject] do
+    stress_it "has length == 1" do
       expect(subject.length).to(be == 1)
     end
   end
 
   shared_examples_for Numeric do |type|
-    let(:start) { Randomized.integer(-100_000..100_000) }
-    let(:length) { Randomized.integer(1..100_000) }
+    let(:start) { Flores::Random.integer(-100_000..100_000) }
+    let(:length) { Flores::Random.integer(1..100_000) }
     let(:range) { start..(start + length) }
 
-    analyze_it "should be a #{type}", [:range] do
+    stress_it "should be a #{type}" do
       expect(subject).to(be_a(type))
     end
 
-    analyze_it "should be within the bounds of the given range", [:range] do
+    stress_it "should be within the bounds of the given range" do
       expect(range).to(include(subject))
     end
   end
 
   describe "#integer" do
     it_behaves_like Numeric, Fixnum do
-      subject { Randomized.integer(range) }
+      subject { Flores::Random.integer(range) }
     end
   end
 
   describe "#number" do
     it_behaves_like Numeric, Float do
-      subject { Randomized.number(range) }
+      subject { Flores::Random.number(range) }
     end
   end
 
   describe "#iterations" do
-    let(:start) { Randomized.integer(1..100_000) }
-    let(:length) { Randomized.integer(1..100_000) }
+    let(:start) { Flores::Random.integer(1..1000) }
+    let(:length) { Flores::Random.integer(1..1000) }
     let(:range) { start..(start + length) }
-    subject { Randomized.iterations(range) }
+    subject { Flores::Random.iterations(range) }
 
-    analyze_it "should return an Enumerable", [:range] do
+    stress_it "should return an Enumerable" do
       expect(subject).to(be_a(Enumerable))
     end
 
-    analyze_it "should have a size within the expected range", [:range] do
-      expect(range).to(include(subject.size))
+    stress_it "should have a size within the expected range" do
+      # Ruby 2.0 added Enumerable#size, so we can't use it here.
+      # Meaning `123.times.size` doesn't work. So for this test,
+      # we use small ranges because Enumerable#count actually
+      # counts (via iteration) and is slow on large numbers.
+      expect(range).to(include(subject.count))
     end
   end
 end 

data/spec/flores/rspec/stress_spec.rb

@@ -0,0 +1,87 @@
+# encoding: utf-8
+# This file is part of ruby-flores.
+# Copyright (C) 2015 Jordan Sissel
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+require "spec_init"
+
+Counter = Class.new do
+  attr_reader :value
+  def initialize
+    @value = 0
+  end
+
+  def incr
+    @value += 1
+  end
+
+  def decr
+    @value -= 1
+  end
+end
+
+describe Flores::RSpec::Stress do
+  subject { Counter.new }
+  before do
+    expect(subject.value).to(be == 0)
+    subject.incr
+    expect(subject.value).to(be == 1)
+  end
+  
+  after do
+    expect(subject.value).to(be == 1)
+    subject.decr
+    expect(subject.value).to(be == 0)
+  end
+
+  stress_it "should call all before and after hooks" do
+    expect(subject.value).to(be == 1)
+  end
+
+  describe "level 1" do
+    before do
+      expect(subject.value).to(be == 1)
+      subject.incr
+      expect(subject.value).to(be == 2)
+    end
+    
+    after do
+      expect(subject.value).to(be == 2)
+      subject.decr
+      expect(subject.value).to(be == 1)
+    end
+
+    stress_it "should call all before and after hooks" do
+      expect(subject.value).to(be == 2)
+    end
+
+    describe "level 2" do
+      before do
+        expect(subject.value).to(be == 2)
+        subject.incr
+        expect(subject.value).to(be == 3)
+      end
+      
+      after do
+        expect(subject.value).to(be == 3)
+        subject.decr
+        expect(subject.value).to(be == 2)
+      end
+
+      stress_it "should call all before and after hooks" do
+        expect(subject.value).to(be == 3)
+      end
+    end
+  end
+end

data/spec/spec_init.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+# This file is part of ruby-flores.
+# Copyright (C) 2015 Jordan Sissel
+# 
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+# 
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+require "flores/random"
+require "flores/rspec"
+
+RSpec.configure do |config|
+  Kernel.srand config.seed
+  Flores::RSpec.configure(config)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flores
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Jordan Sissel
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-21 00:00:00.000000000 Z
+date: 2015-02-24 00:00:00.000000000 Z
 dependencies: []
 description: |2
       Add fuzzing, randomization, and stress to your tests.
@@ -23,42 +23,49 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- ".rubocop.yml"
+- .rubocop.yml
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - Makefile
+- README.md
 - examples/analyze_number.rb
 - examples/socket_acceptance_spec.rb
 - examples/socket_analyze_spec.rb
 - examples/socket_stress_spec.rb
 - flores.gemspec
-- lib/randomized.rb
-- lib/rspec/stress_it.rb
-- spec/randomized_spec.rb
-homepage: 
+- lib/flores/namespace.rb
+- lib/flores/random.rb
+- lib/flores/rspec.rb
+- lib/flores/rspec/analyze.rb
+- lib/flores/rspec/formatters/analyze.rb
+- lib/flores/rspec/stress.rb
+- spec/flores/random_spec.rb
+- spec/flores/rspec/stress_spec.rb
+- spec/spec_init.rb
+homepage:
 licenses:
 - AGPL 3.0 - http://www.gnu.org/licenses/agpl-3.0.html
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.3
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
 specification_version: 4
 summary: Fuzz, randomize, and stress your tests
 test_files: []