flores 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f3e39f321f6d9a2ab60eeb8b478e8c140137b54c
+  data.tar.gz: a46b1752bbc387b7007b212a6b84388d69cd75c8
+SHA512:
+  metadata.gz: 25aabaf1ef95b9fa0c0ddca1a137edc2c3677420bebae35b6f3c697b5dd298ce4a3246b69210111f9de2ade5a1b80adc21bf01420bcaa8f0783a8c26bdd3391c
+  data.tar.gz: 6e4ce72dd9e879c63b66aa12a6585c11b1c9011fcbb9afb2121870c52be7a866996f8bedbdec949112310974f0280c969662ad78cc7ae4a80d35f82af68f987b

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+group "development" do
+  gem "rspec", ">= 3.0.0"
+end

data/Gemfile.lock

@@ -0,0 +1,23 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    rspec (3.2.0)
+      rspec-core (~> 3.2.0)
+      rspec-expectations (~> 3.2.0)
+      rspec-mocks (~> 3.2.0)
+    rspec-core (3.2.0)
+      rspec-support (~> 3.2.0)
+    rspec-expectations (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-mocks (3.2.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.2.0)
+    rspec-support (3.2.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rspec (>= 3.0.0)

data/Makefile

@@ -0,0 +1,50 @@
+GEMSPEC=$(shell ls *.gemspec)
+VERSION=$(shell awk -F\" '/spec.version/ { print $$2 }' $(GEMSPEC))
+NAME=$(shell awk -F\" '/spec.name/ { print $$2 }' $(GEMSPEC))
+GEM=$(NAME)-$(VERSION).gem
+
+.PHONY: test
+test:
+	sh notify-failure.sh ruby test/all.rb
+
+.PHONY: testloop
+testloop:
+	while true; do \
+		$(MAKE) test; \
+		$(MAKE) wait-for-changes; \
+	done
+
+.PHONY: serve-coverage
+serve-coverage:
+	cd coverage; python -mSimpleHTTPServer
+
+.PHONY: wait-for-changes
+wait-for-changes:
+	-inotifywait --exclude '\.swp' -e modify $$(find $(DIRS) -name '*.rb'; find $(DIRS) -type d)
+
+.PHONY: package
+package: | $(GEM)
+
+.PHONY: gem
+gem: $(GEM)
+
+$(GEM):
+	gem build $(GEMSPEC)
+
+.PHONY: test-package
+test-package: $(GEM)
+	# Sometimes 'gem build' makes a faulty gem.
+	gem unpack $(GEM)
+	rm -rf $(NAME)-$(VERSION)/
+
+.PHONY: publish
+publish: test-package
+	gem push $(GEM)
+
+.PHONY: install
+install: $(GEM)
+	gem install $(GEM)
+
+.PHONY: clean
+clean:
+	-rm -rf .yardoc $(GEM) $(NAME)-$(VERSION)/

data/examples/analyze_number.rb

@@ -0,0 +1,13 @@
+require "randomized"
+require "rspec/stress_it"
+
+RSpec.configure do |c|
+  c.extend RSpec::StressIt
+end
+
+describe "number" do
+  let(:number) { Randomized.number(0..200) }
+  analyze_it "should be less than 100", [:number] do
+    expect(number).to(be < 100)
+  end
+end

data/examples/socket_acceptance_spec.rb

@@ -0,0 +1,85 @@
+require "randomized"
+require "socket"
+require "rspec/stress_it"
+
+RSpec.configure do |c|
+  c.extend RSpec::StressIt
+end
+
+class TCPIntegrationTestFactory
+  def initialize(port)
+    @listener = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+    @client = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+    @port = port
+  end
+
+  def teardown
+    @listener.close unless @listener.closed?
+    @client.close unless @listener.closed?
+  end
+
+  def sockaddr
+    Socket.sockaddr_in(@port, "127.0.0.1")
+  end
+
+  def setup
+    @listener.bind(sockaddr)
+    @listener.listen(5)
+  end
+
+  def send_and_receive(text)
+    @client.connect(sockaddr)
+    server, _ = @listener.accept
+
+    @client.syswrite(text)
+    @client.close
+    #expect(client.syswrite(text)).to(be == text.bytesize)
+    server.read
+  ensure
+    @client.close unless @client.closed?
+    server.close unless server.nil? || server.closed?
+  end
+end
+
+describe "TCPServer+TCPSocket" do
+  let(:port) { Randomized.number(1024..65535) }
+  let(:text) { Randomized.text(1..10000) }
+  subject { TCPIntegrationTestFactory.new(port) }
+
+  #describe "using before/after and stress_it2" do
+    #before do
+      #begin
+        #subject.setup
+      #rescue Errno::EADDRINUSE
+        ## We chose a random port that was already in use, let's skip this test.
+        #skip("Port #{port} is in use by another process, skipping")
+      #end
+    #end
+
+    #after do
+      #subject.teardown
+    #end
+
+    #stress_it2 "should send data correctly" do
+      #received = subject.send_and_receive(text)
+      #expect(received).to(be == text)
+    #end
+  #end
+  
+  describe "using stress_it" do
+    stress_it "should send data correctly" do
+      begin
+        subject.setup
+      rescue Errno::EADDRINUSE
+        next # Skip port bindings that are in use
+      end
+
+      begin
+        received = subject.send_and_receive(text)
+        expect(received).to(be == text)
+      ensure
+        subject.teardown
+      end
+    end
+  end
+end

data/examples/socket_analyze_spec.rb

@@ -0,0 +1,21 @@
+require "randomized"
+require "socket"
+require "rspec/stress_it"
+
+RSpec.configure do |c|
+  c.extend RSpec::StressIt
+end
+
+describe TCPServer do
+  subject(:socket) { Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0) }
+  let(:sockaddr) { Socket.sockaddr_in(port, "127.0.0.1") }
+  after { socket.close }
+
+  context "on a random port" do
+    let(:port) { Randomized.number(-100000..100000) }
+    analyze_it "should bind successfully", [:port] do
+      socket.bind(sockaddr)
+      expect(socket.local_address.ip_port).to(be == port)
+    end
+  end
+end

data/examples/socket_stress_spec.rb

@@ -0,0 +1,49 @@
+require "randomized"
+require "socket"
+require "rspec/stress_it"
+
+RSpec.configure do |c|
+  c.extend RSpec::StressIt
+end
+
+describe TCPServer do
+  subject(:socket) { Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0) }
+  let(:sockaddr) { Socket.sockaddr_in(port, "127.0.0.1") }
+  let(:ignore_eaddrinuse) do
+    proc do |m, *args|
+      begin
+        m.call(*args)
+      rescue Errno::EADDRINUSE
+        # ignore
+      end
+    end
+  end
+
+  after do
+    socket.close
+  end
+
+  context "on privileged ports" do
+    let(:port) { Randomized.number(1..1023) }
+    stress_it "should raise Errno::EACCESS" do
+      expect { socket.bind(sockaddr) }.to(raise_error(Errno::EACCES))
+    end
+  end
+
+  context "on unprivileged ports" do
+    let(:port) { Randomized.number(1025..65535) }
+    stress_it "should bind on a port" do
+      # EADDRINUSE is expected since we are picking ports at random
+      # Let's ignore this specific exception
+      allow(socket).to(receive(:bind).and_wrap_original(&ignore_eaddrinuse))
+      expect { socket.bind(sockaddr) }.to_not(raise_error)
+    end
+  end
+
+  context "on port 0" do
+    let(:port) { 0 }
+    stress_it "should bind successfully" do
+      expect { socket.bind(sockaddr) }.to_not(raise_error)
+    end
+  end
+end

data/flores.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |spec|
+  files = %x{git ls-files}.split("\n")
+
+  spec.name = "flores"
+  spec.version = "0.0.1"
+  spec.summary = "Fuzz, randomize, and stress your tests"
+  spec.description = <<-DESCRIPTION
+    Add fuzzing, randomization, and stress to your tests.
+
+    This library is an exploration to build the tools to let you write tests
+    that find bugs.
+
+    In memory of Carlo Flores.
+  DESCRIPTION
+  spec.license = "AGPL 3.0 - http://www.gnu.org/licenses/agpl-3.0.html"
+
+  spec.files = files
+  spec.require_paths << "lib"
+
+  spec.authors = ["Jordan Sissel"]
+  spec.email = ["jls@semicomplete.com"]
+end
+

data/lib/randomized.rb

@@ -0,0 +1,67 @@
+# A collection of methods intended for use in randomized testing.
+module Randomized
+  # Generates text with random characters of a given length (or within a length range)
+  #
+  # * The length can be a number or a range `x..y`. If a range, it must be ascending (x < y)
+  # * Negative lengths are not permitted and will raise an ArgumentError
+  #
+  # @param length [Fixnum or Range] the length of text to generate
+  # @return [String] the 
+  def self.text(length)
+    if length.is_a?(Range)
+      raise ArgumentError, "Requires ascending range, you gave #{length}." if length.end < length.begin
+      raise ArgumentError, "A negative length is not permitted, I received range #{length}" if length.begin < 0
+
+      length = integer(length)
+    else
+      raise ArgumentError, "A negative length is not permitted, I received #{length}" if length < 0
+    end
+
+    length.times.collect { character }.join
+  end # def text
+
+  # Generates a random character (A string of length 1)
+  #
+  # @return [String]
+  def self.character
+    # TODO(sissel): Add support to generate valid UTF-8. I started reading
+    # Unicode 7 (http://www.unicode.org/versions/Unicode7.0.0/) and after much
+    # reading, I realized I wasn't in my house anymore but had somehow lost
+    # track of time and was alone in a field. Civilization had fallen centuries
+    # ago. :P
+    
+    # Until UTF-8 is supported, just return a random lower ASCII character
+    integer(32..127).chr
+  end # def character
+
+  # Return a random integer value within a given range.
+  #
+  # @param range [Range]
+  def self.integer(range)
+    raise ArgumentError, "Range not given, got #{range.class}: #{range.inspect}" if !range.is_a?(Range)
+    rand(range)
+  end # def integer
+
+  # Return a random number within a given range.
+  #
+  # @param range [Range]
+  def self.number(range)
+    raise ArgumentError, "Range not given, got #{range.class}: #{range.inspect}" if !range.is_a?(Range)
+    # Range#size returns the number of elements in the range, not the length of the range.
+    # This makes #size return (abs(range.begin - range.end) + 1), and we want the length, so subtract 1.
+    rand * (range.size - 1) + range.begin
+  end # def number
+   
+  # Run a block a random number of times.
+  #
+  # @param range [Fixnum of Range] same meaning as #integer(range)
+  def self.iterations(range, &block)
+    range = 0..range if range.is_a?(Numeric)
+    if block_given?
+      integer(range).times(&block)
+      nil
+    else
+      integer(range).times
+    end
+  end # def iterations
+end

data/lib/rspec/stress_it.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+require "rspec/core"
+
+module RSpec::StressIt
+  DEFAULT_ITERATIONS = 1..5000
+
+  # Wraps `it` and runs the block many times. Each run has will clear the `let` cache.
+  #
+  # The intent of this is to allow randomized testing for fuzzing and stress testing
+  # of APIs to help find edge cases and weird behavior.
+  #
+  # The default number of iterations is randomly selected between 1 and 1000 inclusive
+  def stress_it(name, options={}, &block)
+    __iterations = Randomized.iterations(options.delete(:stress_iterations) || DEFAULT_ITERATIONS)
+    it(name, options) do
+      # Run the block of an example many times
+      __iterations.each do |i|
+        # Run the block within 'it' scope
+        instance_eval(&block)
+
+        # clear the internal rspec `let` cache this lets us run a test
+        # repeatedly with fresh `let` evaluations.
+        # Reference: https://github.com/rspec/rspec-core/blob/5fc29a15b9af9dc1c9815e278caca869c4769767/lib/rspec/core/memoized_helpers.rb#L124-L127
+        __memoized.clear
+      end
+    end # it ...
+  end # def stress_it
+
+  # Generate a random number of copies of a given example.
+  # The idea is to take 1 `it` and run it N times to help tease out failures.
+  # Of course, the teasing requires you have randomized `let` usage, for example:
+  #
+  #     let(:number) { Randomized.number(0..200) }
+  #     it "should be less than 100" do
+  #       expect(number).to(be < 100)
+  #     end
+  def stress_it2(name, options={}, &block)
+    __iterations = Randomized.iterations(options.delete(:stress_iterations) || DEFAULT_ITERATIONS)
+    __iterations.each do |i|
+      it(example_name + " [#{i}]", *args) do
+        instance_eval(&block)
+      end # it ...
+    end # .times
+  end 
+
+  # Perform analysis on failure scenarios of a given example
+  #
+  # This will run the given example a random number of times and aggregate the
+  # results. If any failures occur, the spec will fail and a report will be
+  # given on that test.
+  #
+  # Example spec:
+  #
+  #     let(:number) { Randomized.number(0..200) }
+  #     fuzz "should be less than 100" do
+  #       expect(number).to(be < 100)
+  #     end
+  #
+  # Example report:
+  def analyze_it(name, variables, &block)
+    it(name) do
+      results = Hash.new { |h,k| h[k] = [] }
+      iterations = Randomized.iterations(DEFAULT_ITERATIONS)
+      iterations.each do |i|
+        state = Hash[variables.collect { |l| [l, __send__(l)] }]
+        begin
+          instance_eval(&block)
+          results[:success] << [state, nil]
+        rescue => e
+          results[e.class] << [state, e]
+        rescue Exception => e
+          results[e.class] << [state, e]
+        end
+
+        # Clear `let` memoizations
+        __memoized.clear
+      end
+
+      if results[:success] != iterations
+        raise Analysis.new(results)
+      end
+    end
+  end
+
+  class Analysis < StandardError
+    def initialize(results)
+      @results = results
+    end
+
+    def total
+      @results.reduce(0) { |m, (k,v)| m + v.length }
+    end
+
+    def success_count
+      if @results.include?(:success)
+        @results[:success].length
+      else
+        0
+      end
+    end
+
+    def percent(count)
+      return (count + 0.0) / total
+    end
+
+    def percent_s(count)
+      return sprintf("%.2f%%", percent(count) * 100)
+    end
+
+    def to_s
+      # This method is crazy complex for a formatter. Should refactor this significantly.
+      report = ["#{percent_s(success_count)} tests successful of #{total} tests"]
+      if success_count < total
+        report << "Failure analysis:"
+        report += @results.sort_by { |k,v| -v.length }.reject { |k,v| k == :success }.collect do |k, v|
+          sample = v.sample(5).collect { |v| v.first }.join(", ")
+          [ 
+            "  #{percent_s(v.length)} -> [#{v.length}] #{k}",
+            "    Sample exception:",
+            v.sample(1).first[1].to_s.gsub(/^/, "      "),
+            "    Samples causing #{k}:",
+            *v.sample(5).collect { |state, _exception| "      #{state}" }
+          ]
+        end.flatten
+      end
+      report.join("\n")
+    end
+  end
+end # module RSpec::StressIt

data/spec/randomized_spec.rb

@@ -0,0 +1,95 @@
+require "randomized"
+require "rspec/stress_it"
+
+RSpec.configure do |c|
+  c.extend RSpec::StressIt
+end
+
+describe Randomized do
+  describe "#text" do
+    context "with no arguments" do
+      it "should raise ArgumentError" do
+        expect { subject.text }.to(raise_error(ArgumentError))
+      end
+    end
+
+    context "with 1 length argument" do
+      subject { described_class.text(length) }
+
+      context "that is positive" do
+        let(:length) { rand(1..1000) }
+        stress_it "should give a string with that length" do
+          expect(subject).to(be_a(String))
+          expect(subject.length).to(eq(length))
+        end
+      end
+
+      context "that is negative" do
+        let(:length) { -1 * rand(1..1000) }
+        stress_it "should raise ArgumentError" do
+          expect { subject }.to(raise_error(ArgumentError))
+        end
+      end
+    end
+
+    context "with 1 range argument" do
+      let(:start)  { rand(1..1000) }
+      let(:length) { rand(1..1000) }
+      subject { described_class.text(range) }
+
+      context "that is ascending" do
+        let(:range) { start .. (start + length) }
+        stress_it "should give a string within that length range" do
+          expect(subject).to(be_a(String))
+          expect(range).to(include(subject.length))
+        end
+      end
+
+      context "that is descending" do
+        let(:range) { start .. (start - length) }
+        stress_it "should raise ArgumentError" do
+          expect { subject }.to(raise_error(ArgumentError))
+        end
+      end
+    end
+  end
+
+  describe "#character" do
+    subject { described_class.character }
+    stress_it "returns a string of length 1" do
+      expect(subject.length).to(be == 1)
+    end
+  end
+
+  shared_examples_for "random numbers within a range" do
+    let(:start) { Randomized.integer(-100000 .. 100000) }
+    let(:length) { Randomized.integer(1 .. 100000) }
+    let(:range) { start .. (start + length) }
+
+    stress_it "should be a Numeric" do
+      expect(subject).to(be_a(Numeric))
+    end
+
+    stress_it "should be within the bounds of the given range" do
+      expect(range).to(include(subject))
+    end
+  end
+
+  describe "#integer" do
+    it_behaves_like "random numbers within a range" do
+      subject { Randomized.integer(range) }
+      stress_it "is a Fixnum" do
+        expect(subject).to(be_a(Fixnum))
+      end
+    end
+  end
+  describe "#number" do
+    it_behaves_like "random numbers within a range" do
+      subject { Randomized.number(range) }
+
+      stress_it "is a Float" do
+        expect(subject).to(be_a(Float))
+      end
+    end
+  end
+end 

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification
+name: flores
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jordan Sissel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-19 00:00:00.000000000 Z
+dependencies: []
+description: |2
+      Add fuzzing, randomization, and stress to your tests.
+
+      This library is an exploration to build the tools to let you write tests
+      that find bugs.
+
+      In memory of Carlo Flores.
+email:
+- jls@semicomplete.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- Makefile
+- examples/analyze_number.rb
+- examples/socket_acceptance_spec.rb
+- examples/socket_analyze_spec.rb
+- examples/socket_stress_spec.rb
+- flores.gemspec
+- lib/randomized.rb
+- lib/rspec/stress_it.rb
+- spec/randomized_spec.rb
+homepage: 
+licenses:
+- AGPL 3.0 - http://www.gnu.org/licenses/agpl-3.0.html
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.3
+signing_key: 
+specification_version: 4
+summary: Fuzz, randomize, and stress your tests
+test_files: []